@ -160,38 +160,10 @@ func (svc *ExchangeService) consumerKline(exchange *Exchange, c <-chan *types.Ch
// 升序排序
collect . SortAsc ( channelK . Klines , func ( k * types . Kline ) int64 { return k . Ts } )
// k线完整性检查, k线是否连续并补齐
padding := false
for _ , kline := range channelK . Klines {
if kline . Confirm {
if kms , ok := kline . Interval . AddMul ( kline . Ts , 1 ) ; ok {
delay := time . Now ( ) . UnixMilli ( ) - kms
zlog . Debugf ( "recv confirm kline: delay=%dms, inst=%s(%s), interval=%s" , delay , channelK . ExgInstId , channelK . Exchange , kline . Interval )
if lastConfirmK := exchangeInst . LastKline . Get ( kline . Interval ) ; lastConfirmK . Ts != 0 {
if expectTs , ok := lastConfirmK . Interval . AddMul ( lastConfirmK . Ts , 1 ) ; ok && expectTs != kline . Ts {
padding = true
startTs := time . Now ( ) . UnixMilli ( )
zlog . Warningf ( "fetching padding klines: inst=%s(%s), interval=%s, ts=%d~%d" , tradeInst . InstId , tradeInst . Exchange , kline . Interval , kline . Ts , lastConfirmK . Ts )
if err := svc . paddingTradeInstanceIntervalKlines ( 4 , exchange , * tradeInst , kline . Interval ) ; err != nil {
zlog . Errorf ( "fetch padding kline error: inst=%s(%s), interval=%s, ts=%d~%d, error=%v" , tradeInst . InstId , tradeInst . Exchange , kline . Interval , kline . Ts , lastConfirmK . Ts , err )
} else {
padding = false
zlog . Infof ( "fetched padding klines: inst=%s(%s), interval=%s, ts=%d~%d, use=%dms" , tradeInst . InstId , tradeInst . Exchange , kline . Interval , kline . Ts , lastConfirmK . Ts , time . Now ( ) . UnixMilli ( ) - startTs )
// flush vmtsdb to disk
if err = svc . exchangeDataPersist . vmtsdb . ForceFlush ( ) ; err != nil {
zlog . Errorf ( "flush vmts db error: " , err )
}
}
}
}
}
break
}
}
// 取出头尾k线
lastKline := channelK . Klines [ len ( channelK . Klines ) - 1 ]
interval := lastKline . Interval
intervalAdder , intervalSupport := types . SupportedIntervals [ interval ]
// 记录实时k线
exchangeInst . LiveKline . Set ( lastKline . Interval , * lastKline )
@ -204,8 +176,11 @@ func (svc *ExchangeService) consumerKline(exchange *Exchange, c <-chan *types.Ch
// zlog.Infof("recv kline: %#v", kline)
confirm := 0
if kline . Confirm {
// 记录最后确认k线
exchangeInst . LastKline . Set ( kline . Interval , * kline )
if intervalSupport {
delay := time . Now ( ) . UnixMilli ( ) - intervalAdder ( kline . Ts , 1 )
zlog . Debugf ( "recv confirm kline: delay=%dms, inst=%s(%s), interval=%s" , delay , tradeInst . InstId , tradeInst . Exchange , kline . Interval )
}
confirm = 1
confirmKlines = append ( confirmKlines , kline )
}
@ -221,6 +196,25 @@ func (svc *ExchangeService) consumerKline(exchange *Exchange, c <-chan *types.Ch
msg . Klines = append ( msg . Klines , pbk )
}
padding := true
if len ( confirmKlines ) > 0 {
if intervalSupport {
// k线完整性检查, k线是否连续并补齐
if lastConfirmK := exchangeInst . LastKline . Get ( interval ) ; lastConfirmK . Ts != 0 {
tempK := make ( [ ] * types . Kline , 0 , len ( confirmKlines ) + 1 )
tempK = append ( tempK , & lastConfirmK )
tempK = append ( tempK , confirmKlines ... )
if err := svc . paddingKlinesIfNotSeries ( exchange , tradeInst . InstId , confirmKlines [ 0 ] . Interval , tempK ) ; err != nil {
padding = false
zlog . Errorf ( "try padding klines error: inst=%s(%s), interval=%s, ts=%d~%d, %v" , tradeInst . InstId , tradeInst . Exchange , lastKline . Interval , lastKline . Ts , lastConfirmK . Ts , err )
}
}
}
// 记录最后确认k线
exchangeInst . LastKline . Set ( interval , * confirmKlines [ len ( confirmKlines ) - 1 ] )
}
// 存储到 tsdb
if _ , ok := types . SupportedIntervals [ lastKline . Interval ] ; ok && len ( confirmKlines ) > 0 {
// tsdb storage todo 异步处理
@ -230,7 +224,7 @@ func (svc *ExchangeService) consumerKline(exchange *Exchange, c <-chan *types.Ch
zlog . Errorf ( "kline save to tsdb error: %v, %#v" , err , confirmKlines )
} else {
// k线未缺失, 初始化状态完成, 更新k线时间戳标记
if ! padding && exchangeInst . Status . Load ( ) == int32 ( data . StatusOk ) {
if intervalSupport && ! padding && exchangeInst . Status . Load ( ) == int32 ( data . StatusOk ) {
latestK := collect . MustMax ( confirmKlines , func ( k * types . Kline ) int64 { return k . Ts } )
// 标记确认k线
tsKey , ex := svc . exchangeDataPersist . SaveHistoryKlineMarkTs ( tradeInst . Exchange , tradeInst . InstId , latestK . Interval , latestK . Ts )
@ -329,7 +323,7 @@ func (svc *ExchangeService) paddingTradeInstanceKlines(exchange *Exchange, trade
// flush vmtsdb to disk
retry . DoWithFixDelay ( 5 , time . Second , func ( retryTimes uint32 ) ( _ struct { } , err error ) {
if err = svc . exchangeDataPersist . vmtsdb . ForceF lush ( ) ; err != nil {
if err = svc . exchangeDataPersist . Flush0 ( ) ; err != nil {
zlog . Errorf ( "flush vmts db error: " , err )
}
return
@ -556,6 +550,46 @@ func (svc *ExchangeService) fetchTaskKlinesToTSDB(exchange *Exchange, task fetch
return
}
// paddingKlinesIfNotSeries 如k线不连续, 从缺失处进行补齐
func ( svc * ExchangeService ) paddingKlinesIfNotSeries ( exchange * Exchange , instId string , interval types . Interval , klines [ ] * types . Kline ) ( err error ) {
// 检查k线是否连续
paddingMarkTs := int64 ( 0 )
for i , k := range klines {
if i > 0 && k . Ts != interval . MustAddMul ( klines [ i - 1 ] . Ts , 1 ) {
paddingMarkTs = klines [ i - 1 ] . Ts
break
}
}
if paddingMarkTs == 0 {
return
}
// k线不连续进行补齐
exchangeInstId , _ := exchange . TradeInstIds . Load ( instId )
exchangeInst , ok := exchange . ExchangeInsts . Load ( exchangeInstId )
if ! ok {
err = fmt . Errorf ( "trade instance not support for exchange: %s(%s)" , instId , exchange . ExchangeType . String ( ) )
return
}
if exchangeInst . Status . CompareAndSwap ( int32 ( data . StatusOk ) , int32 ( data . StatusProcessing ) ) {
defer exchangeInst . Status . CompareAndSwap ( int32 ( data . StatusProcessing ) , int32 ( data . StatusOk ) )
watch := times . NewWatch ( )
zlog . Warningf ( "vmtsdb kline not series, try padding: instId=%s(%s), interval=%s, ts=%d" , instId , exchange . ExchangeType . String ( ) , interval , paddingMarkTs )
if _ , err = svc . exchangeDataPersist . SaveHistoryKlineMarkTs ( exchange . ExchangeType , instId , interval , paddingMarkTs ) ; err != nil {
zlog . Errorf ( "try padding series save markTs error: " , err )
return
}
if err = svc . paddingTradeInstanceIntervalKlines ( 4 , exchange , * exchangeInst . Inst , interval ) ; err != nil {
zlog . Errorf ( "try padding series fetch to vmtsdb error: " , err )
return
}
// flush vmtsdb to disk
svc . exchangeDataPersist . Flush ( )
zlog . Debugf ( "vmtsdb kline not series padding success: instId=%s(%s), interval=%s, ts=%d, use %s" , instId , exchange . ExchangeType . String ( ) , interval , paddingMarkTs , watch . ElapsedFmt ( "." ) )
}
return
}
// Exchanges 支持的交易所列表
func ( svc * ExchangeService ) Exchanges ( ) ( exchanges [ ] pb . ExchangeType , err error ) {
svc . exchanges . Range ( func ( exchange pb . ExchangeType , _ * Exchange ) {
@ -725,7 +759,7 @@ func (svc *ExchangeService) HistoryKline(ctx context.Context, req *pb.ReqHistory
return
}
// 查询历史k线(流式返回)
// 查询历史k线(按时间升序 流式返回)
func ( svc * ExchangeService ) HistoryKlineStream ( req * pb . ReqHistoryKlineStream , stream grpc . ServerStreamingServer [ pb . RspHistoryKlineStream ] ) ( err error ) {
// 交易产品参数检查
if ! svc . exchanges . IsSupport ( req . Exchange ) {
@ -784,7 +818,11 @@ func (svc *ExchangeService) HistoryKlineStream(req *pb.ReqHistoryKlineStream, st
if len ( klines ) == 0 {
return
}
// todo 检查k线是否连续进行补齐
// 检查k线是否连续进行补齐
if err = svc . paddingKlinesIfNotSeries ( exchange , req . InstId , interval , klines ) ; err != nil {
return
}
lastK := klines [ len ( klines ) - 1 ]
// vmtsdb 数据刷盘30s延迟, 使用内存数据替代第一根k线