kline stream padding check

config/exchange.toml

@@ -18,8 +18,8 @@ receiveBuffer = 4096
 marketSubscribeLimit = 16
 consumeBatch = 1024
 consumeLater = 2000 # 时间到达later或者数据累计到batch触发consume
-# httpProxy = "http://192.168.1.5:7890"
+httpProxy = "http://192.168.1.5:7890"
-httpProxy = "http://10.255.183.209:7890"
+# httpProxy = "http://10.255.183.209:7890"
 
 # 模拟盘API交易地址如下：
 # REST：https://www.okx.com

internal/exchange/exchange_service.go

@@ -12,6 +12,7 @@ import (
 	"sig-pub/pkg/publish"
 	"sig-pub/pkg/types"
 	"sig-pub/pkg/utils/collect"
+	"sig-pub/pkg/utils/lang"
 	"sig-pub/pkg/utils/retry"
 	"sig-pub/pkg/utils/times"
 	"sig-pub/pkg/zlog"
@@ -204,7 +205,7 @@ func (svc *ExchangeService) consumerKline(exchange *Exchange, c <-chan *types.Ch
 					tempK := make([]*types.Kline, 0, len(confirmKlines)+1)
 					tempK = append(tempK, &lastConfirmK)
 					tempK = append(tempK, confirmKlines...)
-					if err := svc.paddingKlinesIfNotSeries(exchange, tradeInst.InstId, confirmKlines[0].Interval, tempK); err != nil {
+					if err := svc.paddingKlinesIfNotSeries(exchange, tradeInst.InstId, confirmKlines[0].Interval, tempK, nil, nil); err != nil {
 						padding = false
 						zlog.Errorf("try padding klines error: inst=%s(%s), interval=%s, ts=%d~%d, %v", tradeInst.InstId, tradeInst.Exchange, lastKline.Interval, lastKline.Ts, lastConfirmK.Ts, err)
 					}
@@ -551,13 +552,24 @@ func (svc *ExchangeService) fetchTaskKlinesToTSDB(exchange *Exchange, task fetch
 }
 
 // paddingKlinesIfNotSeries 如k线不连续, 从缺失处进行补齐
-func (svc *ExchangeService) paddingKlinesIfNotSeries(exchange *Exchange, instId string, interval types.Interval, klines []*types.Kline) (err error) {
+func (svc *ExchangeService) paddingKlinesIfNotSeries(exchange *Exchange, instId string, interval types.Interval, klines []*types.Kline, firstKlinePrev, lastKlineNext *types.Kline) (err error) {
+	if len(klines) == 0 {
+		return
+	}
 	// 检查k线是否连续
 	paddingMarkTs := int64(0)
-	for i, k := range klines {
+	if firstKlinePrev != nil && interval.MustAddMul(firstKlinePrev.Ts, 1) != klines[0].Ts {
-		if i > 0 && k.Ts != interval.MustAddMul(klines[i-1].Ts, 1) {
+		paddingMarkTs = firstKlinePrev.Ts
-			paddingMarkTs = klines[i-1].Ts
+	}
-			break
+	if paddingMarkTs == 0 && lastKlineNext != nil && lastKlineNext.Ts != interval.MustAddMul(klines[len(klines)-1].Ts, 1) {
+		paddingMarkTs = klines[len(klines)-1].Ts
+	}
+	if paddingMarkTs == 0 {
+		for i, k := range klines {
+			if i > 0 && k.Ts != interval.MustAddMul(klines[i-1].Ts, 1) {
+				paddingMarkTs = klines[i-1].Ts
+				break
+			}
 		}
 	}
 	if paddingMarkTs == 0 {
@@ -776,6 +788,7 @@ func (svc *ExchangeService) HistoryKline(arg *pb.SeriesRange, recvBranch int, re
 	branch := int64(2000)
 	before, after := beforeTs, afterTs
 	recvBuffer := make([]*pb.Kline, 0, recvBranch)
+	var prevFirstK, prevLastK *types.Kline
 	for range 10000 {
 		if arg.Desc {
 			before = max(intervalAdder(after, -branch+1), beforeTs)
@@ -797,9 +810,13 @@ func (svc *ExchangeService) HistoryKline(arg *pb.SeriesRange, recvBranch int, re
 		}
 
 		// 检查k线是否连续进行补齐
-		if err = svc.paddingKlinesIfNotSeries(exchange, arg.InstId, interval, klines); err != nil {
+		if err = svc.paddingKlinesIfNotSeries(exchange, arg.InstId, interval, klines,
+			lang.Ternary(arg.Desc, nil, prevLastK),
+			lang.Ternary(arg.Desc, prevFirstK, nil),
+		); err != nil {
 			return
 		}
+		prevFirstK, prevLastK = klines[0], klines[len(klines)-1]
 		lastK := klines[len(klines)-1]
 		if after == afterTs && lastK.Ts != afterTs {
 			// vmtsdb 数据落盘30s延迟, 使用内存数据替代最新的一根k线
@@ -835,7 +852,7 @@ func (svc *ExchangeService) HistoryKline(arg *pb.SeriesRange, recvBranch int, re
 			collect.Reverse(klines)
 		}
 
-		zlog.Infof("krange: %s, total=%d, %d~%d", arg.InstId, len(klines), klines[0].Ts, klines[len(klines)-1].Ts)
+		// zlog.Debugf("krange: %s(%s), interval=%s, total=%d, %d~%d", arg.InstId, arg.Exchange, interval, len(klines), klines[0].Ts, klines[len(klines)-1].Ts)
 
 		// 分成小批量recv
 		length := len(klines)