sig-pub/internal/exchange/exchange_service.go

package exchange

import (
	"context"
	"errors"
	"fmt"
	"runtime"
	"sig-pub/api/pb"
	"sig-pub/pkg/aside"
	"sig-pub/pkg/data"
	"sig-pub/pkg/types"
	"sig-pub/pkg/utils/collect"
	"sig-pub/pkg/utils/times"
	"sig-pub/pkg/zlog"
	"sort"
	"sync"
	"sync/atomic"
	"time"

	"google.golang.org/grpc"
)

// ExchangeService 交易所服务
type ExchangeService struct {
	exchangeMap         map[pb.ExchangeType]*Exchange
	tradeInstanceAside  *aside.TradeInstanceAside
	exchangeDataPersist *ExchangeDataPersist

	klinePublisher *Publisher[int64, grpc.BidiStreamingServer[pb.ReqStreamSubscribeKline, pb.RspStreamSubscribeKline]]
}

// exchanges: 支持的数据源交易所
func NewExchangeService(
	tradeInstanceAside *aside.TradeInstanceAside,
	exchangeDataPersist *ExchangeDataPersist,
	exchanges ...*Exchange,
) *ExchangeService {
	exchangeMap := make(map[pb.ExchangeType]*Exchange)
	for _, exchange := range exchanges {
		exchangeMap[exchange.ExchangeType] = exchange
	}

	return &ExchangeService{
		exchangeMap:         exchangeMap,
		tradeInstanceAside:  tradeInstanceAside,
		exchangeDataPersist: exchangeDataPersist,
		klinePublisher:      NewPublisher[int64, grpc.BidiStreamingServer[pb.ReqStreamSubscribeKline, pb.RspStreamSubscribeKline]](16),
	}
}

func (svc *ExchangeService) Init() (err error) {
	svc.subscribeExchanges()
	return
}

// GetKlineSubscriber 订阅k线订阅器
func (svc *ExchangeService) GetKlineSubscriber() (subscriber *Publisher[int64, grpc.BidiStreamingServer[pb.ReqStreamSubscribeKline, pb.RspStreamSubscribeKline]]) {
	subscriber = svc.klinePublisher
	return
}

// 订阅交易所推送行情
func (svc *ExchangeService) subscribeExchanges() {
	// 交易所订阅交易产品
	for _, exchange := range svc.exchangeMap {
		go func(exchange *Exchange) {
			// get exchange all trade instances
			insts, err := svc.tradeInstanceAside.ListExchangeTradeInstance(context.Background(), exchange.ExchangeType)
			if err != nil {
				zlog.Error(err)
				return
			}

			var exchangeInstIds []string
			var processingInsts []types.TradeInstance
			for _, inst := range insts {
				exchangeInstIds = append(exchangeInstIds, inst.ExchangeInstId)
				tradeInst := &types.TradeInstance{
					InstId:         inst.InstId,
					Status:         inst.Status,
					PriceSz:        inst.PriceSz,
					QuantitySz:     inst.QuantitySz,
					ExchangeInstId: inst.ExchangeInstId,
					Leverages:      inst.Leverages,
					Exchange:       exchange.ExchangeType,
				}
				// 待初始化币种数据
				processingInsts = append(processingInsts, *tradeInst)

				exchange.TradeInstIds.Store(inst.InstId, inst.ExchangeInstId)

				exchangeInst := &ExchangeTradeInstance{
					Inst:          tradeInst,
					LiveKline:     types.NewIntervalState[types.Kline](),
					LiveKStartTs:  types.NewIntervalState[int64](),
					HistoryMarkTs: types.NewIntervalState[int64](),
				}
				exchangeInst.Status.Store(int32(data.StatusProcessing))
				exchange.ExchangeInsts.Store(inst.ExchangeInstId, exchangeInst)
			}

			// 订阅实时k线数据
			err = exchange.Subscriber.SubscribeKline(exchangeInstIds...) // []string{"BTC-USDT", "DOGE-USDT-SWAP"}
			if err != nil {
				zlog.Error(err)
				return
			}
			// 消费实时k线数据
			go func() {
				c := exchange.Subscriber.ConsumerKline()
				svc.consumerKline(exchange, c)
				// todo subscribe books 订单簿
				zlog.Infof("unsubscribe exchange: %s", exchange.ExchangeType)
			}()

			// 初始化历史k线数据
			go svc.initialKlines(exchange, processingInsts)
		}(exchange)
	}
}

// consumerKline 消费交易所k线数据
func (svc *ExchangeService) consumerKline(exchange *Exchange, c <-chan *types.ChannelKline) {
	exchangeType := exchange.ExchangeType

	// publish to subscribers
	pubStreamKlineMap := make(map[string]*pb.StreamKline)

	for {
		clear(pubStreamKlineMap)

		channelK, ok := <-c
		if !ok {
			return
		}
		if len(channelK.Klines) == 0 {
			continue
		}
		if len(channelK.Klines) > 1 {
			sort.Slice(channelK.Klines, func(i, j int) bool {
				return channelK.Klines[i].Ts < channelK.Klines[j].Ts
			})
		}
		firstKline, lastKline := channelK.Klines[0], channelK.Klines[len(channelK.Klines)-1]

		// 交易所 instid 转 sig-instid
		var tradeInst *types.TradeInstance
		exchangeInst, ok := exchange.ExchangeInsts.Load(channelK.ExgInstId)
		if !ok || exchangeInst == nil || exchangeInst.Inst == nil {
			zlog.Errorf("unknown exchange instId: %v, %s", channelK.Exchange, channelK.ExgInstId)
			continue
		}
		tradeInst = exchangeInst.Inst

		// 标记交易产品开始订阅k线时间
		exchangeInst.LiveKStartTs.SetIf(firstKline.Interval, firstKline.Ts, func(old int64) bool { return old == 0 })

		// 标记实时k线
		exchangeInst.LiveKline.Set(lastKline.Interval, *lastKline)
		// 记录实时价格
		exchangeInst.Last = lastKline.Close

		var confirmKlines []*types.Kline
		for _, kline := range channelK.Klines {
			// zlog.Infof("recv kline: %#v", kline)
			confirm := 0
			if kline.Confirm {
				confirm = 1
				confirmKlines = append(confirmKlines, kline)
			}
			pubKey := fmt.Sprintf("/kline/%s/%s/%s/%d", exchangeType, tradeInst.InstId, kline.Interval, confirm)
			msg, ok := pubStreamKlineMap[pubKey]
			if !ok {
				msg = new(pb.StreamKline)
				msg.InstId = tradeInst.InstId
				msg.Exchange = channelK.Exchange
				pubStreamKlineMap[pubKey] = msg
			}
			pbk := kline.ToPBKline()
			msg.Klines = append(msg.Klines, pbk)
		}

		if len(confirmKlines) > 0 {
			// tsdb storage todo 异步处理
			err := svc.exchangeDataPersist.SaveKline(*tradeInst, confirmKlines)
			if err != nil {
				zlog.Errorf("kline save to tsdb error: %v, %#v", err, confirmKlines)
			} else {
				// 初始化状态完成, 更新k线时间戳标记
				if exchangeInst.Status.Load() == int32(data.StatusOk) {
					latestK := collect.MustMax(confirmKlines, func(k *types.Kline) int64 { return k.Ts })
					// 标记确认k线
					tsKey, ex := svc.exchangeDataPersist.SaveHistoryKlineMarkTs(tradeInst.Exchange, tradeInst.InstId, latestK.Interval, latestK.Ts)
					if ex != nil {
						zlog.Errorf("history mark inititaled ts error: key=%s, ts=%d, %v", tsKey, latestK.Ts, ex)
					}
					// todo k线完整性检查
				}
			}
		}

		// publish grpc stream klines
		for pubKey, kline := range pubStreamKlineMap {
			if len(kline.Klines) == 0 {
				continue
			}
			subs := svc.klinePublisher.Publisher(pubKey)
			for _, sub := range subs {
				if err := sub.Send(&pb.RspStreamSubscribeKline{Kline: kline}); err != nil {
					zlog.Error(err)
				}
			}
		}
	}
}

const (
	KlineBefore0                     int64  = 1672502400000               // k线开始数据 2023-01-01 00:00:00 GMT+8
	HistoryKlineTsKey                string = "history-kline-ts:%s:%s:%s" // exchange:sig-instid:interval
	SingleKlineFetchTaskMaxFailTimes int32  = 100                         // 单个k线拉取任务最大失败次数
)

type fetchKlineTask struct {
	inst     types.TradeInstance
	interval types.Interval
	afterTs  int64
	beforeTs int64
	times    int32 // 重试次数
}

func (t fetchKlineTask) logKey() string {
	return fmt.Sprintf("%s:%s:%s:%d:%d", t.inst.Exchange, t.inst.InstId, t.interval, t.beforeTs, t.afterTs)
}

// initialKline 初始化交易产品历史k线数据
func (svc *ExchangeService) initialKlines(exchange *Exchange, insts []types.TradeInstance) {
	// 记录成功和失败的交易产品
	var success, failed []types.TradeInstance

	for _, inst := range insts {
		err := svc.initialTradeInstanceKlines(exchange, inst)
		if err != nil {
			zlog.Errorf("initial fetch trade instance error: %s(%s), err=%v", inst.InstId, inst.Exchange, err)
			failed = append(failed, inst)
		} else {
			success = append(success, inst)
		}
	}

	zlog.Infof("%d insts initial finished, success %d, failed %d", len(insts), len(success), len(failed))
}

// initTradeInstanceKlines 初始化交易产品历史k线数据
func (svc *ExchangeService) initialTradeInstanceKlines(exchange *Exchange, tradeInst types.TradeInstance) (err error) {
	exchangeInst, ok := exchange.ExchangeInsts.Load(tradeInst.ExchangeInstId)
	if !ok {
		err = fmt.Errorf("not load exchange trade instance: %s", tradeInst.ExchangeInstId)
		return
	}

	// 并发数
	concurrent := max(8, runtime.NumCPU()*2)
	// 任务 channel
	taskCh := make(chan fetchKlineTask, concurrent)
	retryTaskCh := make(chan fetchKlineTask, concurrent)

	// 发布任务数, 成功任务数, 失败任务次数
	var pubTasks, subTasks, failTasks atomic.Int32
	var pubTaskDone atomic.Bool // 所有任务已发布
	ctx, cancel := context.WithCancel(context.Background())

	defer func() {
		if err != nil {
			// 交易所k线初始化失败
			exchangeInst.Status.Store(int32(data.StatusFailed))
			return
		}
		exchangeInst.HistoryMarkTs.Range(func(_ int, interval types.Interval, ts int64) {
			tsKey, ex := svc.exchangeDataPersist.SaveHistoryKlineMarkTs(tradeInst.Exchange, tradeInst.InstId, interval, ts)
			if ex != nil {
				zlog.Errorf("history mark inititaled ts error: key=%s, ts=%d, %v", tsKey, ts, ex)
			}
		})
		exchangeInst.Status.Store(int32(data.StatusOk))
	}()

	go func() {
		defer func() {
			pubTaskDone.Store(true)
			// 无任务处理
			if subTasks.Load() == 0 {
				cancel()
			}
			zlog.Infof("trade instance initial kline %s(%s), pub %d fetch tasks", tradeInst.InstId, tradeInst.Exchange, pubTasks.Load())
		}()

		for interval, intervalAdder := range types.SupportedIntervals {
			// interval := types.Interval1d
			// intervalAdder := types.SupportedIntervals[interval]
			// history 未补全前, history写 kvdb ts mark, 补全后 ws live 写 ts mark
			beforeTs, ex := svc.exchangeDataPersist.GetHistoryKlineMarkTs(tradeInst.Exchange, tradeInst.InstId, interval)
			if ex != nil {
				err = ex
				zlog.Error(err)
				cancel()
				return
			}
			if beforeTs == 0 {
				beforeTs = intervalAdder(KlineBefore0, -1)
			}

			exchangeInst.HistoryMarkTs.Set(interval, beforeTs)
			for {
				// 判定订阅任务发布完成
				liveStartTs := exchangeInst.LiveKStartTs.Get(interval)
				if liveStartTs != 0 && beforeTs >= liveStartTs {
					break
				}
				if beforeTs > time.Now().UnixMilli() {
					break
				}

				afterTs := intervalAdder(beforeTs, 101)
				task := fetchKlineTask{
					inst:     tradeInst,
					interval: interval,
					afterTs:  afterTs,
					beforeTs: beforeTs,
					times:    0,
				}

				// 发布任务
				select {
				case taskCh <- task:
					pubTasks.Add(1)
				case <-ctx.Done():
					return
				}

				beforeTs = intervalAdder(afterTs, -1)
			}
		}
	}()

	// 任务消费器 多协程并行
	wg := new(sync.WaitGroup)
	for range concurrent {
		wg.Add(1)
		go func() {
			defer wg.Done()

			var task fetchKlineTask
			for {
				select {
				case <-ctx.Done():
					return
				case task = <-taskCh:
				case task = <-retryTaskCh:
				}

				if task.times > 0 {
					zlog.Infof("retry fetch history kline task %d times: task -> %s", task.times, task.logKey())
				}

				if lastKlineTs, ex := svc.fetchTaskKlinesToTSDB(exchange, task); ex != nil {
					failTasks.Add(1)
					if task.times >= SingleKlineFetchTaskMaxFailTimes {
						err = fmt.Errorf("task failed to many times %d, key: %s, err: %v", task.times, task.logKey(), ex)
						cancel()
						return
					}
					// retry task
					task.times++
					select {
					case retryTaskCh <- task:
					case <-ctx.Done():
						return
					}
				} else {
					// 周期任务最后kline时间
					if lastKlineTs != 0 {
						exchangeInst.HistoryMarkTs.SetIf(task.interval, lastKlineTs, func(old int64) bool {
							return lastKlineTs > old
						})
					}

					// 任务都已执行成功结束
					subs := subTasks.Add(1)
					zlog.Debugf("trade instance initial kline tasks processing: %s(%s), pub %d, sub %d, fail %d", tradeInst.InstId, tradeInst.Exchange, pubTasks.Load(), subTasks.Load(), failTasks.Load())
					if pubTaskDone.Load() && subs >= pubTasks.Load() {
						zlog.Infof("trade instance initial kline tasks success finished, %s(%s), pub %d, sub %d, fail %d", tradeInst.InstId, tradeInst.Exchange, pubTasks.Load(), subTasks.Load(), failTasks.Load())
						cancel()
						return
					}
				}
			}
		}()
	}
	wg.Wait()
	return
}

func (svc *ExchangeService) fetchTaskKlinesToTSDB(exchange *Exchange, task fetchKlineTask) (lastKlineTs int64, err error) {
	interval, afterTs, beforeTs := task.interval, task.afterTs, task.beforeTs
	klines, err := exchange.Fetcher.FetchHistoryKlines(context.Background(), task.inst.ExchangeInstId, interval, afterTs, beforeTs)
	if err != nil {
		zlog.Errorf("fetch history kline task error: task -> %s, err -> %v", task.logKey(), err)
		return
	}
	if len(klines) == 0 {
		return
	}

	loc, _ := time.LoadLocation("Asia/Shanghai")

	sts, ets := klines[0].Ts, klines[len(klines)-1].Ts
	lastKlineTs = max(sts, ets)
	ss := time.UnixMilli(sts).In(loc).Format(times.FORMAT_DATE)
	ee := time.UnixMilli(ets).In(loc).Format(times.FORMAT_DATE)
	// zlog.Infof("fetch interval %s %d~%d klines: ret=%d~%d, %d klines, %s~%s", interval, beforeTs, afterTs, ets, sts, len(klines), ee, ss)
	zlog.Infof("fetch history interval klines: %s, %d klines, %s~%s", task.logKey(), len(klines), ee, ss)

	// store to tsdb
	err = svc.exchangeDataPersist.SaveKline(task.inst, klines)
	if err != nil {
		zlog.Errorf("save history klines to tsdb error: task -> %s, err -> %v", task.logKey(), err)
		return
	}
	return
}

// Exchanges 支持的交易所列表
func (svc *ExchangeService) Exchanges() (exchanges []pb.ExchangeType, err error) {
	for exchange := range svc.exchangeMap {
		exchanges = append(exchanges, exchange)
	}
	return
}

// ExchangeInstanceState 交易所交易产品状态
func (svc *ExchangeService) ExchangeInstanceState(allExchange bool, exchangeTypes []pb.ExchangeType, instIds []string) (states []*pb.TradeInstanceState, err error) {
	var exchanges []*Exchange
	if allExchange {
		for _, exg := range svc.exchangeMap {
			exchanges = append(exchanges, exg)
		}
	} else {
		for _, exchangeType := range exchangeTypes {
			exg, ok := svc.exchangeMap[exchangeType]
			if !ok {
				err = fmt.Errorf("not support exchange: %v", exchangeType)
				return
			}
			exchanges = append(exchanges, exg)
		}
	}
	if len(exchanges) == 0 {
		err = errors.New("no support exchanges")
		return
	}

	for _, exchange := range exchanges {
		for _, instId := range instIds {
			// trade instId to exchangeInstId
			exchangeInstId, ok := exchange.TradeInstIds.Load(instId)
			if !ok {
				continue
			}
			inst, ok := exchange.ExchangeInsts.Load(exchangeInstId)
			if !ok {
				continue
			}

			state := &pb.TradeInstanceState{
				Exchange: exchange.ExchangeType,
				InstId:   instId,
				Last:     inst.Last.String(),
			}
			states = append(states, state)
		}
	}
	return
}

// HistoryKline 获取交易产品历史k线 (before < klines... < after)
func (svc *ExchangeService) HistoryKline(ctx context.Context, req *pb.ReqHistoryKline) (klines []*types.Kline, err error) {
	// 交易产品参数检查
	exchange, ok := svc.exchangeMap[req.Exchange]
	if !ok {
		err = fmt.Errorf("exchange not support: %s", req.Exchange)
		return
	}
	exchangeInstId, ok := exchange.TradeInstIds.Load(req.InstId)
	if !ok {
		err = fmt.Errorf("trade instance not support: %s", req.InstId)
		return
	}
	interval := types.Interval(req.Interval)
	intervalAdder, ok := types.SupportedIntervals[interval]
	if !ok {
		err = fmt.Errorf("interval not support: %s", req.Interval)
		return
	}

	exchangeInst, ok := exchange.ExchangeInsts.Load(exchangeInstId)
	if !ok {
		err = fmt.Errorf("trade instance not support for exchange: %s for %s", req.InstId, req.Exchange)
		return
	}

	// k线长度检查
	afterTs, beforeTs, count := int64(req.After), int64(req.Before), int64(req.Count)
	latest := false
	if afterTs == 0 && beforeTs == 0 {
		afterTs = time.Now().UnixMilli()
		latest = true
	}
	if count == 0 {
		count = 120
	}
	if afterTs == 0 {
		afterTs = intervalAdder(beforeTs, count)
	}
	if beforeTs == 0 {
		beforeTs = intervalAdder(afterTs, -count)
	}
	if beforeTs > afterTs {
		err = fmt.Errorf("time range invalid: before must less then after")
		return
	}
	if beforeTs == afterTs {
		return
	}
	total := (afterTs - beforeTs) / intervalAdder(0, 1)
	if total > 1000 {
		err = fmt.Errorf("time range too large")
		return
	}

	klines, err = svc.exchangeDataPersist.ListKline(*exchangeInst.Inst, interval, beforeTs, afterTs)
	if err != nil || len(klines) == 0 {
		return
	}

	// 开闭区间 confirmed
	if req.Open {
		if klines[0].Ts == afterTs {
			klines = klines[1:]
		}
		if len(klines) > 0 && klines[len(klines)-1].Ts == beforeTs {
			klines = klines[:len(klines)-1]
		}
	}

	if latest && req.Live {
		liveK := exchangeInst.LiveKline.Get(interval)
		klines = append([]*types.Kline{&liveK}, klines...)
	}
	return
}