commit acc97b8f8f526560510651101ab693f17cc1aaad Author: strange Date: Thu Jul 24 23:16:05 2025 +0800 okx api, grpc generic, exchange okx subscribe diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c74a22 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# IDE ignore +.idea/ +*.ipr +*.iml +*.iws +.fleet/ + +# temp ignore +*.log +*.cache +*.diff +*.exe +*.exe~ +*.patch +*.tmp +*.swp +*.db + +# system ignore +.DS_Store +Thumbs.db + +# build +/target +/dist +/tmp + +/api/pb +/fs diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..d50f8d7 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,62 @@ +{ + // 使用 IntelliSense 了解相关属性。 + // 悬停以查看现有属性的描述。 + // 欲了解更多信息，请访问: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "sig-admin", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/sig-admin", + "cwd": "${workspaceFolder}", + "output": "${workspaceFolder}/run/sig-admin", + }, + { + "name": "market", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/market", + "cwd": "${workspaceFolder}", + "output": "${workspaceFolder}/run/market", + }, + { + "name": "gateway", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/gateway", + "cwd": "${workspaceFolder}", + "output": "${workspaceFolder}/run/gateway", + }, + { + "name": "exchange", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/exchange", + "cwd": "${workspaceFolder}", + "output": "${workspaceFolder}/run/exchange", + }, + { + "name": "indicator", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/indicator", + "cwd": "${workspaceFolder}", + "output": "${workspaceFolder}/run/indicator", + }, + { + "name": "test", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/cmd/test", + "cwd": "${workspaceFolder}", + "output": "${workspaceFolder}/run/test", + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0a82fa2 --- /dev/null +++ b/README.md @@ -0,0 +1,50 @@ + +## 菜单 +- 币种管理 + - 增改查禁用删 +- 交易所账户管理 +- 订单管理 +- 策略管理 + - 加载,更新,平仓 +- (管理员)权限管理 +- (管理员)用户管理 + +1.看盘功能 + - 币种管理-添加币种 + - 初始化币种，拉取历史k线(1s-3个月)，订阅实时k线，补齐停机/网络错误实时曲线缺失(自动检查,存入时检查相邻时间数据) + - 币种数据落库, 币种数据缓存 + - websocket gateway, grpc stream, 订阅列表(k线/行情/深度)->实时推送前端 +2.自定义指标功能 + - go plugin + - MACD 指标 + - 指标订阅列表 grpc stream +3.自定义策略功能 + - go plugin + - 交易管理-订单 + - 平均价交易信号 + - 移动止盈止损 + - 风险管控 +4.策略模拟盘交易观察 + - 订阅列表-交易管理 grpc stream 盈亏/持仓量 +5.管理系统 +6.go plugin 指标库 +7.go plugin 策略库 +8.ai trade + + +victoriametrics + +下单策略(胜率/方向正确率)，止损参数 +每一单隔离 + +聚合信号, (BTC, GOLD, Oil...), both kline close +- AggregationSignalService +- 时间窗口3/5s, 过期无效(监控信号成功率/延迟率) + +k线推送 +- 实时k线 -> 内存聚合 +- closed -> cache -> async tsdb -> kline signal +- query(监控成功率/缓存命中率) -> cache -> tsdb + + + diff --git a/api/exchange.proto b/api/exchange.proto new file mode 100644 index 0000000..b0b85a7 --- /dev/null +++ b/api/exchange.proto @@ -0,0 +1,27 @@ +syntax = "proto3"; + +import "api/pub.proto"; + +option go_package = "./pb"; + +// 订阅/查询/存储,交易所k线图/热度/深度等基础数据指标 +service ExchangeService { + rpc SubscribeKline(stream ReqStreamSubscribeKline) + returns (stream RspStreamSubscribeKline); +} + +message ReqStreamSubscribeKline { + SubscribeType subType = 1; + repeated Exchange exchanges = 2; // 交易所 + repeated string instIds = 3; + repeated string intervals = 4; + bool onlyConfirm = 5; // 只要confirm的k线 +} + +message RspStreamSubscribeKline { StreamKline kline = 1; } + +message StreamKline { + repeated Kline klines = 2; + Exchange exchange = 3; // 交易所 + string instId = 4; // 交易产品id +} diff --git a/api/indicator.proto b/api/indicator.proto new file mode 100644 index 0000000..cfd27d2 --- /dev/null +++ b/api/indicator.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +import "api/pub.proto"; + +option go_package = "./pb"; + +service IndicatorService { + rpc Subscribe(IndicatorSubReq) returns (stream Indicator); // 订阅指标 + rpc Plot(Indicator) returns (Indicator); // 绘图 +} + +message IndicatorSubReq { + string topic = 1; + string instId = 2; + int32 window = 3; +} + +message Indicator { + Exchange exhcange = 1; + string instId = 2; + string indicator = 3; + string sub = 4; // 指标子标题, MA5, MA10, MA20 + int64 Ts = 5; + bytes payload = 8; +} diff --git a/api/market.proto b/api/market.proto new file mode 100644 index 0000000..e4c5497 --- /dev/null +++ b/api/market.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; + +import "api/pub.proto"; + +option go_package = "./pb"; + +// 交易产品管理 +service Market { + rpc GetTradeInstance(ReqGetTradeInstance) returns (RspGetTradeInstance); + rpc AddTradeInstance(ReqAddTradeInstance) returns (RspAddTradeInstance); + // rpc UpdateTradeInstance(ReqUpdateTradeInstance) returns (RspUpdateTradeInstance); + // rpc StatusTradeInstance(ReqStatusTradeInstance) returns (RspStatusTradeInstance); + // rpc ListTradeInstance(ReqListTradeInstance) returns (RspListTradeInstance); + // todo subscribe update + rpc ListExchangeTradeInstance(ReqListExchangeTradeInstance) returns (RspListExchangeTradeInstance); +} + +message ReqGetTradeInstance { + string instId = 1; +} +message RspGetTradeInstance { + TradeInstance inst = 1; +} + +message ReqAddTradeInstance { + TradeInstance inst = 1; +} +message RspAddTradeInstance { + TradeInstance inst = 1; +} + +message ReqUpdateTradeInstance { + TradeInstance inst = 1; +} +message RspUpdateTradeInstance { + TradeInstance inst = 1; +} + +message ReqStatusTradeInstance { + string instId = 1; +} +message RspStatusTradeInstance { + TradeInstance inst = 1; +} + +message ReqListTradeInstance { + int32 page = 1; + int32 pageSize = 2; + Exchange exchange = 3; // 交易所 + string topic = 4; + string instId = 5; +} +message RspListTradeInstance { + repeated Kline klines = 1; + Exchange exchange = 2; // 交易所 + string instId = 3; // 交易产品id +} + +message ReqListExchangeTradeInstance { + repeated Exchange exchanges = 1; +} +message RspListExchangeTradeInstance { + repeated TradeInstanceExchange exchangeInsts = 1; +} diff --git a/api/pub.proto b/api/pub.proto new file mode 100644 index 0000000..1754fab --- /dev/null +++ b/api/pub.proto @@ -0,0 +1,114 @@ +syntax = "proto3"; + +option go_package = "./pb"; + +enum SubscribeType { + Subscribe = 0; + Unsubscribe = 1; + UnsubscribeAll = 2; +} + +enum Exchange { + SIG = 0; + OKX = 1; + BINANCE = 2; +} + +enum TradeInstanceType { + Unknow = 0; + Spot = 1; // 1现货 + PerpetualContract = 2; // 2永续合约 +} + +enum Event { + UNKNOWN = 0; + SUBSCRIBED = 1; + UNSUBSCRIBED = 2; + SNAPSHOT = 3; + UPDATE = 4; + AUTHENTICATED = 5; + ERROR = 99; +} + +enum Channel { + BOOK = 0; + TRADE = 1; + TICKER = 2; + KLINE = 3; + BALANCE = 4; + ORDER = 5; +} + +enum Side { + BUY = 0; + SELL = 1; +} + +enum OrderType { + MARKET = 0; + LIMIT = 1; + STOP_MARKET = 2; + STOP_LIMIT = 3; + POST_ONLY = 4; + IOC_LIMIT = 5; +} + +message Error { + int64 error_code = 1; + string error_message = 2; +} + +// 交易产品基础信息 +message TradeInstance { + string instId = 1; + string instPair = 2; + string instCoin = 3; + TradeInstanceType instType = 4; + int32 status = 5; + int32 priceSz = 6; + int32 quantitySz = 7; + string icon = 8; + string updateBy = 9; + int64 updateTime = 10; + repeated int32 leverages = 11; + repeated TradeInstanceExchange exchanges = 15; // 交易产品支持的交易所 +} + +// 交易所交易产品 +message TradeInstanceExchange { + string exchangeInstId = 1; + string instId = 2; + int32 status = 3; + int32 exchange = 4; + string updateBy = 5; + int64 updateTime = 6; +} + +message Kline { + int64 Ts = 2; + string interval = 3; // 周期 + string Open = 4; + string High = 5; + string Low = 6; + string Close = 7; + string vol = 8; // 交易量 + string volQuote = 9; // 交易额 + bool Confirm = 10; // k线是否完结 +} + +// https://maicoin.github.io/max-websocket-docs/#/private_channels?id=snapshot +message Order { + string exchange = 1; + string symbol = 2; + string id = 3; + Side side = 4; + OrderType order_type = 5; + string price = 6; + string stop_price = 7; + string status = 9; + string quantity = 11; + string executed_quantity = 12; + string client_order_id = 14; + int64 group_id = 15; + int64 created_at = 10; +} diff --git a/api/trading.proto b/api/trading.proto new file mode 100644 index 0000000..ee3e7d2 --- /dev/null +++ b/api/trading.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +import "api/pub.proto"; + +option go_package = "./pb"; + +service TradingService { + // request-response + rpc SubmitOrder(SubmitOrderReq) returns (SubmitOrderRsp) {} + // rpc CancelOrder(CancelOrderRequest) returns (CancelOrderResponse) {} + // rpc QueryOrder(QueryOrderRequest) returns (QueryOrderResponse) {} + // rpc QueryOrders(QueryOrdersRequest) returns (QueryOrdersResponse) {} + // rpc QueryTrades(QueryTradesRequest) returns (QueryTradesResponse) {} +} + +message SubmitOrder { + string session = 1; + string exchange = 2; + string symbol = 3; + Side side = 4; + string price = 6; + string quantity = 5; + string stop_price = 7; + OrderType order_type = 8; + string client_order_id = 9; + int64 group_id = 10; +} + +message SubmitOrderReq { + string session = 1; + repeated SubmitOrder submit_orders = 2; +} + +message SubmitOrderRsp { + string session = 1; + repeated Order orders = 2; + Error error = 3; +} diff --git a/cmd/exchange/exchange_test.go b/cmd/exchange/exchange_test.go new file mode 100644 index 0000000..1d85494 --- /dev/null +++ b/cmd/exchange/exchange_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "sig-pub/internal/exchange/okx" + "sig-pub/pkg/config" + vmts "sig-pub/pkg/storage/tsdb/victoria_metrics" + "sig-pub/pkg/types" + "sig-pub/pkg/zlog" + "testing" +) + +func TestExchange(t *testing.T) { + channel := okx.NewChannelCandle("candle-0", "http://192.168.1.5:7890") + // channel := okx.NewChannelTickers("tickers-0", "http://192.168.1.5:7890") + // channel := okx.NewChannelTrades("trades-0", "http://192.168.1.5:7890") + // channel := okx.NewChannelBooks("books-0", "http://192.168.1.5:7890") + if err := channel.Init(); err != nil { + panic(err) + } + err := channel.Subscribe("DOGE-USDT") // "BTC-USDT-SWAP", + if err != nil { + panic(err) + } + + C := channel.Consumer() + + vmdb := vmts.NewVictoriaMetricsTSDB(config.VictoriaMetricsConfig{Addr: "http://127.0.0.1:8428"}) + + var klines []*types.Kline + inst := types.TradeInstance{ + InstId: "DOGE-USDT", + } + for candle := range C { + for _, kline := range candle.Klines { + if kline.Confirm { + klines = append(klines, kline) + } + } + + if len(klines) >= 10 { + err := vmdb.SaveKlines(inst, klines) + if err != nil { + zlog.Error(err) + } else { + zlog.Infof("save klines :%d", len(klines)) + } + klines = klines[:0] + } + + zlog.Infof("channel: %s, instId: %s, datas: %#v", candle.InstId, candle.Exchange, candle.Klines[0]) + } +} + +// type ExchangeService struct { +// pb.UnimplementedExchangeServiceServer +// } + +// func NewExchangeService() *ExchangeService { +// return &ExchangeService{} +// } + +// func (ExchangeService) SubscribeKline(req *pb.ReqSubscribeKline, stream grpc.ServerStreamingServer[pb.StreamKline]) error { +// ctx := stream.Context() + +// topic := req.Topic +// log.Printf("Client subscribed to topic: %s", topic) + +// // go func() { +// // <-time.After(time.Second * 20) +// // // stream.Context().Done() +// // // stream.CloseSend() + +// // fmt.Println("cenceled...") +// // }() + +// for { +// select { +// case <-ctx.Done(): +// // 客户端断开连接 +// log.Printf("Client disconnected from topic: %s", topic) +// return nil +// default: +// // 模拟事件生成 +// event := &pb.StreamKline{ +// InstId: "DOGE/USDT", +// Exchange: pb.Exchange_OKX, +// Klines: []*pb.Kline{ +// { +// Ts: time.Now().Unix(), +// }, +// }, +// } + +// // 推送事件 +// if err := stream.Send(event); err != nil { +// log.Printf("Failed to send event to client: %v", err) +// return err +// } + +// // 模拟事件间隔 +// time.Sleep(2 * time.Second) +// } +// } +// } diff --git a/cmd/exchange/main.go b/cmd/exchange/main.go new file mode 100644 index 0000000..bfa2712 --- /dev/null +++ b/cmd/exchange/main.go @@ -0,0 +1,111 @@ +package main + +import ( + "context" + "fmt" + "net" + "sig-pub/api/pb" + "sig-pub/internal/exchange" + "sig-pub/internal/exchange/okx" + "sig-pub/pkg/aside" + "sig-pub/pkg/config" + "sig-pub/pkg/grpc/discovery" + "sig-pub/pkg/grpc/interceptor" + vmts "sig-pub/pkg/storage/tsdb/victoria_metrics" + "sig-pub/pkg/utils/exit" + "sig-pub/pkg/zlog" + + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/reflection" +) + +type ExchangeConf struct { + Register discovery.Server + GrpcReflection bool +} + +func main() { + // load config + conf := config.MustLoadConfig(new(config.Configuration), "config/config.toml") + exchangeConf := config.MustLoadConfig(new(ExchangeConf), "config/exchange.toml") + + // tsdb VictoriaMetrics + vmtsdb := vmts.NewVictoriaMetricsTSDB(conf.Tsdb.Victoriametrics) + tsdbService := exchange.NewExchangeDataService(vmtsdb, nil) + if err := tsdbService.Init(); err != nil { + panic(err) + } + + okxExchange := okx.NewOkxExchange(conf.Exchange.Okx) + if err := okxExchange.Init(); err != nil { + panic(err) + } + + etcdClient, err := clientv3.New(conf.Etcd) + if err != nil { + panic(err) + } + exit.AddHook(func() { _ = etcdClient.Close() }, exit.WithOrderTail()) + + // get market grpc client + dis := discovery.NewEtcdDiscovery(etcdClient) + resolver, err := dis.Resolver() + if err != nil { + panic(err) + } + url := fmt.Sprintf("%s:///%s", discovery.EtcdSchema, pb.Market_ServiceDesc.ServiceName) + conn, err := grpc.NewClient(url, + grpc.WithResolvers(resolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + panic(err) + } + marketClient := pb.NewMarketClient(conn) + tradeInstanceAside := aside.NewTradeInstanceAside(marketClient) + exchangeService := exchange.NewExchangeGrpcServer(tradeInstanceAside, tsdbService, okxExchange) + if err := exchangeService.Init(); err != nil { + panic(err) + } + + grpcServer := grpc.NewServer(config.GetGrpcOptions( + conf.Grpc, + grpc.UnaryInterceptor(interceptor.RecoverInterceptor))..., + ) + if exchangeConf.GrpcReflection { + // 注册反射服务 + reflection.Register(grpcServer) + } + + pb.RegisterExchangeServiceServer(grpcServer, exchangeService) + + registry := discovery.NewEtcdDiscovery(etcdClient) + + register := exchangeConf.Register + if register.Name == "" { + register.Name = pb.Market_ServiceDesc.ServiceName + } + + ctx, cancel := context.WithCancel(context.Background()) + exit.AddHook(cancel, exit.WithOrderFront()) + if err := registry.Registry(ctx, register); err != nil { + panic(err) + } + exit.AddHook(grpcServer.GracefulStop, exit.WithOrderFront()) + + // run grpc server + go func() { + listen, err := net.Listen("tcp", exchangeConf.Register.Addr) + if err != nil { + panic(err) + } + zlog.Infof("%s grpc server running %s\n", register.Name, listen.Addr().String()) + if err := grpcServer.Serve(listen); err != nil { + panic(err) + } + }() + + exit.Await() +} diff --git a/cmd/gateway/main.go b/cmd/gateway/main.go new file mode 100644 index 0000000..7959503 --- /dev/null +++ b/cmd/gateway/main.go @@ -0,0 +1,21 @@ +package main + +import ( + "sig-pub/internal/gateway" + "sig-pub/pkg/config" +) + +type GateConf struct { + HttpAddr string + WsAddr string +} + +// http 网关 +func main() { + gateConf := config.MustLoadConfig(new(GateConf), "config/gate.toml") + server := gateway.Route() + err := server.Run(gateConf.HttpAddr) + if err != nil { + panic(err) + } +} diff --git a/cmd/indicator/main.go b/cmd/indicator/main.go new file mode 100644 index 0000000..75701a1 --- /dev/null +++ b/cmd/indicator/main.go @@ -0,0 +1,150 @@ +package main + +import ( + "context" + "io" + "log" + "sig-pub/api/pb" + "sig-pub/pkg/types" + "sig-pub/pkg/zlog" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" +) + +func main() { + // 配置 keepalive 参数 + keepaliveParams := keepalive.ClientParameters{ + Time: 30 * time.Second, // 发送 ping 的间隔 + Timeout: 3 * time.Second, // ping 超时 + PermitWithoutStream: true, // 允许在没有流的情况下发送 ping + } + + conn, err := grpc.NewClient("localhost:8888", + // grpc.WithInsecure(), + grpc.WithKeepaliveParams(keepaliveParams), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + log.Fatalf("Failed to connect: %v", err) + } + defer conn.Close() + + client := pb.NewExchangeServiceClient(conn) + + // subscribeEvents(client) + for { + subscribeEvents(client) + log.Println("Disconnected, retrying in 5 seconds...") + time.Sleep(5 * time.Second) // 重连间隔 + } +} + +func subscribeEvents(client pb.ExchangeServiceClient) { + // ctx, cancel := context.WithCancel(context.Background()) + ctx := context.Background() + stream, err := client.SubscribeKline(ctx) + if err != nil { + return + } + go func() { + <-time.After(time.Second * 60) + // cancel() + err := stream.CloseSend() // 触发 server 端 err = io.EOF + zlog.Infof("连接取消... %v", err) + }() + + // 接收消息的goroutine + go func() { + for { + msg, err := stream.Recv() + if err == io.EOF { + zlog.Errorf("服务端关闭连接") + return + } + if err != nil { + zlog.Errorf("接收错误: %v", err) + return + } + + for _, k := range msg.Kline.Klines { + kline := new(types.Kline) + kline.ParsePBKline(msg.Kline.Exchange, k) + zlog.Infof("收到服务端消息: %v, %s, %#v", msg.Kline.Exchange, msg.Kline.InstId, kline) + } + } + }() + + // 发送消息的goroutine + instIds := []string{"BTC-USDT", "DOGE-USDT-SWAP"} + msg := &pb.ReqStreamSubscribeKline{ + SubType: pb.SubscribeType_Subscribe, + Exchanges: []pb.Exchange{pb.Exchange_OKX}, + InstIds: instIds, + Intervals: []string{ + string(types.Interval1s), + string(types.Interval1m), + }, + OnlyConfirm: true, + } + zlog.Infof("send stream msg: %#v", msg) + if err = stream.Send(msg); err != nil { + zlog.Errorf("发送失败: %v", err) + return + } + + go func() { + <-time.After(10 * time.Second) + msg := &pb.ReqStreamSubscribeKline{ + SubType: pb.SubscribeType_Unsubscribe, + Exchanges: []pb.Exchange{pb.Exchange_OKX}, + InstIds: []string{"BTC-USDT"}, + Intervals: []string{ + string(types.Interval1s), + string(types.Interval1m), + }, + OnlyConfirm: true, + } + if err = stream.Send(msg); err != nil { + zlog.Errorf("发送失败: %v", err) + return + } + zlog.Infof("unsubscribe...") + }() + + // go func() { + // scanner := bufio.NewScanner(os.Stdin) + // for scanner.Scan() { + // text := scanner.Text() + // if text == "exit" { + // // 关闭发送端 + // if err := stream.CloseSend(); err != nil { + // log.Printf("关闭发送失败: %v", err) + // } + // return + // } + // } + // }() + + // 保持连接 + <-stream.Context().Done() + log.Println("连接关闭") + + // log.Println("subscribeEvents...") + // stream, err := client.SubscribeKline(ctx, &pb.ReqSubscribeKline{Topic: topic}) + // if err != nil { + // log.Fatalf("Failed to subscribe: %v", err) + // return + // } + + // for { + // event, err := stream.Recv() + // if err != nil { + // log.Printf("Failed to receive event: %v", err) + // break + // } + // fmt.Printf("Received event: InstId=%s, Ts=%v\n", event.InstId, event.Exchange) + // } +} diff --git a/cmd/market/main.go b/cmd/market/main.go new file mode 100644 index 0000000..220e380 --- /dev/null +++ b/cmd/market/main.go @@ -0,0 +1,91 @@ +package main + +import ( + "context" + "net" + "sig-pub/api/pb" + "sig-pub/internal/market" + "sig-pub/pkg/config" + "sig-pub/pkg/grpc/discovery" + "sig-pub/pkg/grpc/interceptor" + "sig-pub/pkg/storage/rdb" + "sig-pub/pkg/utils/exit" + "sig-pub/pkg/zlog" + + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/reflection" +) + +type MarketConf struct { + Register discovery.Server + GrpcReflection bool +} + +func main() { + // load config + conf := config.MustLoadConfig(new(config.Configuration), "config/config.toml") + marketConf := config.MustLoadConfig(new(MarketConf), "config/market.toml") + + // etcd discovery + etcdClient, err := clientv3.New(conf.Etcd) + if err != nil { + panic(err) + } + exit.AddHook(func() { _ = etcdClient.Close() }, exit.WithOrderTail()) + + registry := discovery.NewEtcdDiscovery(etcdClient) + + // database + db, err := conf.Database.Postgres.NewGormDB() + if err != nil { + panic(err) + } + rdb := rdb.NewRDB(db) + if err := rdb.Init(); err != nil { + panic(err) + } + + tradeInstanceService := market.NewTradeInstanceService(rdb) + if err := tradeInstanceService.Init(); err != nil { + panic(err) + } + marketGrpcServer := market.NewMarketGrpcServer(tradeInstanceService) + + grpcServer := grpc.NewServer(config.GetGrpcOptions( + conf.Grpc, + grpc.UnaryInterceptor(interceptor.RecoverInterceptor))..., + ) + if marketConf.GrpcReflection { + // 注册反射服务 + reflection.Register(grpcServer) + } + + pb.RegisterMarketServer(grpcServer, marketGrpcServer) + + register := marketConf.Register + if register.Name == "" { + register.Name = pb.Market_ServiceDesc.ServiceName + } + + ctx, cancel := context.WithCancel(context.Background()) + exit.AddHook(cancel, exit.WithOrderFront()) + if err := registry.Registry(ctx, register); err != nil { + panic(err) + } + exit.AddHook(grpcServer.GracefulStop, exit.WithOrderFront()) + + // run grpc server + go func() { + listen, err := net.Listen("tcp", marketConf.Register.Addr) + if err != nil { + panic(err) + } + zlog.Infof("%s grpc server running %s\n", register.Name, listen.Addr().String()) + if err := grpcServer.Serve(listen); err != nil { + panic(err) + } + }() + + exit.Await() +} diff --git a/cmd/postal/main.go b/cmd/postal/main.go new file mode 100644 index 0000000..32257da --- /dev/null +++ b/cmd/postal/main.go @@ -0,0 +1,6 @@ +package main + +// postal 长连接消息投递服务 +func main() { + +} diff --git a/cmd/sig-admin/main.go b/cmd/sig-admin/main.go new file mode 100644 index 0000000..e8bc18c --- /dev/null +++ b/cmd/sig-admin/main.go @@ -0,0 +1,50 @@ +package main + +import ( + "sig-pub/internal/sig" + "sig-pub/pkg/config" + "sig-pub/pkg/grpc/discovery" + "sig-pub/pkg/grpc/generic" + "sig-pub/pkg/utils/exit" + + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +func main() { + defer exit.Await() + // load config + conf := config.MustLoadConfig(new(config.Configuration), "config/config.toml") + + // etcd discovery + etcdClient, err := clientv3.New(conf.Etcd) + if err != nil { + panic(err) + } + exit.AddHook(func() { _ = etcdClient.Close() }, exit.WithOrderTail()) + + dis := discovery.NewEtcdDiscovery(etcdClient) + resolver, err := dis.Resolver() + if err != nil { + panic(err) + } + gpcGenericClientFactory := generic.NewGpcGenericClientFactory( + discovery.EtcdSchema, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithResolvers(resolver), + ) + if err := gpcGenericClientFactory.Init(); err != nil { + panic(err) + } + + sigServer := sig.NewSigServer(gpcGenericClientFactory) + if err := sigServer.Init(); err != nil { + panic(err) + } + go func() { + if err := sigServer.Run(":7001"); err != nil { + panic(err) + } + }() +} diff --git a/cmd/test/generic.go b/cmd/test/generic.go new file mode 100644 index 0000000..92f0f14 --- /dev/null +++ b/cmd/test/generic.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "fmt" + "sig-pub/api/pb" + "sig-pub/pkg/grpc/session" + + "github.com/bytedance/sonic" + "github.com/jhump/protoreflect/v2/grpcdynamic" + "github.com/jhump/protoreflect/v2/grpcreflect" + "google.golang.org/grpc" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/dynamicpb" + + "google.golang.org/grpc/credentials/insecure" + refv1 "google.golang.org/grpc/reflection/grpc_reflection_v1" +) + +func main() { + testGeneric() +} + +func testGeneric() { + // gin: json body -> io.ReadAll(c.Request.Body) + + // conf := config.MustLoadConfig(new(config.Configuration), "config/config.toml") + + // etcdClient, err := clientv3.New(conf.Etcd) + // if err != nil { + // panic(err) + // } + + // dis := discovery.NewEtcdDiscovery(etcdClient) + // resolver, err := dis.Resolver() + // if err != nil { + // panic(err) + // } + + // url := fmt.Sprintf("%s:///%s", discovery.EtcdSchema, "Market") + + cli, err := grpc.NewClient("127.0.0.1:8001", + // grpc.WithResolvers(resolver), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + if err != nil { + panic(err) + } + ctx := context.Background() + // refClient := grpcreflect.NewClientV1Alpha(ctx, cli) + client := grpcreflect.NewClientV1(ctx, refv1.NewServerReflectionClient(cli)) + services, err := client.ListServices() + if err != nil { + panic(err) + } else { + for _, svr := range services { + fmt.Println(svr.Name(), svr.Parent(), svr) + } + } + + marketServiceSymbol, err := client.FileContainingSymbol("Market") + if err != nil { + panic(err) + } + marketService := marketServiceSymbol.Services().ByName("Market") + + fmt.Println(marketServiceSymbol.Services().Len()) + + method := marketService.Methods().ByName("GetTradeInstance") + + req := &pb.ReqGetTradeInstance{InstId: "BTC_USDT"} + reqBytes, err := proto.Marshal(req) + if err != nil { + panic(err) + } + _ = reqBytes + reqJsonBytes, err := sonic.Marshal(req) + if err != nil { + panic(err) + } + _ = reqJsonBytes + + msgDesc := method.Input() + msg := dynamicpb.NewMessage(msgDesc) + // msg.Set(msgDesc.Fields().ByJSONName("instId"), protoreflect.ValueOf("DOGE-USDT")) + + // if err := proto.Unmarshal(reqBytes, msg); err != nil { + // panic(err) + // } + + if err := protojson.Unmarshal(reqJsonBytes, msg); err != nil { + panic(err) + } + + stub := grpcdynamic.NewStub(cli) + + ctx = session.PutSubject(ctx, session.NewRpcSubject("123456")) + r, err := stub.InvokeRpc(ctx, method, msg) + if err != nil { + panic(err) + } + rr, ok := r.(*dynamicpb.Message) + fmt.Println(ok, rr) + fmt.Printf("resp: %T, %#v\n", r, r) + bytes, err := sonic.Marshal(r) + + fmt.Printf("resp: %s, err=%v\n", string(bytes), err) +} diff --git a/cmd/test/test.go b/cmd/test/test.go new file mode 100644 index 0000000..a8d0810 --- /dev/null +++ b/cmd/test/test.go @@ -0,0 +1,68 @@ +package main + +import ( + "sig-pub/api/pb" + "sig-pub/pkg/types" + "sig-pub/pkg/zlog" + + "github.com/VictoriaMetrics/metrics" + "github.com/govalues/decimal" +) + +func main_test() { + open := metrics.NewCounter("open") + open.Set(1234) + + // curl -H 'Content-Type: application/json' --data-binary "@vmdata.json" -X POST http://localhost:8428/api/v1/import + + testDecimal() +} + +type BTC struct { + Price decimal.Decimal +} + +func testDecimal() { + // zlog.Init() + var interval = "1m" + interval0 := types.Interval(interval) + sec, _ := interval0.Seconds() + zlog.Infof("hello...: %s, %s, %d", pb.Exchange_OKX.String(), pb.Exchange_BINANCE.String(), sec) + + // data := `{"price":"1.23456"}` + // btc := new(BTC) + // err := json.Unmarshal([]byte(data), btc) + // if err != nil { + // panic(err) + // } + // fmt.Println(btc) + // bytes, err := json.Marshal(btc) + // if err != nil { + // panic(err) + // } + // fmt.Println(string(bytes)) + + // AIDOGE price + sz := decimal.MustParse("6672864381598761234.752194906") + f, ok := sz.Float64() + if !ok { + zlog.Error("error parse value") + } + zlog.Info(sz, f, int64(f)) + scale, _ := decimal.Ten.PowInt(sz.Scale()) + // price := decimal.MustParse("0.0000000001713") + // // zlog.Info(sz.Scale()) + + // // quantity := decimal.MustNew(int64(sz.Scale()), 0) + // price, _ = price.Mul(scale) + // price_i64, _, _ := price.Int64(0) + // zlog.Info(price_i64) + + _ = scale + price0 := int64(1713) + // price_i64, _ := decimal.NewFromInt64(0, price0, sz.Scale()) + // zlog.Info(price_i64) + price_i64, _ := decimal.NewFromInt64(price0, 0, 0) + price, _ := price_i64.Quo(scale) + zlog.Info(price) +} diff --git a/config/config.toml b/config/config.toml new file mode 100644 index 0000000..26e2268 --- /dev/null +++ b/config/config.toml @@ -0,0 +1,71 @@ + +[grpc] +maxSendMsgSize = "8Mi" +maxRecvMsgSize = "8Mi" +readBufferSize = "8Ki" +writeBufferSize = "8Ki" + +[grpc.keepalive] +idleTimeout = "60s" # 空闲连接超时 +forceCloseWait = "20s" +keepAliveInterval = "60s" # 发送 ping 的间隔 +keepAliveTimeout = "20s" # ping 超时 +maxLifeTime = "2h" + +[etcd] +endpoints = ["127.0.0.1:7079"] +username = "" +password = "" + +[database.mysql] +logMode = "info" +# https://gorm.io/zh_CN/docs/connecting_to_the_database.html +mysql = { DSN = "root:123456@tcp(127.0.0.1:7306)/sig?charset=utf8&parseTime=True&loc=Local" } + +[database.postgres] +logMode = "info" +postgres = { DSN = "host=127.0.0.1 port=5432 user=postgres password=123456 dbname=sig sslmode=disable TimeZone=Asia/Shanghai" } + + +[database.clickhouse] +# dsn = "clickhouse://user:password@192.168.0.137:9000/game?dial_timeout=10s&read_timeout=20s" +addr = "127.0.0.1:7900" +database = "sig" +username = "root" +password = "123456" +dial_timeout = 10 +read_timeout = 20 +logsql = false + +[database.kvrocks] +addr = "127.0.0.1:7666" +password = "" +minIdleConns = 3 + +[tsdb] +active = "victoriametrics" + +[tsdb.victoriametrics] +addr = "http://127.0.0.1:8428" + +[exchange.okx] +# apiKey = "7273282c-90f5-498f-9fe0-140ae07f5a73" +# secretKey = "47F9F373CF43827497CA2EF99E889D48" +# passphrase = "Sopod.2347." +apiKey = "48be46ec-30ad-4f6f-a5e2-c9ea712df9cd" +secretKey = "12323BA1B411A2235650BBD784850953" +passphrase = "Tm.123456789" +receiveBuffer = 4096 +marketSubscribeLimit = 16 +consumeBatch = 1024 +consumeLater = 2000 # 时间到达later或者数据累计到batch触发consume +httpProxy = "http://192.168.1.5:7890" + +# 模拟盘API交易地址如下： +# REST：https://www.okx.com +# WebSocket公共频道：wss://wspap.okx.com:8443/ws/v5/public +# WebSocket私有频道：wss://wspap.okx.com:8443/ws/v5/private +# WebSocket业务频道：wss://wspap.okx.com:8443/ws/v5/business + +# binance key RYgJrvqP4iGqdRth14r0ChgWo8eg0wPEcFqDttsKzvUJDyhOKvPiz42tXxjYIMiG +# secret GLTzNNYzC0AbcINPAYfuKDjkWnAMQUhsyd1ed7ubcdzIRrFZBGUrOAkubqyjekVp diff --git a/config/exchange.toml b/config/exchange.toml new file mode 100644 index 0000000..f4b8135 --- /dev/null +++ b/config/exchange.toml @@ -0,0 +1,6 @@ + +grpcReflection = false # 注册grpc反射服务 + +[register] +addr = ":8011" # grpc 服务端口 +attrs = { weight = 10 } # grpc 服务权重 diff --git a/config/gate.toml b/config/gate.toml new file mode 100644 index 0000000..bc2701c --- /dev/null +++ b/config/gate.toml @@ -0,0 +1,3 @@ + +httpAddr = ":7001" +wsAddr = ":7101" diff --git a/config/influxdb/influx-configs b/config/influxdb/influx-configs new file mode 100644 index 0000000..87eeb36 --- /dev/null +++ b/config/influxdb/influx-configs @@ -0,0 +1,20 @@ +[default] + url = "http://localhost:8086" + token = "FiKkr_GEXUVUoQ5bbR8EL_2HO9Yj-bQbbI1sCGPoXYZMk_Giw4UJrmRG8e2uWltHLR7lxhVHor6GYDMBlGlZgg==" + org = "sig" + active = true +# +# [eu-central] +# url = "https://eu-central-1-1.aws.cloud2.influxdata.com" +# token = "XXX" +# org = "" +# +# [us-central] +# url = "https://us-central1-1.gcp.cloud2.influxdata.com" +# token = "XXX" +# org = "" +# +# [us-west] +# url = "https://us-west-2-1.aws.cloud2.influxdata.com" +# token = "XXX" +# org = "" diff --git a/config/market.toml b/config/market.toml new file mode 100644 index 0000000..063644b --- /dev/null +++ b/config/market.toml @@ -0,0 +1,6 @@ + +grpcReflection = true # 不注册grpc反射服务 + +[register] +addr = ":8001" # grpc 服务端口 +attrs = { weight = 10 } # grpc 服务权重 diff --git a/config/mysql/conf.d/1 b/config/mysql/conf.d/1 new file mode 100644 index 0000000..e69de29 diff --git a/config/mysql/my.cnf b/config/mysql/my.cnf new file mode 100644 index 0000000..b6fba75 --- /dev/null +++ b/config/mysql/my.cnf @@ -0,0 +1,6 @@ +[mysqld] +port = 7306 +max_connections = 1000 +character_set_server = utf8mb4 +collation-server = utf8mb4_general_ci +default-time_zone = '+8:00' diff --git a/config/redis/redis.conf b/config/redis/redis.conf new file mode 100644 index 0000000..1619014 --- /dev/null +++ b/config/redis/redis.conf @@ -0,0 +1,78 @@ +# https://redis.io/docs/latest/operate/oss_and_stack/management/config-file/ + +bind 0.0.0.0 +requirepass 123456 +port 6379 +protected-mode yes +tcp-backlog 511 +timeout 0 +tcp-keepalive 300 +daemonize no +pidfile /var/run/redis_6379.pid +loglevel notice +logfile "" +databases 16 +always-show-logo no +set-proc-title yes +proc-title-template "{title} {listen-addr} {server-mode}" +locale-collate "" +stop-writes-on-bgsave-error yes +rdbcompression yes +rdbchecksum yes +dbfilename dump.rdb +rdb-del-sync-files no +dir ./ +replica-serve-stale-data yes +replica-read-only yes +repl-diskless-sync yes +repl-diskless-sync-delay 5 +repl-diskless-sync-max-replicas 0 +repl-diskless-load disabled +repl-disable-tcp-nodelay no +replica-priority 100 +acllog-max-len 128 +lazyfree-lazy-eviction no +lazyfree-lazy-expire no +lazyfree-lazy-server-del no +replica-lazy-flush no +lazyfree-lazy-user-del no +lazyfree-lazy-user-flush no +oom-score-adj no +oom-score-adj-values 0 200 800 +disable-thp yes +appendonly no +appendfilename "appendonly.aof" +appenddirname "appendonlydir" +appendfsync everysec +no-appendfsync-on-rewrite no +auto-aof-rewrite-percentage 100 +auto-aof-rewrite-min-size 64mb +aof-load-truncated yes +aof-use-rdb-preamble yes +aof-timestamp-enabled no + +slowlog-log-slower-than 10000 +slowlog-max-len 128 +latency-monitor-threshold 0 +notify-keyspace-events "" +hash-max-listpack-entries 512 +hash-max-listpack-value 64 +list-max-listpack-size -2 +list-compress-depth 0 +set-max-intset-entries 512 +set-max-listpack-entries 128 +set-max-listpack-value 64 +zset-max-listpack-entries 128 +zset-max-listpack-value 64 +hll-sparse-max-bytes 3000 +stream-node-max-bytes 4096 +stream-node-max-entries 100 +activerehashing yes +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit replica 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 +hz 10 +dynamic-hz yes +aof-rewrite-incremental-fsync yes +rdb-save-incremental-fsync yes +jemalloc-bg-thread yes diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..54db9e4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,120 @@ +services: + sig-clickhouse: + image: 'clickhouse/clickhouse-server:24.12' + user: 'root' + container_name: sig-clickhouse + hostname: sig-clickhouse + environment: + # - CLICKHOUSE_DB=sig + - CLICKHOUSE_USER=root + - CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 + - CLICKHOUSE_PASSWORD=123456 + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "./fs/clickhouse/ch_data:/var/lib/clickhouse/" + - "./fs/clickhouse/ch_logs:/var/log/clickhouse-server/" + # - ./fs/clickhouse/config.d/config.xml:/etc/clickhouse-server/config.d/config.xml + # - ./fs/clickhouse/users.d/users.xml:/etc/clickhouse-server/users.d/users.xml + ports: + - '7123:8123' + - '7900:9000' + ulimits: + nofile: + soft: "262144" + hard: "262144" + sig-kvrocks: + image: 'apache/kvrocks:nightly' + container_name: sig-kvrocks + hostname: sig-kvrocks + user: 'root' + # restart: always + sysctls: + net.core.somaxconn: 1024 + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "./fs/kvrocks_data:/var/lib/kvrocks" + ports: + - '7666:6666' + command: --bind 0.0.0.0 --dir /var/lib/kvrocks + sig-mysql: + container_name: sig-mysql + image: mysql:8.4.5 + # command: --default-authentication-plugin=mysql_native_password + # restart: always + network_mode: host + environment: + MYSQL_ROOT_PASSWORD: '123456' + MYSQL_DATABASE: sig + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "./config/mysql:/etc/mysql" + - "./fs/mysql-data:/var/lib/mysql" + sig-postgres: + container_name: sig-postgres + image: postgres:17.5 + network_mode: host + environment: + POSTGRES_PASSWORD: '123456' + POSTGRES_DB: sig + PGDATA: /var/lib/postgresql/data/pgdata + volumes: + - "./config/mysql:/etc/mysql" + - "./fs/postgres-data:/var/lib/postgresql/data" + sig-redis: + # dragonflydb + container_name: sig-redis + image: redis:7.4 + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "./config/redis/redis.conf:/usr/local/etc/redis/redis.conf" + - ./fs/redis-data:/data + ports: + - "7379:6379" + sig-influxdb: + container_name: sig-influxdb + image: influxdb:2.7 + environment: + DOCKER_INFLUXDB_INIT_MODE: setup + DOCKER_INFLUXDB_INIT_USERNAME: root + DOCKER_INFLUXDB_INIT_PASSWORD: "123456789" + DOCKER_INFLUXDB_INIT_ORG: sig + DOCKER_INFLUXDB_INIT_BUCKET: kline + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "./config/influxdb:/etc/influxdb2" + - "./fs/influxdb-data:/var/lib/influxdb2" + ports: + - 8086:8086 + # DAc-Mie0PKCqsRdd5I_Hkn-Ar0XJeeQDe-Jcsoo8wmKTPpG8Hp4RrTi3RRd3LQ4GKxZbNYJTESTX6us1O9FVNw== + + sig-vm: + container_name: sig-vm + image: victoriametrics/victoria-metrics:v1.116.0 + volumes: + - "./fs/victoria-metrics-data:/victoria-metrics-data" + ports: + - 8428:8428 + command: -dedup.minScrapeInterval=1s -retentionPeriod=99y + + sig-questdb: + container_name: sig-questdb + image: questdb/questdb:8.3.1 + volumes: + - "/etc/localtime:/etc/localtime:ro" + - "./fs/questdb:/var/lib/questdb" + - "./fs/questdb:/conf/log.conf" + ports: + - 7900:9000 + - 7909:9009 + - 7912:8912 + - 7903:9003 + sig-etcd: + container_name: sig-etcd + image: bitnami/etcd:3.6.1 + environment: + ALLOW_NONE_AUTHENTICATION: yes + ETCD_ADVERTISE_CLIENT_URLS: http://127.0.0.1:7079 + ports: + - 7079:2379 + - 7080:2380 diff --git a/generate.go b/generate.go new file mode 100644 index 0000000..47c263a --- /dev/null +++ b/generate.go @@ -0,0 +1,15 @@ +package main + +import "fmt" + +//go:generate go run generate.go +//go:generate protoc --go_out=./api/ --go-grpc_out=./api/ ./api/pub.proto +//go:generate protoc --go_out=./api/ --go-grpc_out=./api/ ./api/market.proto +//go:generate protoc --go_out=./api/ --go-grpc_out=./api/ ./api/exchange.proto +//go:generate protoc --go_out=./api/ --go-grpc_out=./api/ ./api/indicator.proto +//go:generate protoc --go_out=./api/ --go-grpc_out=./api/ ./api/trading.proto + +// run cmd: go generate generate.go +func main() { + fmt.Println("generating...") +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..7a3a42b --- /dev/null +++ b/go.mod @@ -0,0 +1,88 @@ +module sig-pub + +go 1.23.4 + +require ( + github.com/VictoriaMetrics/metrics v1.36.0 + github.com/bwmarrin/snowflake v0.3.0 + github.com/bytedance/sonic v1.13.2 + github.com/dsnet/golib/unitconv v1.0.2 + github.com/fanjindong/go-cache v0.0.6 + github.com/gin-gonic/gin v1.10.0 + github.com/gorilla/websocket v1.5.3 + github.com/govalues/decimal v0.1.36 + github.com/influxdata/influxdb-client-go/v2 v2.14.0 + github.com/jhump/protoreflect/v2 v2.0.0-beta.2 + github.com/klauspost/compress v1.17.9 + github.com/lib/pq v1.10.9 + github.com/redis/go-redis/v9 v9.7.3 + github.com/spf13/viper v1.20.0 + go.etcd.io/etcd/api/v3 v3.6.1 + go.etcd.io/etcd/client/v3 v3.6.1 + go.uber.org/zap v1.27.0 + golang.org/x/net v0.38.0 + google.golang.org/grpc v1.71.1 + google.golang.org/protobuf v1.36.6 + gopkg.in/natefinch/lumberjack.v2 v2.2.1 + gorm.io/driver/mysql v1.5.7 + gorm.io/driver/postgres v1.6.0 + gorm.io/gorm v1.25.12 +) + +require ( + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-sql-driver/mysql v1.7.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect + github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.6.0 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect + github.com/leodido/go-urn v1.4.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oapi-codegen/runtime v1.0.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/ugorji/go/codec v1.2.12 // indirect + github.com/valyala/fastrand v1.1.0 // indirect + github.com/valyala/histogram v1.2.0 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.6.1 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/arch v0.15.0 // indirect + golang.org/x/crypto v0.36.0 // indirect + golang.org/x/sync v0.12.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..7645bc3 --- /dev/null +++ b/go.sum @@ -0,0 +1,259 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= +github.com/VictoriaMetrics/metrics v1.36.0 h1:f3SZMpLgIG4hJm2zfDs6wicxQ/QNWBZekY5rEGgbHKs= +github.com/VictoriaMetrics/metrics v1.36.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/bufbuild/protocompile v0.14.1 h1:iA73zAf/fyljNjQKwYzUHD6AD4R8KMasmwa/FBatYVw= +github.com/bufbuild/protocompile v0.14.1/go.mod h1:ppVdAIhbr2H8asPk6k4pY7t9zB1OU5DoEw9xY/FUi1c= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dsnet/golib/unitconv v1.0.2 h1:45gXng3Op1vTrnX1PdM9Bla4mEpBFYA5aC8dlqacmwM= +github.com/dsnet/golib/unitconv v1.0.2/go.mod h1:86KTUtTJFLreKjc4sS9xE0rhj4lR44Ox0rEQSEXSWwM= +github.com/fanjindong/go-cache v0.0.6 h1:4xl8MnfW8pFLH9cRjs0uNfVbFNqV342yl/pgX3Ql9gM= +github.com/fanjindong/go-cache v0.0.6/go.mod h1:gxehZ3SqUVta6eFBJAcDlXDT2Q9piXkUqv7s4E0Vj6o= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/govalues/decimal v0.1.36 h1:dojDpsSvrk0ndAx8+saW5h9WDIHdWpIwrH/yhl9olyU= +github.com/govalues/decimal v0.1.36/go.mod h1:Ee7eI3Llf7hfqDZtpj8Q6NCIgJy1iY3kH1pSwDrNqlM= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI= +github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4= +github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jhump/protoreflect/v2 v2.0.0-beta.2 h1:qZU+rEZUOYTz1Bnhi3xbwn+VxdXkLVeEpAeZzVXLY88= +github.com/jhump/protoreflect/v2 v2.0.0-beta.2/go.mod h1:4tnOYkB/mq7QTyS3YKtVtNrJv4Psqout8HA1U+hZtgM= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/oapi-codegen/runtime v1.0.0 h1:P4rqFX5fMFWqRzY9M/3YF9+aPSPPB06IzP2P7oOxrWo= +github.com/oapi-codegen/runtime v1.0.0/go.mod h1:LmCUMQuPB4M/nLXilQXhHw+BLZdDb18B34OO356yJ/A= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= +github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= +github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= +github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ= +github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ= +github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/api/v3 v3.6.1 h1:yJ9WlDih9HT457QPuHt/TH/XtsdN2tubyxyQHSHPsEo= +go.etcd.io/etcd/api/v3 v3.6.1/go.mod h1:lnfuqoGsXMlZdTJlact3IB56o3bWp1DIlXPIGKRArto= +go.etcd.io/etcd/client/pkg/v3 v3.6.1 h1:CxDVv8ggphmamrXM4Of8aCC8QHzDM4tGcVr9p2BSoGk= +go.etcd.io/etcd/client/pkg/v3 v3.6.1/go.mod h1:aTkCp+6ixcVTZmrJGa7/Mc5nMNs59PEgBbq+HCmWyMc= +go.etcd.io/etcd/client/v3 v3.6.1 h1:KelkcizJGsskUXlsxjVrSmINvMMga0VWwFF0tSPGEP0= +go.etcd.io/etcd/client/v3 v3.6.1/go.mod h1:fCbPUdjWNLfx1A6ATo9syUmFVxqHH9bCnPLBZmnLmMY= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= +golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb h1:p31xT4yrYrSM/G4Sn2+TNUkVhFCbG9y8itM2S6Th950= +google.golang.org/genproto/googleapis/api v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:jbe3Bkdp+Dh2IrslsFCklNhweNTBgSYanP1UXhJDhKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1:TLPQVbx1GJ8VKZxz52VAxl1EBgKXXbTiU9Fc5fZeLn4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= +gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4= +gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/internal/exchange/exchange.go b/internal/exchange/exchange.go new file mode 100644 index 0000000..5b23d40 --- /dev/null +++ b/internal/exchange/exchange.go @@ -0,0 +1,22 @@ +package exchange + +import "sig-pub/pkg/types" + +// 交易所行情数据订阅 +type ExchangeSubscriber interface { + // 交易所类型 + ExhcangeType() types.Exchange + + // 消费k线行情数据 + ConsumerKline() <-chan *types.ChannelKline + + // 订阅产品k线行情 + SubscribeKline(instIds ...string) (err error) + + // 取消订阅产品k线行情 + UnsubscribeKline(instIds ...string) (err error) +} + +// 交易所行情数据请求 +type ExchangeFetcher interface { +} diff --git a/internal/exchange/exchange_data_service.go b/internal/exchange/exchange_data_service.go new file mode 100644 index 0000000..547021b --- /dev/null +++ b/internal/exchange/exchange_data_service.go @@ -0,0 +1,34 @@ +package exchange + +import ( + vmts "sig-pub/pkg/storage/tsdb/victoria_metrics" + "sig-pub/pkg/types" +) + +type ExchangeDataService struct { + vmdb *vmts.VictoriaMetricsTSDB + exchangeFetch ExchangeFetcher +} + +func NewExchangeDataService( + vmdb *vmts.VictoriaMetricsTSDB, + exchangeFetch ExchangeFetcher, +) *ExchangeDataService { + return &ExchangeDataService{ + vmdb: vmdb, + exchangeFetch: exchangeFetch, + } +} + +func (svc *ExchangeDataService) Init() (err error) { + return +} + +func (svc *ExchangeDataService) SaveKlines(inst types.TradeInstance, klines []*types.Kline) (err error) { + err = svc.vmdb.SaveKlines(inst, klines) + if err != nil { + return + } + // todo klines 时间点回溯检查 + return +} diff --git a/internal/exchange/exchange_grpc_server.go b/internal/exchange/exchange_grpc_server.go new file mode 100644 index 0000000..e4a5ca1 --- /dev/null +++ b/internal/exchange/exchange_grpc_server.go @@ -0,0 +1,271 @@ +package exchange + +import ( + "context" + "fmt" + "io" + "sig-pub/api/pb" + "sig-pub/pkg/aside" + "sig-pub/pkg/data/entity" + "sig-pub/pkg/types" + "sig-pub/pkg/zlog" + "sync" + "sync/atomic" + + "google.golang.org/grpc" +) + +type Exchange struct { + Type pb.Exchange + Subscriber ExchangeSubscriber + Insts map[string]*entity.TradeInstanceExchange + sync.RWMutex +} + +type ExchangeGrpcServer struct { + pb.UnimplementedExchangeServiceServer + exchangeMap map[pb.Exchange]*Exchange + tradeInstanceAside *aside.TradeInstanceAside + exchangeDataService *ExchangeDataService + + klineStreamId int64 + klinePublisher *Publisher[int64, grpc.BidiStreamingServer[pb.ReqStreamSubscribeKline, pb.RspStreamSubscribeKline]] +} + +// exchanges: 支持的数据源交易所 +func NewExchangeGrpcServer(tradeInstanceAside *aside.TradeInstanceAside, exchangeDataService *ExchangeDataService, exchanges ...ExchangeSubscriber) *ExchangeGrpcServer { + exchangeMap := make(map[pb.Exchange]*Exchange) + for _, exchange := range exchanges { + exchangeType := exchange.ExhcangeType() + pbExchangeType, ok := exchangeType.Exchange2PB() + if !ok { + panic(fmt.Errorf("unknown exchange: %#v", exchangeType)) + } + exchangeMap[pbExchangeType] = &Exchange{ + Type: pbExchangeType, + Subscriber: exchange, + Insts: make(map[string]*entity.TradeInstanceExchange), + } + } + + return &ExchangeGrpcServer{ + exchangeMap: exchangeMap, + tradeInstanceAside: tradeInstanceAside, + exchangeDataService: exchangeDataService, + klinePublisher: NewPublisher[int64, grpc.BidiStreamingServer[pb.ReqStreamSubscribeKline, pb.RspStreamSubscribeKline]](16), + } +} + +func (svc *ExchangeGrpcServer) Init() (err error) { + svc.subscribeExchanges() + return +} + +// 订阅交易所推送行情 +func (svc *ExchangeGrpcServer) subscribeExchanges() { + // consumerKline + // 交易所订阅交易产品 + for _, exchange := range svc.exchangeMap { + go func(exchange *Exchange) { + // get exchange trade instances + insts, err := svc.tradeInstanceAside.ListExchangeTradeInstance(context.Background(), exchange.Type) + if err != nil { + zlog.Error(err) + return + } + var exchangeInstIds []string + exchange.Lock() + for _, inst := range insts { + exchangeInstIds = append(exchangeInstIds, inst.ExchangeInstId) + exchange.Insts[inst.ExchangeInstId] = inst + } + exchange.Unlock() + + // instIds := []string{"BTC-USDT", "DOGE-USDT-SWAP"} + err = exchange.Subscriber.SubscribeKline(exchangeInstIds...) + if err != nil { + zlog.Error(err) + return + } + c := exchange.Subscriber.ConsumerKline() + svc.consumerKline(exchange, c) + // todo subscribe books 订单簿 + zlog.Infof("unsubscribe exchange: %s", exchange.Type.String()) + }(exchange) + } +} + +// consumerKline 消费交易所k线数据 +func (svc *ExchangeGrpcServer) consumerKline(exchange *Exchange, c <-chan *types.ChannelKline) { + for { + channelK, ok := <-c + if !ok { + return + } + + // 交易所 instid 转 sig-instid + var exInst *entity.TradeInstanceExchange + exchange.RLock() + if inst, ok := exchange.Insts[channelK.InstId]; ok && inst != nil { + exchange.RUnlock() + exInst = inst + } else { + exchange.RUnlock() + zlog.Errorf("unknown exchange instId: %v, %s", channelK.Exchange, channelK.InstId) + continue + } + + // tsdb storage + typeInst := types.TradeInstance{ + InstId: exInst.InstId, // channelK.InstId + TickSz: 0, + MinSz: 0, + } + // todo 异步处理 + err := svc.exchangeDataService.SaveKlines(typeInst, channelK.Klines) + if err != nil { + zlog.Errorf("kline save to tsdb error: ", err) + } + + // publish to subscribers + pubMsgMap := make(map[string]*pb.StreamKline) + + // instId := channelK.InstId + exchange, ok := channelK.Exchange.Exchange2PB() + if !ok { + zlog.Errorf("unknown exchange kline: %v", channelK.Exchange) + continue + } + exchangeName := exchange.String() + + for _, kline := range channelK.Klines { + // zlog.Infof("recv kline: %#v", kline) + confirm := 0 + if kline.Confirm { + confirm = 1 + } + pubKey := fmt.Sprintf("/kline/%s/%s/%s/%d", exchangeName, exInst.InstId, kline.Interval, confirm) + // todo 优化没有订阅者就跳过 + msg, ok := pubMsgMap[pubKey] + if !ok { + msg = new(pb.StreamKline) + msg.InstId = exInst.InstId + msg.Exchange = exchange + pubMsgMap[pubKey] = msg + } + + pbk := kline.ToPBKline() + msg.Klines = append(msg.Klines, pbk) + } + + for pubKey, msg := range pubMsgMap { + if len(msg.Klines) == 0 { + continue + } + subs := svc.klinePublisher.Publisher(pubKey) + for _, sub := range subs { + if err := sub.Send(&pb.RspStreamSubscribeKline{Kline: msg}); err != nil { + zlog.Error(err) + } + } + } + } +} + +// SubscribeKline 订阅k线stream +func (svc *ExchangeGrpcServer) SubscribeKline(stream grpc.BidiStreamingServer[pb.ReqStreamSubscribeKline, pb.RspStreamSubscribeKline]) (err0 error) { + streamId := atomic.AddInt64(&svc.klineStreamId, 1) + // subKey = /kline/exchange/instId/interval/confirm + + // 接收消息的goroutine + recvChan := make(chan *pb.ReqStreamSubscribeKline) + go func() { + for { + msg, err := stream.Recv() + if err == io.EOF { + zlog.Infof("关闭EOF: %v", err) + close(recvChan) + return + } + if err != nil { + zlog.Infof("接收错误: %v", err) + close(recvChan) + return + } + zlog.Infof("recv stream msg: %#v", msg) + recvChan <- msg + } + }() + + // 发送和处理消息 + for { + select { + case <-stream.Context().Done(): + // 客户端断开连接 + svc.klinePublisher.UnsubscribeAll(streamId) + return stream.Context().Err() + case msg, ok := <-recvChan: + if !ok { + // 接收通道关闭，结束流 + svc.klinePublisher.UnsubscribeAll(streamId) + return + } + + if msg.SubType == pb.SubscribeType_UnsubscribeAll { + svc.klinePublisher.UnsubscribeAll(streamId) + continue + } + for _, exchange := range msg.Exchanges { + for _, instId := range msg.InstIds { + for _, interval := range msg.Intervals { + confirms := []int{1} + if !msg.OnlyConfirm { + confirms = append(confirms, 0) + } + for _, confirm := range confirms { + subKey := fmt.Sprintf("/kline/%s/%s/%s/%d", exchange.String(), instId, interval, confirm) + zlog.Infof("stream: %d sub: %s", streamId, subKey) + switch msg.SubType { + case pb.SubscribeType_Subscribe: + svc.klinePublisher.Subscribe(subKey, streamId, stream) + case pb.SubscribeType_Unsubscribe: + svc.klinePublisher.Unsubscribe(subKey, streamId) + } + } + } + } + } + } + } + + // for { + // select { + // case <-ctx.Done(): + // // 客户端断开连接 + // // log.Printf("Client disconnected from topic: %s", topic) + // return + // default: + // // 模拟事件生成 + // event := &pb.SubscribeKlineStreamRsp{ + // Kline: &pb.StreamKline{ + // InstId: "DOGE/USDT", + // Exchange: pb.Exchange_OKX, + // Klines: []*pb.Kline{ + // { + // Ts: time.Now().Unix(), + // }, + // }, + // }, + // } + + // // 推送事件 + // if err := streamRsp.Send(event); err != nil { + // log.Printf("Failed to send event to client: %v", err) + // return err + // } + + // // 模拟事件间隔 + // time.Sleep(2 * time.Second) + // } + // } +} diff --git a/internal/exchange/okx/channel_books.go b/internal/exchange/okx/channel_books.go new file mode 100644 index 0000000..fbedda9 --- /dev/null +++ b/internal/exchange/okx/channel_books.go @@ -0,0 +1,72 @@ +package okx + +// 深度频道 https://my.okx.com/docs-v5/zh/#order-book-trading-market-data-ws-order-book-channel +// 获取深度数据，books是400档频道，books5是5档频道， bbo-tbt是先1档后实时推送的频道，books-l2-tbt是先400档后实时推送的频道，books50-l2-tbt是先50档后实时推的频道； +// books 首次推400档快照数据，以后增量推送，每100毫秒推送一次变化的数据 +// books5 首次推5档快照数据，以后定量推送，每100毫秒当5档快照数据有变化推送一次5档数据 +// bbo-tbt 首次推1档快照数据，以后定量推送，每10毫秒当1档快照数据有变化推送一次1档数据 +// books-l2-tbt 首次推400档快照数据，以后增量推送，每10毫秒推送一次变化的数据 +// books50-l2-tbt 首次推50档快照数据，以后增量推送，每10毫秒推送一次变化的数据 +// 单个连接、交易产品维度，深度频道的推送顺序固定为：bbo-tbt -> books-l2-tbt -> books50-l2-tbt -> books -> books5。 +// books-l2-tbt400档深度频道，只允许交易手续费等级VIP5及以上的API用户订阅。 +// books50-l2-tbt50档深度频道，只允许交易手续费等级VIP4及以上的API用户订阅. + +type BooksData struct { + Ts string `json:"ts"` // 成交时间，Unix时间戳的毫秒数格式，如 1597026383085 + Checksum int `json:"checksum"` // 检验和 （下方注解） + PrevSeqId int `json:"prevSeqId"` // 上一个推送的序列号。仅适用 books，books-l2-tbt，books50-l2-tbt + SeqId int `json:"seqId"` // 推送的序列号 （下方注解） + Asks [][]string `json:"asks"` // 卖方深度 asks和bids格式:[[深度价格,此价格的数量,弃用(始终为0),此价格的订单数量]] + Bids [][]string `json:"bids"` // 买方深度 +} + +// ChannelBooks 交易频道 +type ChannelBooks struct { + cfg wsChannelConfig[*[]BooksData, *ChannelData[*[]BooksData]] + wsChannel *wsChannel[*[]BooksData, *ChannelData[*[]BooksData]] +} + +func NewChannelBooks( + traceId string, + httpProxy string, +) *ChannelBooks { + cfg := wsChannelConfig[*[]BooksData, *ChannelData[*[]BooksData]]{ + channelId: traceId, + httpProxy: httpProxy, + wsUrl: "/ws/v5/public", + subscribeChannels: []string{"books5"}, + dataInstanceFunc: func() *[]BooksData { + var d []BooksData + return &d + }, + dataMappingFunc: func(data *ChannelData[*[]BooksData]) (*ChannelData[*[]BooksData], error) { + return data, nil + }, + } + return &ChannelBooks{cfg: cfg} +} + +func (c *ChannelBooks) Init() (err error) { + c.wsChannel = newWsChannel(c.cfg) + return c.wsChannel.Init() +} + +func (c *ChannelBooks) Consumer() <-chan *ChannelData[*[]BooksData] { + return c.wsChannel.Consumer() +} + +func (c *ChannelBooks) Subscribe(instIds ...string) (err error) { + return c.wsChannel.Subscribe(instIds...) +} + +func (c *ChannelBooks) Unsubscribe(instIds ...string) (err error) { + return c.wsChannel.Unsubscribe(instIds...) +} + +func (c *ChannelBooks) IsSubscribe(instId string) bool { + return c.wsChannel.IsSubscribed(instId) +} + +func (c *ChannelBooks) GetSubscribes() (instIds []string) { + return c.wsChannel.GetSubscribes() +} diff --git a/internal/exchange/okx/channel_kline.go b/internal/exchange/okx/channel_kline.go new file mode 100644 index 0000000..01983d6 --- /dev/null +++ b/internal/exchange/okx/channel_kline.go @@ -0,0 +1,162 @@ +package okx + +import ( + "fmt" + "sig-pub/pkg/types" + "strconv" + + "github.com/govalues/decimal" +) + +// WS/K线频道 +// 获取K线数据，推送频率最快是间隔1秒推送一次数据。 + +type CandleData [][]string + +var ( + // k线类型 + // 如 [1s/1m/3m/5m/15m/30m/1H/2H/4H] + // 香港时间开盘价k线：[6H/12H/1D/2D/3D/1W/1M/3M] + // UTC时间开盘价k线：[6Hutc/12Hutc/1Dutc/2Dutc/3Dutc/1Wutc/1Mutc/3Mutc] + // "candle1s" candle3Mutc candle1Mutc candle1Wutc candle1Dutc candle2Dutc candle3Dutc candle5Dutc candle12Hutc candle6Hutc + // candles = []string{"candle3M", "candle1M", "candle1W", "candle1D", "candle2D", "candle3D", "candle5D", "candle12H", "candle6H", "candle4H", "candle2H", "candle1H", "candle30m", "candle15m", "candle5m", "candle3m", "candle1m"} + subCandles = []string{ + // "candle3M", "candle1M", + "candle1W", "candle1D", "candle2D", "candle3D", "candle5D", + "candle12H", "candle6H", "candle4H", "candle2H", "candle1H", + "candle30m", "candle15m", "candle5m", "candle3m", "candle1m", + "candle1s", + } +) + +// ChannelCandle k线订阅频道 +type ChannelCandle struct { + cfg wsChannelConfig[*CandleData, *types.ChannelKline] + wsChannel *wsChannel[*CandleData, *types.ChannelKline] +} + +func NewChannelCandle( + traceId string, + httpProxy string, +) *ChannelCandle { + cfg := wsChannelConfig[*CandleData, *types.ChannelKline]{ + channelId: traceId, + httpProxy: httpProxy, + wsUrl: "/ws/v5/business", + subscribeChannels: subCandles, + dataInstanceFunc: func() *CandleData { + var d CandleData + return &d + }, + dataMappingFunc: candleData2Klines, + } + return &ChannelCandle{cfg: cfg} +} + +func (c *ChannelCandle) Init() (err error) { + c.wsChannel = newWsChannel(c.cfg) + return c.wsChannel.Init() +} + +func (c *ChannelCandle) Consumer() <-chan *types.ChannelKline { + return c.wsChannel.Consumer() +} + +func (c *ChannelCandle) Subscribe(instIds ...string) (err error) { + return c.wsChannel.Subscribe(instIds...) +} + +func (c *ChannelCandle) Unsubscribe(instIds ...string) (err error) { + return c.wsChannel.Unsubscribe(instIds...) +} + +func (c *ChannelCandle) IsSubscribe(instId string) bool { + return c.wsChannel.IsSubscribed(instId) +} + +func (c *ChannelCandle) GetSubscribes() (instIds []string) { + return c.wsChannel.GetSubscribes() +} + +// func candleData2Klines() (klines []*types.Kline) { +func candleData2Klines(channelData *ChannelData[*CandleData]) (r *types.ChannelKline, err error) { + r = &types.ChannelKline{ + InstId: channelData.InstId, + Exchange: types.ExchangeOKX, + } + for _, data := range *channelData.Data { + ts, e := strconv.ParseInt(data[0], 10, 64) + if e != nil { + err = e + return + } + + kline := &types.Kline{ + Ts: ts, + Confirm: data[8] == "1", + } + + switch channelData.Channel { + default: + err = fmt.Errorf("unsupport candle: %s", channelData.Channel) + continue + case "candle1s": + kline.Interval = types.Interval1s + // case "candle5s": + // kline.Interval = types.Interval5s + case "candle1m": + kline.Interval = types.Interval1m + case "candle3m": + kline.Interval = types.Interval3m + case "candle5m": + kline.Interval = types.Interval5m + case "candle15m": + kline.Interval = types.Interval15m + case "candle30m": + kline.Interval = types.Interval30m + case "candle1H": + kline.Interval = types.Interval1h + case "candle2H": + kline.Interval = types.Interval2h + case "candle4H": + kline.Interval = types.Interval4h + case "candle6H": + kline.Interval = types.Interval6h + case "candle12H": + kline.Interval = types.Interval12h + case "candle1D": + kline.Interval = types.Interval1d + case "candle2D": + kline.Interval = types.Interval2d + case "candle3D": + kline.Interval = types.Interval3d + case "candle5D": + kline.Interval = types.Interval5d + case "candle1W": + kline.Interval = types.Interval1w + // case "candle1M": + // kline.Interval = types.Interval1mo + // case "candle3M": + // kline.Interval = types.Interval3mo + } + var kinds = []*decimal.Decimal{&kline.Open, &kline.High, &kline.Low, &kline.Close, &kline.Vol, nil, &kline.VolQuote} + for i := 1; i <= 7; i++ { + if kinds[i-1] == nil { + continue + } + *kinds[i-1], err = decimal.Parse(data[i]) + if err != nil { + return + } + } + + // ims, ok := kline.Interval.Milliseconds() + // if !ok { + // err = fmt.Errorf("unsupport interval: %v", kline.Interval) + // continue + // } + // kline.Tid = ts / ims + r.Klines = append(r.Klines, kline) + } + return +} diff --git a/internal/exchange/okx/channel_tickers.go b/internal/exchange/okx/channel_tickers.go new file mode 100644 index 0000000..c49ad8e --- /dev/null +++ b/internal/exchange/okx/channel_tickers.go @@ -0,0 +1,75 @@ +package okx + +// 行情频道 channel: tickers +// 获取产品的最新成交价、买一价、卖一价和24小时交易量等信息。 +// 最快100ms推送一次，没有触发事件时不推送，触发推送的事件有：成交、买一卖一发生变动。 + +type TickersData struct { + InstType string `json:"instType"` // 产品类型 + InstId string `json:"instId"` // 产品ID + Last string `json:"last"` // 最新成交价 + LastSz string `json:"lastSz"` // 最新成交的数量，0 代表没有成交量 + AskPx string `json:"askPx"` // 卖一价 + AskSz string `json:"askSz"` // 卖一价对应的数量 + BidPx string `json:"bidPx"` // 买一价 + BidSz string `json:"bidSz"` // 买一价对应的数量 + Open24h string `json:"open24h"` // 24小时开盘价 + High24h string `json:"high24h"` // 24小时最高价 + Low24h string `json:"low24h"` // 24小时最低价 + VolCcy24h string `json:"volCcy24h"` // 24小时成交量，以币为单位如果是衍生品合约，数值为交易货币的数量。如果是币币/币币杠杆，数值为计价货币的数量。 + Vol24h string `json:"vol24h"` // 24小时成交量，以张为单位如果是衍生品合约，数值为合约的张数。如果是币币/币币杠杆，数值为交易货币的数量。 + SodUtc0 string `json:"sodUtc0"` // UTC+0 时开盘价 + SodUtc8 string `json:"sodUtc8"` // UTC+8 时开盘价 + Ts string `json:"ts"` // ticker数据产生时间，Unix时间戳的毫秒数格式，如 1597026383085 +} + +// ChannelTickers 行情频道 +type ChannelTickers struct { + cfg wsChannelConfig[*[]TickersData, *ChannelData[*[]TickersData]] + wsChannel *wsChannel[*[]TickersData, *ChannelData[*[]TickersData]] +} + +func NewChannelTickers( + traceId string, + httpProxy string, +) *ChannelTickers { + cfg := wsChannelConfig[*[]TickersData, *ChannelData[*[]TickersData]]{ + channelId: traceId, + httpProxy: httpProxy, + wsUrl: "/ws/v5/public", + subscribeChannels: []string{"tickers"}, + dataInstanceFunc: func() *[]TickersData { + var d []TickersData + return &d + }, + dataMappingFunc: func(data *ChannelData[*[]TickersData]) (*ChannelData[*[]TickersData], error) { + return data, nil + }, + } + return &ChannelTickers{cfg: cfg} +} + +func (c *ChannelTickers) Init() (err error) { + c.wsChannel = newWsChannel(c.cfg) + return c.wsChannel.Init() +} + +func (c *ChannelTickers) Consumer() <-chan *ChannelData[*[]TickersData] { + return c.wsChannel.Consumer() +} + +func (c *ChannelTickers) Subscribe(instIds ...string) (err error) { + return c.wsChannel.Subscribe(instIds...) +} + +func (c *ChannelTickers) Unsubscribe(instIds ...string) (err error) { + return c.wsChannel.Unsubscribe(instIds...) +} + +func (c *ChannelTickers) IsSubscribe(instId string) bool { + return c.wsChannel.IsSubscribed(instId) +} + +func (c *ChannelTickers) GetSubscribes() (instIds []string) { + return c.wsChannel.GetSubscribes() +} diff --git a/internal/exchange/okx/channel_trades.go b/internal/exchange/okx/channel_trades.go new file mode 100644 index 0000000..2338274 --- /dev/null +++ b/internal/exchange/okx/channel_trades.go @@ -0,0 +1,66 @@ +package okx + +// 交易频道 channel: trades +// 获取最近的成交数据，有成交数据就推送，每次推送可能聚合多条成交数据。 +// 根据每个taker订单的不同成交价格推送消息，并使用count字段表示聚合的订单匹配数量。 + +type TradesData struct { + InstId string `json:"instId"` // 产品ID，如 DOGE-USDT-SWAP + TradeId string `json:"tradeId"` // 聚合的多笔交易中最新一笔交易的成交ID + Px string `json:"px"` // 成交价格 + Sz string `json:"sz"` // 成交数量 + Side TradeSide `json:"side"` // 成交方向buy/sell + Ts string `json:"ts"` // 成交时间，Unix时间戳的毫秒数格式，如 1597026383085 + Count string `json:"count"` // 聚合的订单匹配数量 +} + +// ChannelTrades 交易频道 +type ChannelTrades struct { + cfg wsChannelConfig[*[]TradesData, *ChannelData[*[]TradesData]] + wsChannel *wsChannel[*[]TradesData, *ChannelData[*[]TradesData]] +} + +func NewChannelTrades( + traceId string, + httpProxy string, +) *ChannelTrades { + cfg := wsChannelConfig[*[]TradesData, *ChannelData[*[]TradesData]]{ + channelId: traceId, + httpProxy: httpProxy, + wsUrl: "/ws/v5/public", + subscribeChannels: []string{"trades"}, + dataInstanceFunc: func() *[]TradesData { + var d []TradesData + return &d + }, + dataMappingFunc: func(data *ChannelData[*[]TradesData]) (*ChannelData[*[]TradesData], error) { + return data, nil + }, + } + return &ChannelTrades{cfg: cfg} +} + +func (c *ChannelTrades) Init() (err error) { + c.wsChannel = newWsChannel(c.cfg) + return c.wsChannel.Init() +} + +func (c *ChannelTrades) Consumer() <-chan *ChannelData[*[]TradesData] { + return c.wsChannel.Consumer() +} + +func (c *ChannelTrades) Subscribe(instIds ...string) (err error) { + return c.wsChannel.Subscribe(instIds...) +} + +func (c *ChannelTrades) Unsubscribe(instIds ...string) (err error) { + return c.wsChannel.Unsubscribe(instIds...) +} + +func (c *ChannelTrades) IsSubscribe(instId string) bool { + return c.wsChannel.IsSubscribed(instId) +} + +func (c *ChannelTrades) GetSubscribes() (instIds []string) { + return c.wsChannel.GetSubscribes() +} diff --git a/internal/exchange/okx/okx.go b/internal/exchange/okx/okx.go new file mode 100644 index 0000000..cbb5b49 --- /dev/null +++ b/internal/exchange/okx/okx.go @@ -0,0 +1,45 @@ +package okx + +import ( + "sig-pub/pkg/config" + "sig-pub/pkg/types" +) + +// kline +type OkxExchange struct { + conf config.OkxExchange + channelCandle *ChannelCandle // K线频道 +} + +func NewOkxExchange(conf config.OkxExchange) *OkxExchange { + return &OkxExchange{ + conf: conf, + } +} + +func (okx *OkxExchange) Init() (err error) { + // TODO 多个 ChannelCandle 实例 OkxAggregate + okx.channelCandle = NewChannelCandle("candle-0", okx.conf.HttpProxy) + if err = okx.channelCandle.Init(); err != nil { + return + } + return +} + +func (okx *OkxExchange) ExhcangeType() types.Exchange { + return types.ExchangeOKX +} + +func (okx *OkxExchange) ConsumerKline() <-chan *types.ChannelKline { + return okx.channelCandle.Consumer() +} + +// 订阅产品k线行情 +func (okx *OkxExchange) SubscribeKline(instIds ...string) (err error) { + return okx.channelCandle.Subscribe(instIds...) +} + +// 取消订阅产品k线行情 +func (okx *OkxExchange) UnsubscribeKline(instIds ...string) (err error) { + return okx.channelCandle.Unsubscribe(instIds...) +} diff --git a/internal/exchange/okx/types.go b/internal/exchange/okx/types.go new file mode 100644 index 0000000..5e2b5f5 --- /dev/null +++ b/internal/exchange/okx/types.go @@ -0,0 +1,67 @@ +package okx + +import ( + "time" +) + +// 产品类型 +type InstType string + +const ( + InstTypeSpot InstType = "SPOT" // SPOT：币币 + InstTypeMargin InstType = "MARGIN" // MARGIN：币币杠杆 + InstTypeSwap InstType = "SWAP" // SWAP：永续合约 + InstTypeFutures InstType = "FUTURES" // FUTURES：交割合约 + InstTypeOption InstType = "OPTION" // OPTION：期权 +) + +type TradeSide string + +const ( + TradeSideBuy TradeSide = "buy" + TradeSideSell TradeSide = "sell" +) + +// 推送数据动作,增量推送数据还是全量推送数据 +type ActionType string + +const ( + ActionTypeSnapshot ActionType = "snapshot" // 全量 + ActionTypeUpdate ActionType = "update" // 增量 +) + +type ChannelEventType string + +const ( + ChannelEventTypeLogin ChannelEventType = "login" + ChannelEventTypeError ChannelEventType = "error" + ChannelEventTypeSubscribe ChannelEventType = "subscribe" + ChannelEventTypeUnsubscribe ChannelEventType = "unsubscribe" + ChannelEventTypeConnectionInfo ChannelEventType = "channel-conn-count" + ChannelEventTypeConnectionError ChannelEventType = "channel-conn-count-error" + ChannelEventTypeNotice ChannelEventType = "notice" +) + +// 订阅频道推送消息 +type ChannelData[T any] struct { + Channel string `json:"channel"` // 订阅的频道 candle1s/candle1m/candle3M + InstId string `json:"instId"` // 产品ID，如 BTC-USDT + Action ActionType `json:"action"` // books交易深度订阅,推送数据动作类型 + Data T `json:"data"` +} + +type Order struct { + Symbol string + Side string // buy/sell + Type string // limit/market + Price float64 + Amount float64 + ContractType string // spot/futures +} + +type MarketData struct { + Symbol string + Price float64 + Volume float64 + Timestamp time.Time +} diff --git a/internal/exchange/okx/ws_channel.go b/internal/exchange/okx/ws_channel.go new file mode 100644 index 0000000..70a445b --- /dev/null +++ b/internal/exchange/okx/ws_channel.go @@ -0,0 +1,276 @@ +package okx + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "sig-pub/pkg/utils/collect" + "sig-pub/pkg/zlog" + "sync" + "time" + + "github.com/bytedance/sonic" + "github.com/gorilla/websocket" + "golang.org/x/net/http/httpproxy" +) + +// https://my.okx.com/docs-v5/zh/#overview-websocket-overview +// 连接限制：3 次/秒 (基于IP) +// 当订阅公有频道时，使用公有服务的地址；当订阅私有频道时，使用私有服务的地址 +// 请求限制：每个连接 对于 订阅/取消订阅/登录 请求的总次数限制为 480 次/小时 + +// 公共频道无需登录，包括行情频道，K线频道，交易数据频道，资金费率频道，限价范围频道，深度数据频道，标记价格频道等。 +// 私有频道需登录，包括用户账户频道，用户交易频道，用户持仓频道等。 + +const ( + WsBaseUrl = "wss://ws.okx.com:8443" + // WsBaseUrl = "wss://wseeapap.okx.com:8443" // 模拟盘 +) + +// SubscribeStatus 订阅状态 +type SubscribeStatus int32 + +const ( + StatusUnsubscribe SubscribeStatus = 0 // 未订阅 + StatusSubscribing SubscribeStatus = 1 // 订阅中 + StatusSubscribed SubscribeStatus = 2 // 已订阅 +) + +// WebSocketEvent okx推送事件 +type WebSocketEvent struct { + // 订阅数据 + Event ChannelEventType `json:"event"` + Code string `json:"code,omitempty"` + Msg string `json:"msg,omitempty"` + ConnId string `json:"connId,omitempty"` + // 推送数据 + Arg WebSocketEventArg `json:"arg,omitempty"` + Action ActionType `json:"action"` // books交易深度订阅,推送数据动作类型 + Data json.RawMessage `json:"data"` +} + +type WebSocketEventArg struct { + Channel string `json:"channel"` // 订阅的频道 candle1s/candle1m/candle3M + InstId string `json:"instId"` // 产品ID +} + +// 订阅配置 +type wsChannelConfig[T, R any] struct { + channelId string // ws日志id + httpProxy string // http 代理 + wsUrl string // 服务地址, K线-> /ws/v5/business + subscribeChannels []string // 订阅频道列表 + dataInstanceFunc func() T // 生成一个data实例用以反序列化 + dataMappingFunc func(*ChannelData[T]) (R, error) // 数据映射函数 +} + +// okx websocket 频道订阅 +// 行情订阅 todo 单连接上限, 上百种币多个ws连接订阅 +type wsChannel[T, R any] struct { + sync.Mutex + channelId string + cfg wsChannelConfig[T, R] + conn *websocket.Conn + subscribeInsts *collect.ConcurrentMap[string, SubscribeStatus] // + dataC chan R // data channel + + prevLogFullMs int64 + // todo subscribe logs... +} + +func newWsChannel[T, R any](cfg wsChannelConfig[T, R]) *wsChannel[T, R] { + subInsts := collect.NewConcurrentMap[string, SubscribeStatus](8, func(k string) string { return k }) + ms := &wsChannel[T, R]{ + channelId: cfg.channelId, + cfg: cfg, + subscribeInsts: subInsts, + dataC: make(chan R, 4*1024), // 4k + } + return ms +} + +func (c *wsChannel[T, R]) Init() (err error) { + if c.cfg.dataInstanceFunc == nil { + return errors.New("ws channel payloadInstanceFunc can't be nil") + } + err = c.connect() + return +} + +func (c *wsChannel[T, R]) connect() (err error) { + var proxyFunc = http.ProxyFromEnvironment + if c.cfg.httpProxy != "" { + // 自定义 http proxy + proxyFunc = func(req *http.Request) (*url.URL, error) { + return (&httpproxy.Config{ + HTTPProxy: c.cfg.httpProxy, + HTTPSProxy: c.cfg.httpProxy, + }).ProxyFunc()(req.URL) + } + } + wsDialer := websocket.Dialer{ + Proxy: proxyFunc, + HandshakeTimeout: 10 * time.Second, + } + + url := fmt.Sprintf("%s%s", WsBaseUrl, c.cfg.wsUrl) + conn, _, err := wsDialer.Dial(url, nil) + if err != nil { + zlog.Error("Error connecting to websocket channel:", url, err) + go c.reconnect() + return + } + c.conn = conn + zlog.Infof("websocket channel %s connected: %s", c.channelId, url) + + go c.readPump() + return +} + +func (c *wsChannel[T, R]) reconnect() { + zlog.Infof("channel %s reconnecting", c.channelId) + + var err error + func() { + c.Lock() + defer c.Unlock() + + c.close() + // wait 1sec + <-time.After(time.Second) + err = c.connect() + }() + if err != nil { + return + } + + // 重新订阅 + var insts []string + c.subscribeInsts.Range(func(instId string, status SubscribeStatus) bool { + if status == StatusUnsubscribe { + insts = append(insts, instId) + } + return true + }) + + if len(insts) > 0 { + err = c.Subscribe(insts...) + if err != nil { + zlog.Error("channel %s subscribe %v error", c.channelId, insts, err) + go c.reconnect() + } + } +} + +func (c *wsChannel[T, R]) close() { + if c.conn != nil { + err := c.conn.Close() + if err != nil { + zlog.Infof("channel %s close error: %v", c.channelId, err) + } + c.conn = nil + } + + // 重置到待订阅状态 + c.subscribeInsts.RangeUpdate(func(instId string, status SubscribeStatus) (bool, bool, SubscribeStatus) { + return true, false, StatusUnsubscribe + }) +} + +// 从ws读取数据并解析 +func (c *wsChannel[T, R]) readPump() { + for { + t, bytes, err := c.conn.ReadMessage() + if err != nil { + go c.reconnect() + zlog.Errorf("channel %s reading message error: %v", c.channelId, err) + return + } + switch t { + default: + zlog.Errorf("channel %s unknown message type: %d", c.channelId, t) + continue + case websocket.PingMessage: + c.conn.WriteMessage(websocket.PongMessage, []byte{}) + continue + case websocket.CloseMessage: + go c.reconnect() + zlog.Infof("channel %s close message", c.channelId) + return + case websocket.TextMessage: + } + + // fmt.Println(string(bytes)) + var event WebSocketEvent + if err = sonic.Unmarshal(bytes, &event); err != nil { + zlog.Errorf("parse websocket event error: ", err) + continue + } + + switch event.Event { + case ChannelEventTypeSubscribe: + c.subscribeInsts.Store(event.Arg.InstId, StatusSubscribed) + zlog.Infof("channel %s subscribed %s %s", c.channelId, event.Arg.InstId, event.Arg.Channel) + continue + case ChannelEventTypeUnsubscribe: + c.subscribeInsts.Delete(event.Arg.InstId) + zlog.Infof("channel %s unsubscribed %s", c.channelId, event.Arg.InstId) + continue + case ChannelEventTypeError: + zlog.Errorf("channel %s event error: %#v", c.channelId, event) + continue + case ChannelEventTypeConnectionInfo: // 新链接订阅频道时, 消息同步链接数量 + zlog.Infof("channel-conn-count event %s: %#v", c.channelId, event) + continue + case ChannelEventTypeConnectionError: // 当超出限制时 + zlog.Errorf("channel-conn-count-error event %s: %#v", c.channelId, event) + continue + case ChannelEventTypeNotice: // websocket 服务升级断线通知 + zlog.Errorf("channel event type notice %s: %#v", c.channelId, event) + continue + // case WsEventTypeLogin: // todo + default: + // 推送数据 + } + if event.Event != "" { + zlog.Infof("unhandle event %s: %#v", c.channelId, event) + continue + } + + // 解析数据 + data := c.cfg.dataInstanceFunc() + if err := sonic.Unmarshal(event.Data, data); err != nil { + zlog.Error("unmarshal event data error: ", err) + continue + } + // zlog.Infof("read data: %v", data) + + // 数据打包 + channelData := &ChannelData[T]{ + Channel: event.Arg.Channel, + InstId: event.Arg.InstId, + Action: event.Action, + Data: data, + } + + // 数据映射 + dataR, err := c.cfg.dataMappingFunc(channelData) + if err != nil { + zlog.Error("okx channel data mapping error: ", err) + continue + } + + select { + case c.dataC <- dataR: + default: + // channel 满了 + now := time.Now().UnixMilli() + if now-c.prevLogFullMs > 5000 { // 每5秒打印不要太频繁 + c.prevLogFullMs = now + zlog.Warningf("DataC %s full, sub %d insts", c.channelId, c.subscribeInsts.Size()) + } + } + } +} diff --git a/internal/exchange/okx/ws_channel_subscribe.go b/internal/exchange/okx/ws_channel_subscribe.go new file mode 100644 index 0000000..e49bed8 --- /dev/null +++ b/internal/exchange/okx/ws_channel_subscribe.go @@ -0,0 +1,104 @@ +package okx + +import ( + "sig-pub/pkg/zlog" + + "github.com/bytedance/sonic" + "github.com/gorilla/websocket" +) + +func (c *wsChannel[T, R]) Consumer() <-chan R { + return c.dataC +} + +func (c *wsChannel[T, R]) Subscribe(instIds ...string) (err error) { + c.Lock() + defer c.Unlock() + + var args []WebSocketEventArg + for _, instId := range instIds { + if status, ok := c.subscribeInsts.Load(instId); !ok || status == StatusUnsubscribe { + for _, channel := range c.cfg.subscribeChannels { + args = append(args, WebSocketEventArg{ + Channel: channel, + InstId: instId, + }) + } + c.subscribeInsts.Store(instId, StatusSubscribing) // 订阅中 + } + } + if len(args) == 0 { + return + } + + // 发送订阅消息 + subscribeMsg := map[string]any{ + "op": "subscribe", + "args": args, + } + msg, err := sonic.Marshal(subscribeMsg) + if err != nil { + return + } + err = c.conn.WriteMessage(websocket.TextMessage, msg) + if err != nil { + zlog.Errorf("channel %s write subscribe msg error: %v", c.channelId, err) + return + } + return +} + +// Unsubscribe todo 无订阅时关闭 channel +func (c *wsChannel[T, R]) Unsubscribe(instIds ...string) (err error) { + c.Lock() + defer c.Unlock() + + var args []WebSocketEventArg + for _, instId := range instIds { + if status, ok := c.subscribeInsts.Load(instId); ok && status != StatusUnsubscribe { + for _, channel := range c.cfg.subscribeChannels { + args = append(args, WebSocketEventArg{ + Channel: channel, + InstId: instId, + }) + } + c.subscribeInsts.Store(instId, StatusUnsubscribe) // todo 取消订阅中 + } + } + if len(args) == 0 { + return + } + + // send subscribe message + unsubscribeMsg := map[string]any{ + "op": ChannelEventTypeUnsubscribe, + "args": args, + } + msg, err := sonic.Marshal(unsubscribeMsg) + if err != nil { + return + } + err = c.conn.WriteMessage(websocket.TextMessage, msg) + if err != nil { + zlog.Error("Error write connecting to server:", err) + return + } + return +} + +// 产品是否已订阅 +func (c *wsChannel[T, R]) IsSubscribed(instId string) bool { + status, ok := c.subscribeInsts.Load(instId) + return ok && (status == StatusSubscribed) +} + +// 已订阅产品列表 +func (c *wsChannel[T, R]) GetSubscribes() (instIds []string) { + c.subscribeInsts.Range(func(instId string, status SubscribeStatus) bool { + if status == StatusSubscribed { + instIds = append(instIds, instId) + } + return true + }) + return +} diff --git a/internal/exchange/publisher.go b/internal/exchange/publisher.go new file mode 100644 index 0000000..1cacf2e --- /dev/null +++ b/internal/exchange/publisher.go @@ -0,0 +1,59 @@ +package exchange + +import "sig-pub/pkg/utils/collect" + +type Publisher[TID comparable, T any] struct { + m0 *collect.ConcurrentMap[string, map[TID]T] +} + +func NewPublisher[TID comparable, T any](concurrentLevel int) *Publisher[TID, T] { + m := collect.NewConcurrentMap[string, map[TID]T](concurrentLevel, func(k string) string { return k }) + return &Publisher[TID, T]{ + m0: m, + } +} + +// Subscribe 订阅 +func (p *Publisher[TID, T]) Subscribe(k string, tid TID, ele T) { + _ = p.m0.LoadAndUpdate(k, func(v map[TID]T) (remove bool, nextV map[TID]T) { + if v == nil { + v = make(map[TID]T) + } + v[tid] = ele + return false, v + }) +} + +// Unsubscribe 取消订阅 +func (p *Publisher[TID, T]) Unsubscribe(k string, tid TID) { + p.m0.LoadAndUpdate(k, func(v map[TID]T) (remove bool, nextV map[TID]T) { + delete(v, tid) + if len(v) == 0 { + return true, v + } + return false, v + }) +} + +// UnsubscribeAll 取消所有订阅 +func (p *Publisher[TID, T]) UnsubscribeAll(tid TID) { + p.m0.RangeUpdate(func(key string, v map[TID]T) (next bool, remove bool, newV map[TID]T) { + delete(v, tid) + remove = len(v) == 0 + return true, remove, v + }) +} + +// Publisher 获取匹配的 subscribers +func (p *Publisher[TID, T]) Publisher(k string) (subs []T) { + p.m0.LoadRLock(k, func(subIds map[TID]T, ok bool) { + if !ok { + return + } + for _, sub := range subIds { + subs = append(subs, sub) + } + }) + // todo cache subs + return +} diff --git a/internal/gateway/gateway.go b/internal/gateway/gateway.go new file mode 100644 index 0000000..1f28840 --- /dev/null +++ b/internal/gateway/gateway.go @@ -0,0 +1,35 @@ +package gateway + +import ( + "net/http" + "runtime/debug" + "sig-pub/pkg/utils/resp" + "sig-pub/pkg/zlog" + + "github.com/gin-gonic/gin" +) + +// grpc generic call + +func Route() *gin.Engine { + router := gin.Default() + router.Use(func(c *gin.Context) { + // global recover + defer func() { + if r := recover(); r != nil { + zlog.Error("http server recover error:", r) + debug.PrintStack() + c.JSON(http.StatusInternalServerError, resp.Error("server error")) + c.Abort() + } + }() + + c.Next() + }) + + router.GET("/api/hello", func(c *gin.Context) { + c.JSON(http.StatusOK, resp.Success("hello")) + }) + + return router +} diff --git a/internal/indicator/indicator.go b/internal/indicator/indicator.go new file mode 100644 index 0000000..a543720 --- /dev/null +++ b/internal/indicator/indicator.go @@ -0,0 +1,3 @@ +package indicator + +// load indicator plugin diff --git a/internal/market/market_grpc_server.go b/internal/market/market_grpc_server.go new file mode 100644 index 0000000..07a19e3 --- /dev/null +++ b/internal/market/market_grpc_server.go @@ -0,0 +1,73 @@ +package market + +import ( + "context" + "fmt" + "sig-pub/api/pb" + "sig-pub/pkg/grpc/session" + "sig-pub/pkg/mapping" +) + +type MarketGrpcServer struct { + pb.UnimplementedMarketServer + tradeInstanceService *TradeInstanceService +} + +func NewMarketGrpcServer(instanceService *TradeInstanceService) *MarketGrpcServer { + return &MarketGrpcServer{ + tradeInstanceService: instanceService, + } +} + +func (svr *MarketGrpcServer) GetTradeInstance(ctx context.Context, req *pb.ReqGetTradeInstance) (rsp *pb.RspGetTradeInstance, err error) { + subject, err := session.GetSubject(ctx) + if err != nil { + return + } + fmt.Printf("subject: %#v\n", subject) + + inst, err := svr.tradeInstanceService.GetInstance(req.InstId) + if err != nil { + return + } + + rsp = &pb.RspGetTradeInstance{ + Inst: mapping.TradeInstance2Proto(inst), + } + return +} + +func (svr *MarketGrpcServer) AddTradeInstance(ctx context.Context, req *pb.ReqAddTradeInstance) (rsp *pb.RspAddTradeInstance, err error) { + subject, err := session.GetSubject(ctx) + if err != nil { + return + } + + inst := mapping.Proto2TradeInstance(req.Inst) + err = svr.tradeInstanceService.InsertInstance(subject.Uid, inst) + if err != nil { + return + } + + rsp = &pb.RspAddTradeInstance{ + Inst: mapping.TradeInstance2Proto(inst), + } + return +} + +// ListExchangeTradeInstance 获取指定交易所的正常状态的交易产品 +func (svr *MarketGrpcServer) ListExchangeTradeInstance(ctx context.Context, req *pb.ReqListExchangeTradeInstance) (rsp *pb.RspListExchangeTradeInstance, err error) { + rsp = new(pb.RspListExchangeTradeInstance) + var exchanges []int32 + for _, exchange := range req.Exchanges { + exchanges = append(exchanges, int32(exchange)) + } + exchangeInsts, err := svr.tradeInstanceService.ListExchangeTradeInstance(exchanges) + if err != nil { + return + } + for _, inst := range exchangeInsts { + rsp.ExchangeInsts = append(rsp.ExchangeInsts, mapping.ExchangeInstance2Proto(inst)) + } + return +} diff --git a/internal/market/trade_instance_service.go b/internal/market/trade_instance_service.go new file mode 100644 index 0000000..61c13d5 --- /dev/null +++ b/internal/market/trade_instance_service.go @@ -0,0 +1,148 @@ +package market + +import ( + "fmt" + "sig-pub/pkg/data" + "sig-pub/pkg/data/args" + "sig-pub/pkg/data/entity" + "sig-pub/pkg/storage/rdb" + "time" +) + +// TradeInstanceService 交易产品管理 +// TODO cache +type TradeInstanceService struct { + db *rdb.RDB +} + +func NewTradeInstanceService(db *rdb.RDB) *TradeInstanceService { + return &TradeInstanceService{ + db: db, + } +} + +func (s *TradeInstanceService) Init() (err error) { + return +} + +func (s *TradeInstanceService) GetInstance(instId string) (inst *entity.TradeInstance, err error) { + inst = new(entity.TradeInstance) + err = s.db.Select(inst, `select * from t_trade_instance where inst_id = ? and status != ?`, instId, data.StatusDeleted) + if err != nil { + return + } + if inst.InstId == "" { + err = data.ErrorNotExists + return + } + err = s.AttachInstExchanges([]*entity.TradeInstance{inst}) + return +} + +func (s *TradeInstanceService) ListAllInstance() (insts []*entity.TradeInstance, err error) { + err = s.db.Select(&insts, `select * from t_trade_instance where status != ?`, data.StatusDeleted) + if err != nil { + return + } + err = s.AttachInstExchanges(insts) + return +} + +func (s *TradeInstanceService) PageInstance(page, size int, instCoin string) (total int64, insts []*entity.TradeInstance, err error) { + var filter string + var filterArgs []any + if instCoin != "" { + filter = " and inst_coin = ?" + filterArgs = append(filterArgs, instCoin) + } + + var args = []any{data.StatusDeleted} + args = append(args, filterArgs...) + err = s.db.Select(&total, fmt.Sprintf(`select count(*) from t_trade_instance where status != ? %s`, filter), args...) + if err != nil { + return + } + if total == 0 { + return + } + + offset, limit := data.PageCalc(page, size) + args = append(args, offset, limit) + err = s.db.Select(&insts, fmt.Sprintf(`select * from t_trade_instance where status != ? %s limit ?, ?`, filter), args...) + if err != nil { + return + } + err = s.AttachInstExchanges(insts) + if err != nil { + return + } + return +} + +func (s *TradeInstanceService) AttachInstExchanges(insts []*entity.TradeInstance) (err error) { + // 交易产品交易所 + var instIds []string + var instMap = make(map[string]*entity.TradeInstance, len(insts)) + for _, inst := range insts { + instIds = append(instIds, inst.InstId) + instMap[inst.InstId] = inst + } + + var instExchanges []*entity.TradeInstanceExchange + err = s.db.Select(&instExchanges, ` + select * from t_trade_instance_exchange where inst_id in ? and status != ? order by inst_id, exchange + `, instIds, data.StatusDeleted) + if err != nil { + return + } + for _, instEx := range instExchanges { + if inst, ok := instMap[instEx.InstId]; ok { + inst.Exchanges = append(inst.Exchanges, instEx) + } + } + return +} + +func (s *TradeInstanceService) InsertInstance(updateBy string, inst *entity.TradeInstance) (err error) { + if err = validateTradeInstance(inst); err != nil { + return + } + inst.UpdateBy = updateBy + inst.UpdateTime = time.Now().UnixMilli() + err = s.db.Insert(inst) + if err != nil { + return + } + for _, ex := range inst.Exchanges { + ex.UpdateBy = updateBy + ex.UpdateTime = time.Now().UnixMilli() + err = s.db.Insert(ex) + if err != nil { + return + } + } + return +} + +func (s *TradeInstanceService) UpdateInstance(inst *entity.TradeInstance) (err error) { + _, err = s.db.UpdateBy(inst) + return +} + +// 更新交易产品状态 +func (s *TradeInstanceService) UpdateInstanceStatus(inst *args.UpdateTradeInstanceStatusArg) (err error) { + s.db.Update("update t_trade_instance set status = ") + _, err = s.db.UpdateBy(inst) + return +} + +// ListExchangeTradeInstance 获取指定交易所的正常状态的交易产品 +func (s *TradeInstanceService) ListExchangeTradeInstance(exchanges []int32) (exchangesInsts []*entity.TradeInstanceExchange, err error) { + err = s.db.Select(&exchangesInsts, ` + select * from t_trade_instance_exchange where exchange in ? and status in ? order by inst_id, exchange + `, exchanges, []int{data.StatusOk, data.StatusProcessing}) + if err != nil { + return + } + return +} diff --git a/internal/market/validation.go b/internal/market/validation.go new file mode 100644 index 0000000..6952246 --- /dev/null +++ b/internal/market/validation.go @@ -0,0 +1,28 @@ +package market + +import ( + "errors" + "fmt" + "sig-pub/api/pb" + "sig-pub/pkg/data/entity" + "sig-pub/pkg/types" + "sig-pub/pkg/utils/validator" +) + +// 检查交易产品基础信息 +func validateTradeInstance(inst *entity.TradeInstance) (err error) { + if inst == nil { + return errors.New("inst is nil") + } + if validator.IsAnyBlank(inst.InstId, inst.InstPair, inst.InstCoin) { + return errors.New("InstId InstPair InstCoin can't be blank") + } + // todo validate others + for _, ex := range inst.Exchanges { + _, ok := types.ExchangePBParse(pb.Exchange(ex.Exchange)) + if !ok { + return fmt.Errorf("exchange %d not valid", ex.Exchange) + } + } + return +} diff --git a/internal/sig/sig_server.go b/internal/sig/sig_server.go new file mode 100644 index 0000000..bdc5ab7 --- /dev/null +++ b/internal/sig/sig_server.go @@ -0,0 +1,116 @@ +package sig + +import ( + "context" + "encoding/json" + "io" + "net/http" + "runtime/debug" + "sig-pub/pkg/grpc/generic" + "sig-pub/pkg/grpc/session" + "sig-pub/pkg/resp" + "sig-pub/pkg/utils/strs" + "sig-pub/pkg/zlog" + "time" + + "github.com/gin-gonic/gin" + "google.golang.org/protobuf/encoding/protojson" +) + +type SigServer struct { + engine *gin.Engine + grpcGenericClientFactory *generic.GrpcGenericClientFactory +} + +func NewSigServer(grpcGenericClientFactory *generic.GrpcGenericClientFactory) *SigServer { + return &SigServer{ + grpcGenericClientFactory: grpcGenericClientFactory, + } +} + +func (s *SigServer) Init() (err error) { + s.initGinServer() + return +} + +func (s *SigServer) initGinServer() { + s.engine = gin.Default() + s.engine.Use(func(c *gin.Context) { + // global recover + defer func() { + if r := recover(); r != nil { + zlog.Error("http server recover error:", r) + debug.PrintStack() + c.JSON(http.StatusInternalServerError, resp.Error("server error")) + c.Abort() + } + }() + c.Next() + }) + + routerGroup := s.engine.Group("/api") + routerGroup.POST("/v1/:svr/:method", s.handleGrpcGenericCall) + // inst api + // tradeInstanceApi := inst.NewTradeInstanceApi() + // tradeInstanceApi.InitRoute(routerGroup) +} + +func (s *SigServer) Run(addr string) (err error) { + // todo grpc server run + return s.engine.Run(addr) +} + +func (s *SigServer) handleGrpcGenericCall(c *gin.Context) { + svc := strs.UpperInitialLetter(c.Param("svr")) + method := strs.UpperInitialLetter(c.Param("method")) + + if svc == "" || method == "" { + c.JSON(http.StatusBadRequest, resp.Error("service not found")) + return + } + + // todo service white list + + // get request body + jsonBody, err := io.ReadAll(c.Request.Body) + if err != nil { + c.JSON(http.StatusBadRequest, resp.Error("parse request body error: "+err.Error())) + return + } + + ctx1, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + grpcGenericClient, err := s.grpcGenericClientFactory.GetClient(ctx1, svc) + if err != nil { + zlog.Error(err) + c.JSON(http.StatusForbidden, resp.Error(err.Error())) + return + } + + // put session + ctx := context.Background() + ctx = session.PutSubject(ctx, session.NewRpcSubject("123456")) + ctx, cancel = context.WithTimeout(ctx, time.Second*10) + defer cancel() + + // generic call with json + rsp, err := grpcGenericClient.InvokeUnaryJsonBytes(ctx, method, jsonBody) + if err != nil { + if err == generic.ErrorMethodNotExists { + c.JSON(http.StatusNotFound, resp.Error(err.Error())) + return + } + c.JSON(http.StatusInternalServerError, resp.Error(err.Error())) + return + } + + // decode response + bytes, err := protojson.Marshal(rsp) + if err != nil { + c.JSON(http.StatusInternalServerError, resp.Error(err.Error())) + return + } + + r := json.RawMessage(bytes) + c.JSON(http.StatusOK, resp.Success(r)) +} diff --git a/internal/sig/trade_instance/trade_instance_api.go b/internal/sig/trade_instance/trade_instance_api.go new file mode 100644 index 0000000..32e25ec --- /dev/null +++ b/internal/sig/trade_instance/trade_instance_api.go @@ -0,0 +1,63 @@ +package inst + +import ( + "net/http" + "sig-pub/internal/market" + "sig-pub/pkg/data/args" + "sig-pub/pkg/resp" + + "github.com/gin-gonic/gin" +) + +type TradeInstanceApi struct { + service *market.TradeInstanceService // todo grpc call +} + +func NewTradeInstanceApi() *TradeInstanceApi { + return &TradeInstanceApi{} +} + +func (a *TradeInstanceApi) InitRoute(r *gin.RouterGroup) { + instGroup := r.Group("/inst") + { + instGroup.GET("/one/:inst_id", a.getInst) + instGroup.GET("/listAll", a.listAllInst) + instGroup.POST("/list", a.listInst) + } +} + +// getInst 获取指定交易产品 +func (a *TradeInstanceApi) getInst(c *gin.Context) { + instId := c.Param("inst_id") + inst, err := a.service.GetInstance(instId) + if err != nil { + c.JSON(http.StatusOK, resp.Error(err.Error())) + return + } + c.JSON(http.StatusOK, resp.Success(inst)) +} + +// listAllInst 获取所有交易产品 +func (a *TradeInstanceApi) listAllInst(c *gin.Context) { + // list, err := a.service.ListAllInstance() + // if err != nil { + // c.JSON(http.StatusOK, resp.Error(err.Error())) + // return + // } + // c.JSON(http.StatusOK, resp.Success(list)) +} + +// listInst 交易产品分页查询 +func (a *TradeInstanceApi) listInst(c *gin.Context) { + pageArg := args.ParsePageArgs(c) + + total, list, err := a.service.PageInstance(pageArg.Page, pageArg.Size, "") + if err != nil { + c.JSON(http.StatusOK, resp.Error(err.Error())) + return + } + c.JSON(http.StatusOK, resp.Success(resp.H{ + "total": total, + "data": list, + })) +} diff --git a/internal/trading/okx/trading.go b/internal/trading/okx/trading.go new file mode 100644 index 0000000..d3e2348 --- /dev/null +++ b/internal/trading/okx/trading.go @@ -0,0 +1,3 @@ +package okxtrade + +// okx 订单交易接口 diff --git a/internal/trading/service.go b/internal/trading/service.go new file mode 100644 index 0000000..ade22b8 --- /dev/null +++ b/internal/trading/service.go @@ -0,0 +1,7 @@ +package trading + +// 交易服务 +// load strategy plugins +// order 订单服务... + +// 订单交易，个人账户查询 diff --git a/pkg/aside/trade_instance_client.go b/pkg/aside/trade_instance_client.go new file mode 100644 index 0000000..70ace86 --- /dev/null +++ b/pkg/aside/trade_instance_client.go @@ -0,0 +1,76 @@ +package aside + +import ( + "context" + "sig-pub/api/pb" + "sig-pub/pkg/data/entity" + "sig-pub/pkg/mapping" + "sig-pub/pkg/utils/kvcache" + "time" + + cache "github.com/fanjindong/go-cache" + "golang.org/x/sync/singleflight" +) + +// TradeInstanceAside 交易实例客户端带缓存 +// todo nats 更新监听 更新缓存 +// grpc trade instance client +type TradeInstanceAside struct { + client pb.MarketClient + cache *kvcache.KVCache[*entity.TradeInstance] + cacheSf singleflight.Group +} + +func NewTradeInstanceAside(client pb.MarketClient) *TradeInstanceAside { + return &TradeInstanceAside{ + client: client, + cache: kvcache.NewExpireStore[*entity.TradeInstance]( + time.Minute, + cache.WithShards(16), + cache.WithClearInterval(5*time.Minute), + ), + } +} + +// GetTradeInstance 获取交易实例 +// 高频访问，使用缓存 + singleflight +func (c *TradeInstanceAside) GetTradeInstance(ctx context.Context, instId string) (inst *entity.TradeInstance, err error) { + r, err, _ := c.cacheSf.Do(instId, func() (r any, err error) { + return c.getTradeInstance0(ctx, instId) + }) + if err != nil { + return + } + inst = r.(*entity.TradeInstance) + return +} + +func (c *TradeInstanceAside) getTradeInstance0(ctx context.Context, instId string) (inst *entity.TradeInstance, err error) { + inst, ok := c.cache.Get(instId) + if ok { + return + } + // grpc 获取 + rsp, err := c.client.GetTradeInstance(ctx, &pb.ReqGetTradeInstance{InstId: instId}) + if err != nil { + return + } + inst = mapping.Proto2TradeInstance(rsp.Inst) + c.cache.Set(instId, inst) + return +} + +// ListExchangeTradeInstance 获取交易所支持的交易实例 +func (c *TradeInstanceAside) ListExchangeTradeInstance(ctx context.Context, exchange pb.Exchange) (exInsts []*entity.TradeInstanceExchange, err error) { + rsp, err := c.client.ListExchangeTradeInstance(ctx, &pb.ReqListExchangeTradeInstance{ + Exchanges: []pb.Exchange{exchange}, + }) + if err != nil { + return + } + for _, pbExInst := range rsp.ExchangeInsts { + exInst := mapping.Proto2ExchangeTradeInstance(pbExInst) + exInsts = append(exInsts, exInst) + } + return +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..20e6c4d --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,144 @@ +package config + +import ( + "sig-pub/pkg/zlog" + + "github.com/redis/go-redis/v9" + clientv3 "go.etcd.io/etcd/client/v3" + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/logger" +) + +type Configuration struct { + // App map[string]any + Exchange ExchangeConf + + Grpc GrpcConfig + Etcd clientv3.Config + Database Database + Tsdb TsdbConfig +} + +type GrpcConfig struct { + Log bool + Address string + MaxSendMsgSize string + MaxRecvMsgSize string + ReadBufferSize string + WriteBufferSize string + keepalive GrpcKeepalive +} + +type GrpcKeepalive struct { + IdleTimeout string + ForceCloseWait string + KeepAliveInterval string + KeepAliveTimeout string + MaxLifeTime string +} + +// ============== Gate ============== +type GateConf struct { + HttpAddr string + WsAddr string +} + +// ============== database ============== +type Database struct { + Mysql MysqlConfig + Postgres PostgresConfig + Clickhouse ClickhouseConfig + Kvrocks redis.Options +} + +type MysqlConfig struct { + LogMode string // silent error warn info + gorm.Config + Mysql mysql.Config +} + +func (conf MysqlConfig) NewGormDB() (db *gorm.DB, err error) { + logLevel := logger.Silent + switch conf.LogMode { + default: + zlog.Errorf("unknow mysql gorm logmode: %s", conf.LogMode) + case "": + case "silent": + logLevel = logger.Silent + case "error": + logLevel = logger.Error + case "warn": + logLevel = logger.Warn + case "info": + logLevel = logger.Info + } + conf.Logger = logger.Default.LogMode(logLevel) + + db, err = gorm.Open(mysql.New(conf.Mysql), &conf) + return +} + +type PostgresConfig struct { + LogMode string // silent error warn info + gorm.Config + Postgres postgres.Config +} + +func (conf PostgresConfig) NewGormDB() (db *gorm.DB, err error) { + logLevel := logger.Silent + switch conf.LogMode { + default: + zlog.Errorf("unknow mysql gorm logmode: %s", conf.LogMode) + case "": + case "silent": + logLevel = logger.Silent + case "error": + logLevel = logger.Error + case "warn": + logLevel = logger.Warn + case "info": + logLevel = logger.Info + } + conf.Logger = logger.Default.LogMode(logLevel) + db, err = gorm.Open(postgres.New(conf.Postgres), &conf) + return +} + +type ClickhouseConfig struct { + Logsql bool + Addr string + Database string + Username string + Password string + DialTimeout int // second + ReadTimeout int // second + NotAutoMigrateTable bool + DsnParams map[string]any +} + +// ============== tsdb ============== +type TsdbConfig struct { + Active string + Victoriametrics VictoriaMetricsConfig +} + +type VictoriaMetricsConfig struct { + Addr string +} + +// ============== exchange ============== +type ExchangeConf struct { + Okx OkxExchange +} +type OkxExchange struct { + ApiKey string + SecretKey string + Passphrase string + ReceiveBuffer int + MarketSubscribeLimit int + ConsumeBatch int // 订阅 channel 单次处理消息数量 + ConsumeLater int // 毫秒,时间到达later或者数据累计到batch触发consume + HttpProxy string +} diff --git a/pkg/config/grpc_options.go b/pkg/config/grpc_options.go new file mode 100644 index 0000000..b95c090 --- /dev/null +++ b/pkg/config/grpc_options.go @@ -0,0 +1,45 @@ +package config + +import ( + "sig-pub/pkg/utils/conver" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +func GetGrpcOptions(c GrpcConfig, customOpts ...grpc.ServerOption) (opts []grpc.ServerOption) { + if c.MaxSendMsgSize != "" { + opts = append(opts, grpc.MaxSendMsgSize(conver.MustParseDataUnitInt(c.MaxSendMsgSize))) + } + if c.MaxRecvMsgSize != "" { + opts = append(opts, grpc.MaxSendMsgSize(conver.MustParseDataUnitInt(c.MaxRecvMsgSize))) + } + if c.ReadBufferSize != "" { + opts = append(opts, grpc.MaxSendMsgSize(conver.MustParseDataUnitInt(c.ReadBufferSize))) + } + if c.WriteBufferSize != "" { + opts = append(opts, grpc.MaxSendMsgSize(conver.MustParseDataUnitInt(c.WriteBufferSize))) + } + + // grpc server keepalive + keep := keepalive.ServerParameters{} + if c.keepalive.IdleTimeout != "" { + keep.MaxConnectionIdle = conver.MustParseDuration(c.keepalive.IdleTimeout) + } + if c.keepalive.ForceCloseWait != "" { + keep.MaxConnectionAgeGrace = conver.MustParseDuration(c.keepalive.ForceCloseWait) + } + if c.keepalive.KeepAliveInterval != "" { + keep.Time = conver.MustParseDuration(c.keepalive.KeepAliveInterval) + } + if c.keepalive.KeepAliveTimeout != "" { + keep.Timeout = conver.MustParseDuration(c.keepalive.KeepAliveTimeout) + } + if c.keepalive.MaxLifeTime != "" { + keep.MaxConnectionAge = conver.MustParseDuration(c.keepalive.MaxLifeTime) + } + opts = append(opts, grpc.KeepaliveParams(keep)) + + opts = append(opts, customOpts...) + return +} diff --git a/pkg/config/loader.go b/pkg/config/loader.go new file mode 100644 index 0000000..0e561dd --- /dev/null +++ b/pkg/config/loader.go @@ -0,0 +1,73 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sig-pub/pkg/zlog" + "strings" + + "github.com/spf13/viper" +) + +func parseConfPath(confPath string) (filePath, fileName, confType string) { + filePath, fileName = filepath.Split(confPath) + ext := filepath.Ext(fileName) + fileName = strings.Replace(fileName, ext, "", -1) + confType = strings.Replace(ext, ".", "", 1) + return +} + +func MustLoadConfig[T any](conf T, confPathArg ...string) T { + err := LoadConfig(conf, confPathArg...) + if err != nil { + panic(err) + } + return conf +} + +// LoadConfig load config.toml +// appConf service custom config +// return common service configuration +func LoadConfig[T any](conf T, confPathArg ...string) error { + confPath := "" + confName := "config" + confType := "toml" + envPrefix := "SO" + + if len(confPathArg) > 0 { + confPath = confPathArg[0] + } + + confPath, confName, confType = parseConfPath(confPath) + if confPath == "" { + confPath = "./" + } + + filePath := fmt.Sprintf("%s%s.%s", confPath, confName, confType) + dir, err := os.Getwd() + if err != nil { + zlog.Warning(err) + } else { + filePath = filepath.Join(dir, filePath) + } + zlog.Infof("use config file: %s, env prefix=%s\n", filePath, envPrefix) + + v := viper.New() + v.AddConfigPath(confPath) + v.SetConfigName(confName) + v.SetConfigType(confType) + v.SetEnvPrefix(envPrefix) + v.AutomaticEnv() + v.AllowEmptyEnv(true) + + if err := v.ReadInConfig(); err != nil { + return errors.New("viper read config fail: " + err.Error()) + } + if err := v.Unmarshal(conf); err != nil { + return errors.New("viper unmarshal config failed: " + err.Error()) + } + + return nil +} diff --git a/pkg/data/args/pager.go b/pkg/data/args/pager.go new file mode 100644 index 0000000..4b1863e --- /dev/null +++ b/pkg/data/args/pager.go @@ -0,0 +1,32 @@ +package args + +import ( + "strconv" + + "github.com/gin-gonic/gin" +) + +type PageArg struct { + Page int + Size int + SortBy string + Asc bool +} + +func ParsePageArgs(c *gin.Context) (arg PageArg) { + page := c.Param("page") + size := c.Param("size") + sortBy := c.Param("sortBy") + asc, _ := strconv.Atoi(c.Param("asc")) + arg.Page, _ = strconv.Atoi(page) + arg.Size, _ = strconv.Atoi(size) + arg.SortBy = sortBy + arg.Asc = asc == 1 + if arg.Page <= 0 { + arg.Page = 1 + } + if arg.Size <= 0 || arg.Size > 1000 { + arg.Size = 20 + } + return +} diff --git a/pkg/data/args/trade_instance.go b/pkg/data/args/trade_instance.go new file mode 100644 index 0000000..7ede832 --- /dev/null +++ b/pkg/data/args/trade_instance.go @@ -0,0 +1,12 @@ +package args + +type UpdateTradeInstanceStatusArg struct { + InstId string `json:"InstId"` // 交易产品id + Status int32 `json:"Status"` // 0禁用,1正常,4删除 +} + +type UpdateTradeInstanceExchangeStatusArg struct { + ExchangeInstId string `json:"ExchangeInstId"` // 交易所交易产品id + InstId string `json:"InstId"` // 交易产品id + Status int32 `json:"Status"` // 0禁用,1正常,4删除 +} diff --git a/pkg/data/common.go b/pkg/data/common.go new file mode 100644 index 0000000..4286c4c --- /dev/null +++ b/pkg/data/common.go @@ -0,0 +1,31 @@ +package data + +import "errors" + +// 状态枚举 +type Status int32 + +const ( + StatusDisabled = 0 + StatusOk = 1 + StatusProcessing = 2 + StatusDeleted = 4 +) + +var ( + ErrorNotExists = errors.New("not exists") +) + +func PageCalc(page, size int) (offset, limit int) { + if page <= 0 { + page = 1 + } + switch { + case size > 1000: + size = 1000 + case size <= 0: + size = 20 + } + offset = (page - 1) * size + return offset, size +} diff --git a/pkg/data/entity/trade_instance.go b/pkg/data/entity/trade_instance.go new file mode 100644 index 0000000..4e8e330 --- /dev/null +++ b/pkg/data/entity/trade_instance.go @@ -0,0 +1,40 @@ +package entity + +import "github.com/lib/pq" + +// 交易产品基础信息 +type TradeInstance struct { + InstId string `gorm:"column:inst_id;primaryKey" json:"InstId"` // 交易产品id BTC_USDT_SWAP + InstPair string `gorm:"column:inst_pair" json:"instPair"` // 交易对 BTCUSDT + InstType int32 `gorm:"column:inst_type" json:"instType"` // 交易类型: 1现货2永续合约 + InstCoin string `gorm:"column:inst_coin" json:"instCoin"` // 所属币种 BTC + Status int32 `gorm:"column:status" json:"status"` // 状态: 0禁用,1正常,4删除 + PriceSz int32 `gorm:"column:price_sz" json:"priceSz"` // 价格小数点位数 + QuantitySz int32 `gorm:"column:quantity_sz" json:"quantitySz"` // 数量小数点位数 + Icon string `gorm:"column:icon" json:"icon"` // 币种图标 + Leverages pq.Int32Array `gorm:"column:leverages;type:int[]" json:"leverages"` // 杠杆倍数[5,10,20,50,100] + UpdateBy string `gorm:"column:update_by" json:"updateBy"` // 更新人 + UpdateTime int64 `gorm:"column:update_time" json:"updateTime"` // 更新时间戳毫秒 + Exchanges []*TradeInstanceExchange `gorm:"-" json:"exchanges"` // 交易产品支持交易所 + // 合约面值(0.0001/BTC) 杠杆倍数[5,10,20,50,100] 前端显示(BTC/USDT) 是否可交易 开空 开多 市价开空 市价开多 交易对(btcusdt) + // 资金费率 资金周期 最小下单量 最大下单量 最小下单金额 最大下单金额 + // 开仓手续费 平仓手续费 +} + +func (TradeInstance) TableName() string { + return "t_trade_instance" +} + +// 交易所交易产品 +type TradeInstanceExchange struct { + ExchangeInstId string `gorm:"column:exchange_inst_id;primaryKey" json:"exchangeInstId"` // 交易所交易产品id(交易所交易对) + Exchange int32 `gorm:"column:exchange;primaryKey" json:"exchange"` // 交易所: 1.okx 2.binance + InstId string `gorm:"column:inst_id" json:"instId"` // 交易产品id + Status int32 `gorm:"column:status" json:"status"` // 状态: 0禁用,1正常,2初始化中,4删除 + UpdateBy string `gorm:"column:update_by" json:"updateBy"` // 更新人 + UpdateTime int64 `gorm:"column:update_time" json:"updateTime"` // 更新时间戳毫秒 +} + +func (TradeInstanceExchange) TableName() string { + return "t_trade_instance_exchange" +} diff --git a/pkg/grpc/client/direct_client_factory.go b/pkg/grpc/client/direct_client_factory.go new file mode 100755 index 0000000..6f6b2ce --- /dev/null +++ b/pkg/grpc/client/direct_client_factory.go @@ -0,0 +1,41 @@ +package client + +import ( + "context" + "google.golang.org/grpc" + "sync" +) + +type GrpcDirectClientFactory struct { + defaultOpts []grpc.DialOption + clientCache *sync.Map +} + +func NewGrpcDirectClientFactory(defaultOpts ...grpc.DialOption) *GrpcDirectClientFactory { + return &GrpcDirectClientFactory{ + defaultOpts: defaultOpts, + clientCache: &sync.Map{}, + } +} + +func (f *GrpcDirectClientFactory) NewConn(ctx context.Context, addr string, opts ...grpc.DialOption) (*grpc.ClientConn, error) { + dialOpts := make([]grpc.DialOption, 0, len(f.defaultOpts)+len(opts)) + dialOpts = append(dialOpts, f.defaultOpts...) + dialOpts = append(dialOpts, opts...) + + return grpc.DialContext(ctx, addr, dialOpts...) +} + +func (f *GrpcDirectClientFactory) GetConn(ctx context.Context, addr string, opts ...grpc.DialOption) (conn *grpc.ClientConn, err error) { + val, ok := f.clientCache.Load(addr) + if ok { + conn = val.(*grpc.ClientConn) + return + } + conn, err = f.NewConn(ctx, addr, opts...) + if err != nil { + return + } + f.clientCache.Store(addr, conn) + return +} diff --git a/pkg/grpc/discovery/discovery.go b/pkg/grpc/discovery/discovery.go new file mode 100755 index 0000000..01eb099 --- /dev/null +++ b/pkg/grpc/discovery/discovery.go @@ -0,0 +1,59 @@ +package discovery + +import ( + "context" + "sig-pub/pkg/zlog" + "strconv" + + "google.golang.org/grpc/resolver" +) + +type Registry interface { + // Registry server instance + Registry(ctx context.Context, server Server) (err error) +} + +type GrpcResolver interface { + DialUrl(serviceName string) string + // Resolver get grpc dial resolver + Resolver() (builder resolver.Builder, err error) +} + +type Resolver interface { + // ResolveAll get service all instance + ResolveAll(ctx context.Context, serviceName string) (servers []Server, err error) + // Watch when service instance change, send current all instances to channel + Watch(ctx context.Context, serviceName string) (ch chan []Server, err error) +} + +type Discovery interface { + Registry + Resolver + GrpcResolver +} + +// Server registry format +type Server struct { + Name string `json:"name"` + Addr string `json:"addr"` // 地址 + Attrs map[string]string `json:"attrs"` // attributes +} + +func (s Server) GetWeight() (weight int) { + weight = 1 + if len(s.Attrs) == 0 { + return + } + + v, ok := s.Attrs["weight"] + if !ok { + return + } + w, err := strconv.Atoi(v) + if err != nil { + zlog.Warning("failed parse discovery server attr weight: ", v) + return + } + weight = w + return +} diff --git a/pkg/grpc/discovery/etcd/instance.go b/pkg/grpc/discovery/etcd/instance.go new file mode 100755 index 0000000..3aa3b56 --- /dev/null +++ b/pkg/grpc/discovery/etcd/instance.go @@ -0,0 +1,72 @@ +package etcd + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "google.golang.org/grpc/resolver" +) + +// Server registry format +type Server struct { + Name string `json:"name"` + Addr string `json:"addr"` // 地址 + Attrs map[string]string `json:"attrs"` // attributes +} + +func SplitPath(path string) (Server, error) { + server := Server{} + strs := strings.Split(path, "/") + if len(strs) == 0 { + return server, errors.New("invalid path") + } + + server.Addr = strs[len(strs)-1] + + return server, nil +} + +// Exist helper function +func Exist(l []resolver.Address, addr resolver.Address) bool { + for i := range l { + if l[i].Addr == addr.Addr { + return true + } + } + + return false +} + +// Remove helper function +func Remove(s []resolver.Address, addr resolver.Address) ([]resolver.Address, bool) { + for i := range s { + if s[i].Addr == addr.Addr { + s[i] = s[len(s)-1] + return s[:len(s)-1], true + } + } + return nil, false +} + +func BuildResolverUrl(app string) string { + return "etcd:///" + app +} + +func BuildPrefix(server Server) string { + return fmt.Sprintf("/%s/", server.Name) +} + +func BuildRegisterPath(server Server) string { + return fmt.Sprintf("%s%s", BuildPrefix(server), server.Addr) +} + +func ParseValue(value []byte) (Server, error) { + server := Server{} + if err := json.Unmarshal(value, &server); err != nil { + return server, err + } + + return server, nil +} diff --git a/pkg/grpc/discovery/etcd/register.go b/pkg/grpc/discovery/etcd/register.go new file mode 100755 index 0000000..8f5c801 --- /dev/null +++ b/pkg/grpc/discovery/etcd/register.go @@ -0,0 +1,181 @@ +package etcd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "sig-pub/pkg/utils/nets" + "strings" + "time" + + "google.golang.org/grpc/grpclog" + + clientv3 "go.etcd.io/etcd/client/v3" +) + +var DefaultRegisterTTL int64 = 30 + +func RegisterIpPort(addr string) (ip string, port int, err error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + return + } + port = tcpAddr.Port + if tcpAddr.IP != nil { + ip = tcpAddr.IP.String() + } else { + ip, err = nets.GetHostIpv4() + if err != nil { + return + } + } + return +} +func RegisterAddress(addr string) (fullAddr string, err error) { + ip, port, err := RegisterIpPort(addr) + fullAddr = fmt.Sprintf("%s:%d", ip, port) + return +} + +func MustRegisterAddress(addr string) (fullAddr string) { + var err error + fullAddr, err = RegisterAddress(addr) + if err != nil { + panic(err) + } + return +} + +// Register +// Deprecated +type Register struct { + DialTimeout int + + closeCh chan struct{} + leasesID clientv3.LeaseID + keepAliveCh <-chan *clientv3.LeaseKeepAliveResponse + + srvInfo Server + srvTTL int64 + cli *clientv3.Client +} + +// NewRegister create a register based on etcd +func NewRegister(client *clientv3.Client) *Register { + return &Register{ + cli: client, + DialTimeout: 3, + } +} + +// Register a user +func (r *Register) Register(srvInfo Server) (err error) { + if strings.Split(srvInfo.Addr, ":")[0] == "" { + return errors.New("invalid ip address") + } + + r.srvInfo = srvInfo + r.srvTTL = DefaultRegisterTTL + + if err = r.register(); err != nil { + return err + } + + if r.closeCh == nil { + r.closeCh = make(chan struct{}) + } + + go r.keepAlive() + + return nil +} + +func (r *Register) register() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(r.DialTimeout)*time.Second) + defer cancel() + + leaseResp, err := r.cli.Grant(ctx, r.srvTTL) + if err != nil { + return err + } + + r.leasesID = leaseResp.ID + + if r.keepAliveCh, err = r.cli.KeepAlive(context.Background(), r.leasesID); err != nil { + return err + } + + data, err := json.Marshal(r.srvInfo) + if err != nil { + return err + } + + _, err = r.cli.Put(context.Background(), BuildRegisterPath(r.srvInfo), string(data), clientv3.WithLease(r.leasesID)) + + return err +} + +// Stop stop register +func (r *Register) Stop() { + if r.closeCh == nil { + return + } + r.closeCh <- struct{}{} + <-r.closeCh // 阻塞到关闭 + close(r.closeCh) +} + +// unregister 删除节点 +func (r *Register) unregister() error { + _, err := r.cli.Delete(context.Background(), BuildRegisterPath(r.srvInfo)) + return err +} + +func (r *Register) keepAlive() { + ticker := time.NewTicker(time.Duration(r.srvTTL) * time.Second) + defer ticker.Stop() + + for { + select { + case <-r.closeCh: + if err := r.unregister(); err != nil { + grpclog.Error("unregister failed, error: ", err) + } + if _, err := r.cli.Revoke(context.Background(), r.leasesID); err != nil { + grpclog.Error("revoke failed, error: ", err) + } + r.closeCh <- struct{}{} + return + case res := <-r.keepAliveCh: + if res == nil { + if err := r.register(); err != nil { + grpclog.Error("register failed, error: ", err) + } + } + case <-ticker.C: + if r.keepAliveCh == nil { + if err := r.register(); err != nil { + grpclog.Error("register failed, error: ", err) + } + } + } + } +} + +func (r *Register) GetServerInfo() (Server, error) { + resp, err := r.cli.Get(context.Background(), BuildRegisterPath(r.srvInfo)) + if err != nil { + return r.srvInfo, err + } + + server := Server{} + if resp.Count >= 1 { + if err := json.Unmarshal(resp.Kvs[0].Value, &server); err != nil { + return server, err + } + } + + return server, err +} diff --git a/pkg/grpc/discovery/etcd/resolver.go b/pkg/grpc/discovery/etcd/resolver.go new file mode 100755 index 0000000..92bc237 --- /dev/null +++ b/pkg/grpc/discovery/etcd/resolver.go @@ -0,0 +1,171 @@ +package etcd + +import ( + "context" + "sig-pub/pkg/zlog" + "time" + + clientv3 "go.etcd.io/etcd/client/v3" + "google.golang.org/grpc/resolver" +) + +const ( + schema = "etcd" +) + +// Resolver for grpc client +// Deprecated +type Resolver struct { + schema string + DialTimeout int + + closeCh chan struct{} + watchCh clientv3.WatchChan + cli *clientv3.Client + keyPrefix string + srvAddrsList []resolver.Address + + cc resolver.ClientConn +} + +// NewResolver create a new resolver.Builder base on etcd +func NewResolver(client *clientv3.Client) *Resolver { + return &Resolver{ + cli: client, + schema: schema, + DialTimeout: 3, + } +} + +// Scheme returns the scheme supported by this resolver. +func (r *Resolver) Scheme() string { + return r.schema +} + +// Build creates a new resolver.Resolver for the given target +func (r *Resolver) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (rr resolver.Resolver, err error) { + r.cc = cc + r.keyPrefix = BuildPrefix(Server{Name: target.Endpoint()}) + if err = r.start(); err != nil { + return nil, err + } + return r, nil +} + +// ResolveNow resolver.Resolver interface +func (r *Resolver) ResolveNow(o resolver.ResolveNowOptions) {} + +// Close resolver.Resolver interface +func (r *Resolver) Close() { + if r.closeCh == nil { + return + } + r.closeCh <- struct{}{} + <-r.closeCh +} + +// start +func (r *Resolver) start() error { + var err error + + resolver.Register(r) + + if r.closeCh == nil { + r.closeCh = make(chan struct{}) + } + + if err = r.sync(); err != nil { + return err + } + + go r.watch() + + return nil +} + +// watch update events +func (r *Resolver) watch() { + ticker := time.NewTicker(time.Minute) + r.watchCh = r.cli.Watch(context.Background(), r.keyPrefix, clientv3.WithPrefix()) + + for { + select { + case <-r.closeCh: + r.closeCh <- struct{}{} + return + case res, ok := <-r.watchCh: + if ok { + r.update(res.Events) + } + case <-ticker.C: + if err := r.sync(); err != nil { + zlog.Error("resolver sync failed: ", err) + } + } + } +} + +// update +func (r *Resolver) update(events []*clientv3.Event) { + for _, ev := range events { + var info Server + var err error + + switch ev.Type { + case clientv3.EventTypePut: + info, err = ParseValue(ev.Kv.Value) + if err != nil { + continue + } + addr := resolver.Address{Addr: info.Addr} + for k, v := range info.Attrs { + addr.Attributes = addr.Attributes.WithValue(k, v) + } + if !Exist(r.srvAddrsList, addr) { + r.srvAddrsList = append(r.srvAddrsList, addr) + err = r.cc.UpdateState(resolver.State{Addresses: r.srvAddrsList}) + if err != nil { + zlog.Error("resolver conn update put state err: ", err) + } + } + case clientv3.EventTypeDelete: + info, err = SplitPath(string(ev.Kv.Key)) + if err != nil { + continue + } + addr := resolver.Address{Addr: info.Addr} + if s, ok := Remove(r.srvAddrsList, addr); ok { + r.srvAddrsList = s + err = r.cc.UpdateState(resolver.State{Addresses: r.srvAddrsList}) + if err != nil { + zlog.Error("resolver conn update delete state err: ", err) + } + } + } + } +} + +// sync 同步获取所有地址信息 +func (r *Resolver) sync() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + res, err := r.cli.Get(ctx, r.keyPrefix, clientv3.WithPrefix()) + if err != nil { + return + } + r.srvAddrsList = []resolver.Address{} + + for _, v := range res.Kvs { + info, err := ParseValue(v.Value) + if err != nil { + continue + } + addr := resolver.Address{Addr: info.Addr} + for k, v := range info.Attrs { + addr.Attributes = addr.Attributes.WithValue(k, v) + } + r.srvAddrsList = append(r.srvAddrsList, addr) + } + err = r.cc.UpdateState(resolver.State{Addresses: r.srvAddrsList}) + return +} diff --git a/pkg/grpc/discovery/etcd_naming.go b/pkg/grpc/discovery/etcd_naming.go new file mode 100755 index 0000000..23a86d8 --- /dev/null +++ b/pkg/grpc/discovery/etcd_naming.go @@ -0,0 +1,239 @@ +package discovery + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sig-pub/pkg/utils/nets" + "sig-pub/pkg/zlog" + "time" + + "github.com/bytedance/sonic" + "go.etcd.io/etcd/api/v3/mvccpb" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/naming/endpoints" + etcdResolver "go.etcd.io/etcd/client/v3/naming/resolver" + "google.golang.org/grpc/resolver" +) + +var ( + DefaultRegisterTTL int64 = 30 +) + +const ( + EtcdSchema = "etcd" +) + +func EtcdDialUrl(serviceName string) string { + return fmt.Sprintf("%s:///%s", EtcdSchema, serviceName) +} + +type EtcdDiscovery struct { + client *clientv3.Client +} + +func NewEtcdDiscovery(client *clientv3.Client) *EtcdDiscovery { + return &EtcdDiscovery{ + client: client, + } +} + +func (r *EtcdDiscovery) Registry(ctx context.Context, server Server) (err error) { + em, err := endpoints.NewManager(r.client, server.Name) + if err != nil { + return + } + ip, port, err := RegisterIpPort(server.Addr) + if err != nil { + return + } + + addr := fmt.Sprintf("%s:%d", ip, port) + // 序列化 metadata 信息 + meta := "{}" + if server.Attrs != nil { + bytes, e := json.Marshal(server.Attrs) + if e != nil { + err = e + return + } + meta = string(bytes) + } + + ctxLease, cancelLease := context.WithTimeout(ctx, time.Second*5) + defer cancelLease() + lease, err := r.client.Grant(ctxLease, DefaultRegisterTTL) // 使用租约注册端点确保如果主机无法维持保活心跳, 从服务中删除 + if err != nil { + return + } + endpointKey := fmt.Sprintf("%s/%s", server.Name, addr) + err = em.AddEndpoint(ctx, + endpointKey, + endpoints.Endpoint{ + Addr: addr, + Metadata: meta, + }, + clientv3.WithLease(lease.ID), + ) + + // keepalive lease + keepCtx, keepCancel := context.WithCancel(context.Background()) + keepAliveCh, err := r.client.KeepAlive(keepCtx, lease.ID) + if err != nil { + keepCancel() + zlog.Error("registry keepalive error: ", err) + return + } + + // ticker := time.NewTicker(time.Second * time.Duration(DefaultRegisterTTL)) + w := r.client.Watch(ctx, endpointKey) + go func() { + for { + select { + case <-ctx.Done(): + zlog.Info("registry keepalive done") + ctx, c := context.WithTimeout(context.Background(), time.Second*3) + _, _ = r.client.Revoke(ctx, lease.ID) + c() + keepCancel() + //ticker.Stop() + return + case <-keepAliveCh: + //zlog.Infof("keepalive: lease id %d, %+v", lease.ID, res) + case res := <-w: + if err := res.Err(); err != nil { + zlog.Errorf("registry watch endpoint %s error: %v", endpointKey, err) + continue + } + deleted := false + for _, event := range res.Events { + if event.Type == clientv3.EventTypeDelete { + deleted = true + } + } + // endpoint key被删除, 重放 + if deleted { + zlog.Infof("registry endpoint %s deleted ", endpointKey) + // 删除旧的 lease + keepCancel() + _, _ = r.client.Revoke(context.Background(), lease.ID) + lease, err = r.client.Grant(ctx, DefaultRegisterTTL) + if err != nil { + zlog.Error("registry grant lease again error: ", err) + } + err = em.AddEndpoint(ctx, + endpointKey, + endpoints.Endpoint{ + Addr: addr, + Metadata: meta, + }, + clientv3.WithLease(lease.ID), + ) + if err != nil { + zlog.Error("refresh endpoint error: ", err) + continue + } + // keepalive lease + keepCtx, keepCancel = context.WithCancel(context.Background()) + keepAliveCh, err = r.client.KeepAlive(keepCtx, lease.ID) + if err != nil { + zlog.Error("registry keepalive error: ", err) + } + } + //case <-ticker.C: // 定时重放防止etcd中key被删除 + } + } + }() + return +} + +func (r *EtcdDiscovery) DialUrl(serviceName string) string { + return fmt.Sprintf("%s:///%s", EtcdSchema, serviceName) +} + +func (r *EtcdDiscovery) Resolver() (builder resolver.Builder, err error) { + builder, err = etcdResolver.NewBuilder(r.client) + return +} + +func (r *EtcdDiscovery) ResolveAll(ctx context.Context, serviceName string) (servers []Server, err error) { + res, err := r.client.Get(ctx, serviceName+"/", clientv3.WithPrefix()) + if err != nil { + return + } + + for _, kv := range res.Kvs { + endpoint := endpoints.Endpoint{} + if err = sonic.Unmarshal(kv.Value, &endpoint); err != nil { + zlog.Errorf("resolve service %s error: ", string(kv.Value)) + return + } + server := Server{ + Name: serviceName, + Addr: endpoint.Addr, + } + if endpoint.Metadata != nil { + if strMeta, ok := endpoint.Metadata.(string); ok { + err = json.Unmarshal([]byte(strMeta), &server.Attrs) + if err != nil { + return + } + } + } + + servers = append(servers, server) + } + return +} + +func (r *EtcdDiscovery) Watch(ctx context.Context, serviceName string) (ch chan []Server, err error) { + w := r.client.Watch(ctx, serviceName+"/", clientv3.WithPrefix()) + ch = make(chan []Server, 1) + go func() { + for { + select { + case <-ctx.Done(): + close(ch) + return + case res := <-w: + if err := res.Err(); err != nil { + zlog.Errorf("watch service %s error: %v", serviceName, err) + continue + } + + for _, event := range res.Events { + switch event.Type { + case mvccpb.DELETE: + fallthrough + case mvccpb.PUT: + servers, err := r.ResolveAll(context.Background(), serviceName) + if err != nil { + zlog.Errorf("watch event %v for service %s error: %v", event.Type, serviceName, err) + continue + } + ch <- servers + } + } + } + } + }() + return +} + +func RegisterIpPort(addr string) (ip string, port int, err error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", addr) + if err != nil { + return + } + port = tcpAddr.Port + if tcpAddr.IP != nil { + ip = tcpAddr.IP.String() + } else { + ip, err = nets.GetHostIpv4() + if err != nil { + return + } + } + return +} diff --git a/pkg/grpc/discovery/etcd_naming_test.go b/pkg/grpc/discovery/etcd_naming_test.go new file mode 100755 index 0000000..be9b3aa --- /dev/null +++ b/pkg/grpc/discovery/etcd_naming_test.go @@ -0,0 +1,43 @@ +package discovery + +import ( + "context" + clientv3 "go.etcd.io/etcd/client/v3" + "testing" + "time" +) + +func TestResolveAll(t *testing.T) { + client, err := clientv3.New(clientv3.Config{ + Endpoints: []string{"127.0.0.1:2379"}, + }) + if err != nil { + t.Error(err) + } + dis := NewEtcdDiscovery(client) + + serviceName := "TestService" + // registry + s1 := Server{ + Addr: "127.0.0.1:1234", + Name: serviceName, + Attrs: map[string]string{"weight": "10"}, + } + ctx, cancel := context.WithCancel(context.Background()) + err = dis.Registry(ctx, s1) + if err != nil { + t.Error(err) + } + + // resolve + servers, err := dis.ResolveAll(context.Background(), serviceName) + if err != nil { + t.Error(err) + } + if len(servers) == 0 || servers[0].Addr != s1.Addr { + t.Error("resolveAll server addr error") + } + + cancel() + time.Sleep(time.Second) +} diff --git a/pkg/grpc/generic/errors.go b/pkg/grpc/generic/errors.go new file mode 100644 index 0000000..a0a379d --- /dev/null +++ b/pkg/grpc/generic/errors.go @@ -0,0 +1,7 @@ +package generic + +import "errors" + +var ( + ErrorMethodNotExists = errors.New("method not exists") +) diff --git a/pkg/grpc/generic/generic_client.go b/pkg/grpc/generic/generic_client.go new file mode 100644 index 0000000..9252488 --- /dev/null +++ b/pkg/grpc/generic/generic_client.go @@ -0,0 +1,100 @@ +package generic + +import ( + "context" + "sig-pub/pkg/zlog" + "time" + + "github.com/bytedance/sonic" + "github.com/jhump/protoreflect/v2/grpcdynamic" + "github.com/jhump/protoreflect/v2/grpcreflect" + "google.golang.org/grpc" + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/types/dynamicpb" + + refv1 "google.golang.org/grpc/reflection/grpc_reflection_v1" +) + +type GrpcGenericClient struct { + serviceName string + conn *grpc.ClientConn + + serviceDesc protoreflect.ServiceDescriptor + stub *grpcdynamic.Stub +} + +func NewGpcGenericClient(serviceName string, conn *grpc.ClientConn) *GrpcGenericClient { + return &GrpcGenericClient{ + serviceName: serviceName, + conn: conn, + } +} + +func (c *GrpcGenericClient) Init(ctx context.Context) (err error) { + client := grpcreflect.NewClientV1(ctx, refv1.NewServerReflectionClient(c.conn)) + marketServiceSymbol, err := client.FileContainingSymbol(protoreflect.FullName(c.serviceName)) + if err != nil { + return + } + c.serviceDesc = marketServiceSymbol.Services().ByName(protoreflect.Name(c.serviceName)) + c.stub = grpcdynamic.NewStub(c.conn) + return +} + +func (c *GrpcGenericClient) ServiceName() string { + return c.serviceName +} + +func (c *GrpcGenericClient) InvokeUnary(ctx context.Context, method string, reqBytes []byte, opts ...grpc.CallOption) (resp proto.Message, err error) { + caller, err := c.getMethodCaller(method) + if err != nil { + return + } + + request := dynamicpb.NewMessage(caller.Input()) + if err = proto.Unmarshal(reqBytes, request); err != nil { + return + } + ms := time.Now().UnixMilli() + resp, err = c.stub.InvokeRpc(ctx, caller, request, opts...) + // resp -> *dynamicpb.Message + if delay := time.Now().UnixMilli() - ms; delay > 2000 { + zlog.Warningf("api %s.%s process use %dms\n", c.serviceName, method, delay) + } + return +} + +func (c *GrpcGenericClient) InvokeUnaryJson(ctx context.Context, method string, jsonBody any, opts ...grpc.CallOption) (resp proto.Message, err error) { + jsonBytes, err := sonic.Marshal(jsonBody) + if err != nil { + return + } + resp, err = c.InvokeUnaryJsonBytes(ctx, method, jsonBytes, opts...) + return +} + +func (c *GrpcGenericClient) InvokeUnaryJsonBytes(ctx context.Context, method string, jsonBytes []byte, opts ...grpc.CallOption) (resp proto.Message, err error) { + caller, err := c.getMethodCaller(method) + if err != nil { + return + } + request := dynamicpb.NewMessage(caller.Input()) + if err = protojson.Unmarshal(jsonBytes, request); err != nil { + return + } + + resp, err = c.stub.InvokeRpc(ctx, caller, request, opts...) + return +} + +// getMethodCaller load generic resource from cache +func (c *GrpcGenericClient) getMethodCaller(method string) (caller protoreflect.MethodDescriptor, err error) { + caller = c.serviceDesc.Methods().ByName(protoreflect.Name(method)) + if caller == nil { + err = ErrorMethodNotExists + return + } + return +} diff --git a/pkg/grpc/generic/generic_client_factory.go b/pkg/grpc/generic/generic_client_factory.go new file mode 100644 index 0000000..30bc093 --- /dev/null +++ b/pkg/grpc/generic/generic_client_factory.go @@ -0,0 +1,49 @@ +package generic + +import ( + "context" + "fmt" + "sig-pub/pkg/utils/collect" + + "google.golang.org/grpc" +) + +type GrpcGenericClientFactory struct { + scheme string + defaultOpts []grpc.DialOption + clientCache *collect.ConcurrentMap[string, *GrpcGenericClient] +} + +func NewGpcGenericClientFactory(scheme string, defaultOpts ...grpc.DialOption) *GrpcGenericClientFactory { + return &GrpcGenericClientFactory{ + scheme: scheme, + defaultOpts: defaultOpts, + } +} + +func (f *GrpcGenericClientFactory) Init() (err error) { + f.clientCache = collect.NewConcurrentMap[string, *GrpcGenericClient](8, func(serviceName string) string { return serviceName }) + return +} + +func (f *GrpcGenericClientFactory) NewClient(ctx context.Context, serviceName string, opts ...grpc.DialOption) (client *GrpcGenericClient, err error) { + addr := fmt.Sprintf("%s:///%s", f.scheme, serviceName) + dialOpts := make([]grpc.DialOption, 0, len(f.defaultOpts)+len(opts)) + dialOpts = append(dialOpts, f.defaultOpts...) + dialOpts = append(dialOpts, opts...) + conn, err := grpc.NewClient(addr, dialOpts...) + if err != nil { + return + } + client = NewGpcGenericClient(serviceName, conn) + err = client.Init(ctx) + return +} + +func (f *GrpcGenericClientFactory) GetClient(ctx context.Context, serviceName string, opts ...grpc.DialOption) (client *GrpcGenericClient, err error) { + // todo 优化 + client, err, _ = f.clientCache.ComputeIfAbsentE(serviceName, func(serviceName string) (*GrpcGenericClient, error) { + return f.NewClient(ctx, serviceName, opts...) + }) + return +} diff --git a/pkg/grpc/interceptor/recover_interceptor.go b/pkg/grpc/interceptor/recover_interceptor.go new file mode 100755 index 0000000..2381834 --- /dev/null +++ b/pkg/grpc/interceptor/recover_interceptor.go @@ -0,0 +1,39 @@ +package interceptor + +import ( + "context" + "errors" + "runtime/debug" + "sig-pub/pkg/zlog" + + "google.golang.org/grpc" + "google.golang.org/grpc/grpclog" + "google.golang.org/protobuf/types/known/emptypb" +) + +func RecoverInterceptor(ctx context.Context, req any, server *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { + defer func() { + if r := recover(); r != nil { + switch r.(type) { + case error: + err = r.(error) + case string: + err = errors.New(r.(string)) + default: + err = errors.New("grpc server recover error") + } + grpclog.Error("grpc server error: ", r) + debug.PrintStack() + } + }() + + resp, err = handler(ctx, req) + if err == nil { + if empty, ok := resp.(*emptypb.Empty); ok && empty == nil { + resp = &emptypb.Empty{} // grpc: error while marshaling: proto: Marshal called with nil + } else if resp == nil { + zlog.Warningf("grpc request: has no response: method=%s, %v", server.FullMethod, req) + } + } + return +} diff --git a/pkg/grpc/session/grpc_subject.go b/pkg/grpc/session/grpc_subject.go new file mode 100644 index 0000000..4c83c97 --- /dev/null +++ b/pkg/grpc/session/grpc_subject.go @@ -0,0 +1,50 @@ +package session + +import ( + "context" + "errors" + + "google.golang.org/grpc/metadata" +) + +const ( + UidKey = "uid" +) + +var ( + UnauthorizedRequestError = errors.New("unauthorized request") +) + +type RpcSubject struct { + Uid string +} + +func NewRpcSubject(uid string) *RpcSubject { + return &RpcSubject{ + Uid: uid, + } +} + +func PutSubject(ctx context.Context, subject *RpcSubject) context.Context { + return metadata.NewOutgoingContext(ctx, metadata.Pairs(UidKey, subject.Uid)) +} + +func GetSubject(ctx context.Context) (*RpcSubject, error) { + uid, ok := GetUid(ctx) + if !ok { + return nil, UnauthorizedRequestError + } + return &RpcSubject{Uid: uid}, nil +} + +func GetUid(ctx context.Context) (string, bool) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "", false + } + uids := md.Get(UidKey) + if len(uids) == 0 { + return "", false + } + return uids[0], true +} diff --git a/pkg/influx/influxdb.go b/pkg/influx/influxdb.go new file mode 100644 index 0000000..6488471 --- /dev/null +++ b/pkg/influx/influxdb.go @@ -0,0 +1,18 @@ +package influx + +import ( + influxdb2 "github.com/influxdata/influxdb-client-go/v2" +) + +func NewClient() { + url := "" + token := "" + client := influxdb2.NewClient(url, token) + _ = client +} + +type KlineStorage interface { + Save() + Find() + FindInRange() +} diff --git a/pkg/mapping/market.go b/pkg/mapping/market.go new file mode 100644 index 0000000..366be1c --- /dev/null +++ b/pkg/mapping/market.go @@ -0,0 +1,74 @@ +package mapping + +import ( + "sig-pub/api/pb" + "sig-pub/pkg/data/entity" +) + +func TradeInstance2Proto(inst *entity.TradeInstance) (pbInst *pb.TradeInstance) { + pbInst = new(pb.TradeInstance) + if inst == nil { + return + } + pbInst.InstId = inst.InstId + pbInst.InstPair = inst.InstPair + pbInst.InstCoin = inst.InstCoin + pbInst.InstType = pb.TradeInstanceType(inst.InstType) + pbInst.Status = inst.Status + pbInst.PriceSz = inst.PriceSz + pbInst.QuantitySz = inst.QuantitySz + pbInst.Icon = inst.Icon + pbInst.UpdateBy = inst.UpdateBy + pbInst.UpdateTime = inst.UpdateTime + pbInst.Leverages = inst.Leverages + for _, ex := range inst.Exchanges { + pbInst.Exchanges = append(pbInst.Exchanges, ExchangeInstance2Proto(ex)) + } + return +} + +func ExchangeInstance2Proto(exInst *entity.TradeInstanceExchange) (pbExInst *pb.TradeInstanceExchange) { + pbExInst = &pb.TradeInstanceExchange{ + ExchangeInstId: exInst.ExchangeInstId, + InstId: exInst.InstId, + Status: exInst.Status, + Exchange: exInst.Exchange, + UpdateBy: exInst.UpdateBy, + UpdateTime: exInst.UpdateTime, + } + return +} + +func Proto2TradeInstance(pbInst *pb.TradeInstance) (inst *entity.TradeInstance) { + inst = new(entity.TradeInstance) + if pbInst == nil { + return + } + inst.InstId = pbInst.InstId + inst.InstPair = pbInst.InstPair + inst.InstCoin = pbInst.InstCoin + inst.InstType = int32(pbInst.InstType) + inst.Status = pbInst.Status + inst.PriceSz = pbInst.PriceSz + inst.QuantitySz = pbInst.QuantitySz + inst.Icon = pbInst.Icon + inst.UpdateBy = pbInst.UpdateBy + inst.UpdateTime = pbInst.UpdateTime + inst.Leverages = pbInst.Leverages + for _, ex := range pbInst.Exchanges { + inst.Exchanges = append(inst.Exchanges, Proto2ExchangeTradeInstance(ex)) + } + return +} + +func Proto2ExchangeTradeInstance(pbExInst *pb.TradeInstanceExchange) (exInst *entity.TradeInstanceExchange) { + exInst = &entity.TradeInstanceExchange{ + ExchangeInstId: pbExInst.ExchangeInstId, + InstId: pbExInst.InstId, + Status: pbExInst.Status, + Exchange: pbExInst.Exchange, + UpdateBy: pbExInst.UpdateBy, + UpdateTime: pbExInst.UpdateTime, + } + return +} diff --git a/pkg/resp/resp.go b/pkg/resp/resp.go new file mode 100644 index 0000000..4b31b23 --- /dev/null +++ b/pkg/resp/resp.go @@ -0,0 +1,62 @@ +package resp + +import ( + "sig-pub/pkg/zlog" + + "github.com/bytedance/sonic" +) + +const ( + CodeOK = 200 + CodeFail = 400 + CodeError = 500 +) + +type H map[string]any + +// Response 响应体包装 +type Response struct { + Seq int `json:"seq,omitempty"` + Code int `json:"code,omitempty"` + Msg string `json:"msg,omitempty"` + Data any `json:"data,omitempty"` + Extra map[string]any `json:"extra,omitempty"` +} + +func (resp *Response) Json() []byte { + json, err := sonic.Marshal(resp) + if err != nil { + zlog.Error("unknown json error: ", err) + return nil + } + return json +} + +func (resp *Response) JsonString() string { + return string(resp.Json()) +} + +func SeqResp(seq int, code int, msg string, data any) *Response { + return &Response{ + Seq: seq, + Code: code, + Msg: msg, + Data: data, + } +} + +func Resp(code int, msg string, data any) *Response { + return SeqResp(0, code, msg, data) +} + +func Success(data any) *Response { + return Resp(CodeOK, "", data) +} + +func Fail(msg string) *Response { + return Resp(CodeFail, msg, nil) +} + +func Error(msg string) *Response { + return Resp(CodeError, msg, nil) +} diff --git a/pkg/router/router.go b/pkg/router/router.go new file mode 100644 index 0000000..4f04538 --- /dev/null +++ b/pkg/router/router.go @@ -0,0 +1,117 @@ +package router + +// Copyright 2013 Julien Schmidt. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. +// https://github.com/julienschmidt/httprouter + +import ( + "context" +) + +// Handle is a function that can be registered to a route to handle HTTP +// requests. Like http.HandlerFunc, but has a third parameter for the values of +// wildcards (variables). +type Handle func(ns, path string, ps Params) + +// Param is a single URL parameter, consisting of a key and a value. +type Param struct { + Key string + Value string +} + +// Params is a Param-slice, as returned by the router. +// The slice is ordered, the first URL parameter is also the first slice value. +// It is therefore safe to read values by the index. +type Params []Param + +// ByName returns the value of the first Param which key matches the given name. +// If no matching Param is found, an empty string is returned. +func (ps Params) ByName(name string) string { + for i := range ps { + if ps[i].Key == name { + return ps[i].Value + } + } + return "" +} + +type paramsKey struct{} + +// ParamsKey is the request context key under which URL params are stored. +var ParamsKey = paramsKey{} + +// ParamsFromContext pulls the URL parameters from a request context, +// or returns nil if none are present. +func ParamsFromContext(ctx context.Context) Params { + p, _ := ctx.Value(ParamsKey).(Params) + return p +} + +// Router is a http.Handler which can be used to dispatch requests to different +// handler functions via configurable routes +type Router struct { + trees map[string]*node + + // Function to handle panics recovered from handlers. + PanicHandler func(namespace, path string, rev any) +} + +// New returns a new initialized Router. +// Path auto-correction, including trailing slashes, is enabled by default. +func New() *Router { + return &Router{ + // trees: make(map[string]*node), + } +} + +// Route registers a new request handle with the given path and method. +func (r *Router) Route(namespace, path string, handle Handle) { + if len(path) < 1 || path[0] != '/' { + panic("path must begin with '/' in path '" + path + "'") + } + + if r.trees == nil { + r.trees = make(map[string]*node) + } + + root := r.trees[namespace] + if root == nil { + root = new(node) + r.trees[namespace] = root + } + + root.addRoute(path, handle) +} + +// Lookup allows the manual lookup of a namespace + path combo. +func (r *Router) Lookup(namespace, path string) (Handle, Params) { + if r.trees != nil { + if root := r.trees[namespace]; root != nil { + handle, params, _ := root.getValue(path) + return handle, params + } + } + return nil, nil +} + +func (r *Router) recv(ns, path string) { + if rcv := recover(); rcv != nil { + r.PanicHandler(ns, path, rcv) + } +} + +// Handle router and execute handle. +func (r *Router) Handle(ns, path string) (found bool) { + if r.PanicHandler != nil { + defer r.recv(ns, path) + } + + if handle, ps := r.Lookup(ns, path); handle != nil { + found = true + handle(ns, path, ps) + return + } + + return false +} diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go new file mode 100644 index 0000000..4a5d8e2 --- /dev/null +++ b/pkg/router/router_test.go @@ -0,0 +1,37 @@ +package router + +import ( + "fmt" + "testing" +) + +func TestRouter(t *testing.T) { + ns1 := "default" + nsKline := "kline" + router := New() + router.Route(ns1, "/hello/:name", func(ns, path string, ps Params) { + _ = ps.ByName("name") + // fmt.Printf("handle hello: name=%s, ns=%s, path=%s\n", ps.ByName("name"), ns, path) + }) + router.Route(nsKline, "/kline/:bar", func(ns, path string, ps Params) { + fmt.Printf("handle kline: bar=%s, ns=%s, path=%s\n", ps.ByName("bar"), ns, path) + }) + + found := router.Handle(ns1, "/hello/jack") + if !found { + t.Error("not found...") + } + h, p := router.Lookup(nsKline, "/kline/5m") + if h != nil { + h(nsKline, "/kline/5m", p) + } + + // start := time.Now().UnixMilli() + // for i := range 10000000 { + // found := router.Handle(ns1, fmt.Sprintf("/hello/sunny%d", i)) + // if !found { + // t.Error("not found...") + // } + // } + // fmt.Printf("use %dms\n", time.Now().UnixMilli()-start) // use 2375ms +} diff --git a/pkg/router/tree.go b/pkg/router/tree.go new file mode 100644 index 0000000..7947f07 --- /dev/null +++ b/pkg/router/tree.go @@ -0,0 +1,458 @@ +package router + +// Copyright 2013 Julien Schmidt. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be found +// in the LICENSE file. + +import ( + "strings" +) + +func min(a, b int) int { + if a <= b { + return a + } + return b +} + +const maxParamCount uint8 = ^uint8(0) + +func countParams(path string) uint8 { + var n uint + for i := 0; i < len(path); i++ { + if path[i] != ':' && path[i] != '*' { + continue + } + n++ + } + if n >= uint(maxParamCount) { + return maxParamCount + } + + return uint8(n) +} + +type nodeType uint8 + +const ( + static nodeType = iota // default + root + param + catchAll +) + +type node struct { + path string + wildChild bool + nType nodeType + maxParams uint8 + priority uint32 + indices string + children []*node + handle Handle +} + +// increments priority of the given child and reorders if necessary +func (n *node) incrementChildPrio(pos int) int { + n.children[pos].priority++ + prio := n.children[pos].priority + + // adjust position (move to front) + newPos := pos + for newPos > 0 && n.children[newPos-1].priority < prio { + // swap node positions + n.children[newPos-1], n.children[newPos] = n.children[newPos], n.children[newPos-1] + + newPos-- + } + + // build new index char string + if newPos != pos { + n.indices = n.indices[:newPos] + // unchanged prefix, might be empty + n.indices[pos:pos+1] + // the index char we move + n.indices[newPos:pos] + n.indices[pos+1:] // rest without char at 'pos' + } + + return newPos +} + +// addRoute adds a node with the given handle to the path. +// Not concurrency-safe! +func (n *node) addRoute(path string, handle Handle) { + fullPath := path + n.priority++ + numParams := countParams(path) + + // non-empty tree + if len(n.path) > 0 || len(n.children) > 0 { + walk: + for { + // Update maxParams of the current node + if numParams > n.maxParams { + n.maxParams = numParams + } + + // Find the longest common prefix. + // This also implies that the common prefix contains no ':' or '*' + // since the existing key can't contain those chars. + i := 0 + max := min(len(path), len(n.path)) + for i < max && path[i] == n.path[i] { + i++ + } + + // Split edge + if i < len(n.path) { + child := node{ + path: n.path[i:], + wildChild: n.wildChild, + nType: static, + indices: n.indices, + children: n.children, + handle: n.handle, + priority: n.priority - 1, + } + + // Update maxParams (max of all children) + for i := range child.children { + if child.children[i].maxParams > child.maxParams { + child.maxParams = child.children[i].maxParams + } + } + + n.children = []*node{&child} + // []byte for proper unicode char conversion, see #65 + n.indices = string([]byte{n.path[i]}) + n.path = path[:i] + n.handle = nil + n.wildChild = false + } + + // Make new node a child of this node + if i < len(path) { + path = path[i:] + + if n.wildChild { + n = n.children[0] + n.priority++ + + // Update maxParams of the child node + if numParams > n.maxParams { + n.maxParams = numParams + } + numParams-- + + // Check if the wildcard matches + if len(path) >= len(n.path) && n.path == path[:len(n.path)] && + // Adding a child to a catchAll is not possible + n.nType != catchAll && + // Check for longer wildcard, e.g. :name and :names + (len(n.path) >= len(path) || path[len(n.path)] == '/') { + continue walk + } else { + // Wildcard conflict + var pathSeg string + if n.nType == catchAll { + pathSeg = path + } else { + pathSeg = strings.SplitN(path, "/", 2)[0] + } + prefix := fullPath[:strings.Index(fullPath, pathSeg)] + n.path + panic("'" + pathSeg + + "' in new path '" + fullPath + + "' conflicts with existing wildcard '" + n.path + + "' in existing prefix '" + prefix + + "'") + } + } + + c := path[0] + + // slash after param + if n.nType == param && c == '/' && len(n.children) == 1 { + n = n.children[0] + n.priority++ + continue walk + } + + // Check if a child with the next path byte exists + for i := 0; i < len(n.indices); i++ { + if c == n.indices[i] { + i = n.incrementChildPrio(i) + n = n.children[i] + continue walk + } + } + + // Otherwise insert it + if c != ':' && c != '*' { + // []byte for proper unicode char conversion, see #65 + n.indices += string([]byte{c}) + child := &node{ + maxParams: numParams, + } + n.children = append(n.children, child) + n.incrementChildPrio(len(n.indices) - 1) + n = child + } + n.insertChild(numParams, path, fullPath, handle) + return + + } else if i == len(path) { // Make node a (in-path) leaf + if n.handle != nil { + panic("a handle is already registered for path '" + fullPath + "'") + } + n.handle = handle + } + return + } + } else { // Empty tree + n.insertChild(numParams, path, fullPath, handle) + n.nType = root + } +} + +func (n *node) insertChild(numParams uint8, path, fullPath string, handle Handle) { + var offset int // already handled bytes of the path + + // find prefix until first wildcard (beginning with ':'' or '*'') + for i, max := 0, len(path); numParams > 0; i++ { + c := path[i] + if c != ':' && c != '*' { + continue + } + + // find wildcard end (either '/' or path end) + end := i + 1 + for end < max && path[end] != '/' { + switch path[end] { + // the wildcard name must not contain ':' and '*' + case ':', '*': + panic("only one wildcard per path segment is allowed, has: '" + + path[i:] + "' in path '" + fullPath + "'") + default: + end++ + } + } + + // check if this Node existing children which would be + // unreachable if we insert the wildcard here + if len(n.children) > 0 { + panic("wildcard route '" + path[i:end] + + "' conflicts with existing children in path '" + fullPath + "'") + } + + // check if the wildcard has a name + if end-i < 2 { + panic("wildcards must be named with a non-empty name in path '" + fullPath + "'") + } + + if c == ':' { // param + // split path at the beginning of the wildcard + if i > 0 { + n.path = path[offset:i] + offset = i + } + + child := &node{ + nType: param, + maxParams: numParams, + } + n.children = []*node{child} + n.wildChild = true + n = child + n.priority++ + numParams-- + + // if the path doesn't end with the wildcard, then there + // will be another non-wildcard subpath starting with '/' + if end < max { + n.path = path[offset:end] + offset = end + + child := &node{ + maxParams: numParams, + priority: 1, + } + n.children = []*node{child} + n = child + } + + } else { // catchAll + if end != max || numParams > 1 { + panic("catch-all routes are only allowed at the end of the path in path '" + fullPath + "'") + } + + if len(n.path) > 0 && n.path[len(n.path)-1] == '/' { + panic("catch-all conflicts with existing handle for the path segment root in path '" + fullPath + "'") + } + + // currently fixed width 1 for '/' + i-- + if path[i] != '/' { + panic("no / before catch-all in path '" + fullPath + "'") + } + + n.path = path[offset:i] + + // first node: catchAll node with empty path + child := &node{ + wildChild: true, + nType: catchAll, + maxParams: 1, + } + // update maxParams of the parent node + if n.maxParams < 1 { + n.maxParams = 1 + } + n.children = []*node{child} + n.indices = string(path[i]) + n = child + n.priority++ + + // second node: node holding the variable + child = &node{ + path: path[i:], + nType: catchAll, + maxParams: 1, + handle: handle, + priority: 1, + } + n.children = []*node{child} + + return + } + } + + // insert remaining path part and handle to the leaf + n.path = path[offset:] + n.handle = handle +} + +// Returns the handle registered with the given path (key). The values of +// wildcards are saved to a map. +// If no handle can be found, a TSR (trailing slash redirect) recommendation is +// made if a handle exists with an extra (without the) trailing slash for the +// given path. +func (n *node) getValue(path string) (handle Handle, p Params, tsr bool) { +walk: // outer loop for walking the tree + for { + if len(path) > len(n.path) { + if path[:len(n.path)] == n.path { + path = path[len(n.path):] + // If this node does not have a wildcard (param or catchAll) + // child, we can just look up the next child node and continue + // to walk down the tree + if !n.wildChild { + c := path[0] + for i := 0; i < len(n.indices); i++ { + if c == n.indices[i] { + n = n.children[i] + continue walk + } + } + + // Nothing found. + // We can recommend to redirect to the same URL without a + // trailing slash if a leaf exists for that path. + tsr = (path == "/" && n.handle != nil) + return + + } + + // handle wildcard child + n = n.children[0] + switch n.nType { + case param: + // find param end (either '/' or path end) + end := 0 + for end < len(path) && path[end] != '/' { + end++ + } + + // save param value + if p == nil { + // lazy allocation + p = make(Params, 0, n.maxParams) + } + i := len(p) + p = p[:i+1] // expand slice within preallocated capacity + p[i].Key = n.path[1:] + p[i].Value = path[:end] + + // we need to go deeper! + if end < len(path) { + if len(n.children) > 0 { + path = path[end:] + n = n.children[0] + continue walk + } + + // ... but we can't + tsr = (len(path) == end+1) + return + } + + if handle = n.handle; handle != nil { + return + } else if len(n.children) == 1 { + // No handle found. Check if a handle for this path + a + // trailing slash exists for TSR recommendation + n = n.children[0] + tsr = (n.path == "/" && n.handle != nil) + } + + return + + case catchAll: + // save param value + if p == nil { + // lazy allocation + p = make(Params, 0, n.maxParams) + } + i := len(p) + p = p[:i+1] // expand slice within preallocated capacity + p[i].Key = n.path[2:] + p[i].Value = path + + handle = n.handle + return + + default: + panic("invalid node type") + } + } + } else if path == n.path { + // We should have reached the node containing the handle. + // Check if this node has a handle registered. + if handle = n.handle; handle != nil { + return + } + + if path == "/" && n.wildChild && n.nType != root { + tsr = true + return + } + + // No handle found. Check if a handle for this path + a + // trailing slash exists for trailing slash recommendation + for i := 0; i < len(n.indices); i++ { + if n.indices[i] == '/' { + n = n.children[i] + tsr = (len(n.path) == 1 && n.handle != nil) || + (n.nType == catchAll && n.children[0].handle != nil) + return + } + } + + return + } + + // Nothing found. We can recommend to redirect to the same URL with an + // extra trailing slash if a leaf exists for that path + tsr = (path == "/") || + (len(n.path) == len(path)+1 && n.path[len(path)] == '/' && + path == n.path[:len(n.path)-1] && n.handle != nil) + return + } +} diff --git a/pkg/storage/rdb/rdb.go b/pkg/storage/rdb/rdb.go new file mode 100644 index 0000000..49b89f7 --- /dev/null +++ b/pkg/storage/rdb/rdb.go @@ -0,0 +1,43 @@ +package rdb + +import ( + "gorm.io/gorm" +) + +// RDB gorm关系型数据 +type RDB struct { + db *gorm.DB +} + +func NewRDB(db *gorm.DB) *RDB { + return &RDB{ + db: db, + } +} + +func (m *RDB) Init() (err error) { + return +} + +func (m *RDB) Insert(data any) (err error) { + return m.db.Create(data).Error +} + +// sql查询数据 +func (m *RDB) Select(ret any, sql string, args ...any) (err error) { + return m.db.Raw(sql, args...).Scan(ret).Error +} + +// sql更新数据 +func (m *RDB) Update(sql string, args ...any) (rowsAffected int64, err error) { + tx := m.db.Exec(sql, args...) + rowsAffected, err = tx.RowsAffected, tx.Error + return +} + +// sql更新数据 +func (m *RDB) UpdateBy(data any) (rowsAffected int64, err error) { + tx := m.db.Model(data).Updates(data) + rowsAffected, err = tx.RowsAffected, tx.Error + return +} diff --git a/pkg/storage/tsdb/tsdb.go b/pkg/storage/tsdb/tsdb.go new file mode 100644 index 0000000..2c877a7 --- /dev/null +++ b/pkg/storage/tsdb/tsdb.go @@ -0,0 +1,16 @@ +package tsdb + +import ( + "context" + "sig-pub/pkg/types" +) + +type TSDB interface { + Init(ctx context.Context) + SaveKlines(kline []types.Kline) + GetRangeKline() +} + +func Register(name string, newer func() TSDB) { + +} diff --git a/pkg/storage/tsdb/victoria_metrics/metric.go b/pkg/storage/tsdb/victoria_metrics/metric.go new file mode 100644 index 0000000..c9e3220 --- /dev/null +++ b/pkg/storage/tsdb/victoria_metrics/metric.go @@ -0,0 +1,86 @@ +package vmts + +import ( + "fmt" + "sig-pub/pkg/types" + + "github.com/bytedance/sonic" +) + +var ( + rawValueLimit = 5 // 避免单行数据过大 +) + +type Metric struct { + Metric map[string]string `json:"metric"` // {"__name__":"open","instance":"DOGE-USDT-SWAP"} + Values []float64 `json:"values"` + Timestamps []int64 `json:"timestamps"` +} + +func NewMetric(name string, tags ...string) *Metric { + if len(tags)%2 != 0 { + panic(fmt.Sprintf("%s error length tags: %v", name, tags)) + } + metric := map[string]string{"__name__": name} + for i := 0; i < len(tags); i += 2 { + metric[tags[i]] = tags[i+1] + } + return &Metric{ + Metric: metric, + } +} + +func (m *Metric) AddTag(name, value string) { + m.Metric[name] = value +} + +func (m *Metric) AddTsValue(ts int64, value float64) { + m.Timestamps = append(m.Timestamps, ts) + m.Values = append(m.Values, value) +} + +func (m *Metric) ToRowJson() ([]byte, error) { + return sonic.Marshal(m) +} + +func Kline2Metrics(inst types.TradeInstance, klines []*types.Kline) (metrics []*Metric) { + instMetrics := make(map[string][6]*Metric) + // id := inst.InstId + // inst.InstId = "doge_udst" + // defer func() { + // inst.InstId = id + // }() + + for _, kline := range klines { + ms, ok := instMetrics[inst.InstId] + if !ok || (rawValueLimit > 0 && len(ms[0].Values) >= rawValueLimit) { + ms = [6]*Metric{ + NewMetric(inst.InstId, "interval", string(kline.Interval), "kind", "open"), + NewMetric(inst.InstId, "interval", string(kline.Interval), "kind", "high"), + NewMetric(inst.InstId, "interval", string(kline.Interval), "kind", "low"), + NewMetric(inst.InstId, "interval", string(kline.Interval), "kind", "close"), + NewMetric(inst.InstId, "interval", string(kline.Interval), "kind", "vol"), + NewMetric(inst.InstId, "interval", string(kline.Interval), "kind", "volQuote"), + } + instMetrics[inst.InstId] = ms + for i := range len(ms) { + metrics = append(metrics, ms[i]) + } + } + + // TODO open sz price + open, _ := kline.Open.Float64() + high, _ := kline.High.Float64() + low, _ := kline.Low.Float64() + close, _ := kline.Close.Float64() + vol, _ := kline.Vol.Float64() + volQuote, _ := kline.VolQuote.Float64() + ms[0].AddTsValue(kline.Ts, open) + ms[1].AddTsValue(kline.Ts, high) + ms[2].AddTsValue(kline.Ts, low) + ms[3].AddTsValue(kline.Ts, close) + ms[4].AddTsValue(kline.Ts, vol) + ms[5].AddTsValue(kline.Ts, volQuote) + } + return +} diff --git a/pkg/storage/tsdb/victoria_metrics/metric_test.go b/pkg/storage/tsdb/victoria_metrics/metric_test.go new file mode 100644 index 0000000..1a4e4e8 --- /dev/null +++ b/pkg/storage/tsdb/victoria_metrics/metric_test.go @@ -0,0 +1,22 @@ +package vmts + +import ( + "strings" + "testing" +) + +func TestMetric(t *testing.T) { + m := NewMetric("doge-usdt-swap-open") + m.AddTag("interval", "5m") + m.AddTsValue(1746720962969, 0.242) + m.AddTsValue(1746721051250, 0.252) + bytes, err := m.ToRowJson() + if err != nil { + t.Error(err) + } + raw := string(bytes) + if strings.Contains(raw, "\n") { + t.Error("raw data contains \\n symble") + } + t.Log(raw) +} diff --git a/pkg/storage/tsdb/victoria_metrics/vm.go b/pkg/storage/tsdb/victoria_metrics/vm.go new file mode 100644 index 0000000..a2cb539 --- /dev/null +++ b/pkg/storage/tsdb/victoria_metrics/vm.go @@ -0,0 +1,120 @@ +package vmts + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "sig-pub/pkg/config" + "sig-pub/pkg/types" + "sig-pub/pkg/zlog" + + "github.com/klauspost/compress/zstd" +) + +// VictoriaMetricsTSDB 时序库 +type VictoriaMetricsTSDB struct { + addr string +} + +func NewVictoriaMetricsTSDB(conf config.VictoriaMetricsConfig) *VictoriaMetricsTSDB { + return &VictoriaMetricsTSDB{ + addr: conf.Addr, + } +} + +func (vm *VictoriaMetricsTSDB) SaveKlines(inst types.TradeInstance, klines []*types.Kline) (err error) { + metrics := Kline2Metrics(inst, klines) + err = vm.batchWriteMetrics(metrics) + return +} + +func (vm *VictoriaMetricsTSDB) batchWriteMetrics(metrics []*Metric) (err error) { + var buf bytes.Buffer + // gz := gzip.NewWriter(&buf) + var data []byte + for _, metric := range metrics { + data, err = metric.ToRowJson() + if err != nil { + return + } + buf.Write(data) + buf.Write([]byte("\n")) + // gz.Write(data) + // gz.Write([]byte("\n")) + } + // gz.Close() + + // file, _ := os.Open(fmt.Sprintf("%d.json", time.Now().Unix())) + // defer file.Close() + // err = os.WriteFile(fmt.Sprintf("./%d.json", time.Now().Unix()), buf.Bytes(), os.ModeAppend) + // if err != nil { + // return + // } + + // datas, err := compressData(buf.Bytes()) + // {kind="high"}[30m] + + resp, err := http.Post(fmt.Sprintf("%s/api/v1/import", vm.addr), "application/json", &buf) + if err != nil { + return + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + zlog.Info("batch import status error: ", resp.Status) + } + return +} + +func compressData(data []byte) ([]byte, error) { + var b bytes.Buffer + encoder, _ := zstd.NewWriter(&b) + defer encoder.Close() + + if _, err := encoder.Write(data); err != nil { + return nil, err + } + return b.Bytes(), nil +} + +// 获取原始k线列表 +func (vm *VictoriaMetricsTSDB) GetRangeKline(inst types.TradeInstance, interval types.Interval, start, end int64) (err error) { + if inst.InstId == "" { + err = errors.New("instid is empty") + return + } + if start <= 0 || end <= 0 { + err = errors.New("invalid time range") + return + } + + match := fmt.Sprintf("%s{interval=\"%s\"}", inst.InstId, interval) + params := fmt.Sprintf("start=%d&end=%d&match[]=%s", start, end, url.QueryEscape(match)) + resp, err := http.Get(fmt.Sprintf("%s/api/v1/export?%s", vm.addr, params)) + if err != nil { + return + } + defer func() { + if err := resp.Body.Close(); err != nil { + zlog.Error("close vmtsdb response error:", err) + } + }() + + // read response json line + reader := bufio.NewReader(resp.Body) + for { + line, err2 := reader.ReadBytes('\n') + if err2 == io.EOF { + break + } + if err2 != nil { + zlog.Error(err) + err = err2 + return + } + zlog.Infof("response body line: %s", string(line)) + } + return +} diff --git a/pkg/storage/tsdb/victoria_metrics/vm_test.go b/pkg/storage/tsdb/victoria_metrics/vm_test.go new file mode 100644 index 0000000..5d5adca --- /dev/null +++ b/pkg/storage/tsdb/victoria_metrics/vm_test.go @@ -0,0 +1,20 @@ +package vmts + +import ( + "sig-pub/pkg/config" + "sig-pub/pkg/types" + "testing" + "time" +) + +var test_addr = "http://127.0.0.1:8428" + +func TestGetRangeKline(t *testing.T) { + vmdb := NewVictoriaMetricsTSDB(config.VictoriaMetricsConfig{Addr: test_addr}) + inst := types.TradeInstance{ + InstId: "BTC_USDT", + } + + vmdb.GetRangeKline(inst, types.Interval1m, 1753368047691, time.Now().UnixMilli()) + +} diff --git a/pkg/stream/stream.go b/pkg/stream/stream.go new file mode 100644 index 0000000..93535ac --- /dev/null +++ b/pkg/stream/stream.go @@ -0,0 +1,11 @@ +package stream + +const ( + StreamKlineLive = "/stream/kline/live/:bar" // 实时k线推送, 1s,5s,1m... stream.kline.live.* + StreamKlineLiveClose = "/stream/kline/live/close" // 实时K线关闭时推送 + StreamKlineLiveClose1s = "/stream/kline/live/close/1s" // 实时K线关闭时推送 stream.kline.live.close.{bar} +) + +// todo AntPathMatcher +// consumer -> sub /k/1s, /k/5s -> handler /k/1s +// producer -> /k/1s -> get subs -> broadcast diff --git a/pkg/types/decimal.go b/pkg/types/decimal.go new file mode 100644 index 0000000..9b0f298 --- /dev/null +++ b/pkg/types/decimal.go @@ -0,0 +1,5 @@ +package types + +import "github.com/govalues/decimal" + +type Decimal decimal.Decimal diff --git a/pkg/types/exchange.go b/pkg/types/exchange.go new file mode 100644 index 0000000..cdc9a84 --- /dev/null +++ b/pkg/types/exchange.go @@ -0,0 +1,33 @@ +package types + +import "sig-pub/api/pb" + +type Exchange string + +var ( + ExchangeSIG = Exchange(pb.Exchange_SIG.String()) + ExchangeOKX = Exchange(pb.Exchange_OKX.String()) // okx + ExchangeBINANCE = Exchange(pb.Exchange_BINANCE.String()) // 币安 +) + +func (ex Exchange) Exchange2PB() (pb.Exchange, bool) { + switch ex { + case ExchangeOKX: + return pb.Exchange_OKX, true + case ExchangeBINANCE: + return pb.Exchange_BINANCE, true + default: + return pb.Exchange_SIG, false + } +} + +func ExchangePBParse(pbex pb.Exchange) (Exchange, bool) { + switch pbex { + case pb.Exchange_OKX: + return ExchangeOKX, true + case pb.Exchange_BINANCE: + return ExchangeBINANCE, true + default: + return ExchangeSIG, false + } +} diff --git a/pkg/types/indicator.go b/pkg/types/indicator.go new file mode 100644 index 0000000..faac4d5 --- /dev/null +++ b/pkg/types/indicator.go @@ -0,0 +1,46 @@ +package types + +import ( + "github.com/govalues/decimal" +) + +// 定义 +type IndicatorDef struct { + IndicatorName string + Intervals []Interval +} + +// 指标定义接口, todo bbgo指标逻辑定义 +type Indicator interface { + Meta() IndicatorMeta + // init id, 计算完成 publish 时使用id, + // exchange: 封装indicator访问, 封装历史k线访问 + // args: 动态指标参数, 执行时创建 + Init(indId int64, exchange any, args map[string]any) + Dependes() []IndicatorDef // 需要订阅的指标列表(包括k线, 实时k线/关闭k线) + OnKline(Kline) // 驱动k线数据, 待驱动k线到达后, 再待subscribe计算完成后执行 + Emit(func(klineTs int64, indicators map[string]decimal.Decimal)) // 指标计算完成后发送, 由指标执行器进行存储或分发 +} + +// 指标注册/执行器系统分配 +// k线频率, 执行时指定 +// 指标数据(频率)存储, 实时计算 ? +type IndicatorMeta struct { + Name string `json:"name"` + // Desc string `json:"desc"` + // DescEn string `json:"descEn"` +} + +type IndicatorMacd struct { + Indicator +} + +func (ind *IndicatorMacd) Meta() IndicatorMeta { + return IndicatorMeta{ + Name: "macd", + } +} + +func (ind *IndicatorMacd) Init(indId int64, engine any) { + +} diff --git a/pkg/types/instance.go b/pkg/types/instance.go new file mode 100644 index 0000000..1ae9826 --- /dev/null +++ b/pkg/types/instance.go @@ -0,0 +1,9 @@ +package types + +// TradeInstance 交易产品 +type TradeInstance struct { + InstId string + TickSz int32 + MinSz int32 + // InstType string +} diff --git a/pkg/types/interval.go b/pkg/types/interval.go new file mode 100644 index 0000000..ec98b04 --- /dev/null +++ b/pkg/types/interval.go @@ -0,0 +1,86 @@ +package types + +var LossEmoji = "🔥" +var ProfitEmoji = "💰" + +type Interval string + +func (i Interval) Minutes() (int64, bool) { + m, ok := SupportedIntervals[i] + if !ok || m <= 0 { + return m, false + } + return m / 60, true +} + +func (i Interval) Seconds() (int64, bool) { + m, ok := SupportedIntervals[i] + if !ok || m <= 0 { + return m, false + } + return m, true +} + +func (i Interval) Milliseconds() (int64, bool) { + m, ok := SupportedIntervals[i] + if !ok || m <= 0 { + return m, false + } + return m * 1000, true +} + +var ( + Interval1s = Interval("1s") + Interval1m = Interval("1m") + Interval3m = Interval("3m") + Interval5m = Interval("5m") + Interval15m = Interval("15m") + Interval30m = Interval("30m") + Interval1h = Interval("1h") + Interval2h = Interval("2h") + Interval4h = Interval("4h") + Interval6h = Interval("6h") + Interval12h = Interval("12h") + Interval1d = Interval("1d") + Interval2d = Interval("2d") + Interval3d = Interval("3d") + Interval5d = Interval("5d") + Interval1w = Interval("1w") + Interval1mo = Interval("1mo") + Interval3mo = Interval("3mo") +) + +// IntervalWindow is used by the indicators +type IntervalWindow struct { + // The interval of kline + Interval Interval `json:"interval"` + + // The windows size of the indicator (for example, EWMA and SMA) + Window int `json:"window"` + + // RightWindow is used by the pivot indicator + RightWindow *int `json:"rightWindow"` +} + +type IntervalMap map[Interval]int64 + +var SupportedIntervals = IntervalMap{ + Interval1s: 1, + Interval1m: 1 * 60, + Interval3m: 3 * 60, + Interval5m: 5 * 60, + Interval15m: 15 * 60, + Interval30m: 30 * 60, + Interval1h: 60 * 60, + Interval2h: 60 * 60 * 2, + Interval4h: 60 * 60 * 4, + Interval6h: 60 * 60 * 6, + Interval12h: 60 * 60 * 12, + Interval1d: 60 * 60 * 24, + Interval2d: 60 * 60 * 24 * 2, + Interval3d: 60 * 60 * 24 * 3, + Interval5d: 60 * 60 * 24 * 5, + Interval1w: 60 * 60 * 24 * 7, + // Interval1mo: 60 * 60 * 24 * 30, + // Interval3mo: 60 * 60 * 24 * 30 * 3, +} diff --git a/pkg/types/kline.go b/pkg/types/kline.go new file mode 100644 index 0000000..781b153 --- /dev/null +++ b/pkg/types/kline.go @@ -0,0 +1,62 @@ +package types + +import ( + "sig-pub/api/pb" + + "github.com/govalues/decimal" +) + +// 时序数据 k线 + +type Kline struct { + // InstId string `json:"instId"` // 交易产品id + // Tid int64 `json:"tid"` // ts除interval + // Exchange string `json:"exchange"` // 交易所 + Interval Interval `json:"interval"` // 周期 + Ts int64 `json:"ts"` // k线时间戳ms + Open decimal.Decimal `json:"open"` // 开盘 + High decimal.Decimal `json:"high"` // 最高 + Low decimal.Decimal `json:"low"` // 最低 + Close decimal.Decimal `json:"close"` // 收盘 + Vol decimal.Decimal `json:"vol"` // 交易量 如果是币币，数值为交易货币的数量。 + VolQuote decimal.Decimal `json:"volQuote"` // 交易额 (交易量，以计价货币为单位) + Confirm bool `json:"confirm"` // k线是否完结 +} + +// okx.CandleData{[]string{"1746842757000", "0.20545", "0.20545", "0.2054", "0.2054", "7.57", "7570", "1555.2542", "1"}} + +func (k *Kline) ParsePBKline(exchange pb.Exchange, kline *pb.Kline) { + // k.Exchange = exchange.String() + k.Interval = Interval(kline.Interval) + k.Ts = kline.Ts + k.Open = decimal.MustParse(kline.Open) + k.High = decimal.MustParse(kline.High) + k.Low = decimal.MustParse(kline.Low) + k.Close = decimal.MustParse(kline.Close) + k.Vol = decimal.MustParse(kline.Vol) + k.VolQuote = decimal.MustParse(kline.VolQuote) + k.Confirm = kline.Confirm +} + +func (k *Kline) ToPBKline() (kline *pb.Kline) { + kline = &pb.Kline{ + Ts: k.Ts, + Interval: string(k.Interval), + Open: k.Open.String(), + High: k.High.String(), + Low: k.Low.String(), + Close: k.Close.String(), + Vol: k.Vol.String(), + VolQuote: k.VolQuote.String(), + Confirm: k.Confirm, + } + return +} + +// ChannelKline k线订阅消息 +type ChannelKline struct { + InstId string `json:"instId"` // 交易产品id，如 BTC-USDT-SWAP + ExchangeInstId string `json:"exchangeInstId"` // 交易所交易产品id + Exchange Exchange `json:"exchange"` // 交易所 + Klines []*Kline `json:"klines"` +} diff --git a/pkg/types/priority.go b/pkg/types/priority.go new file mode 100644 index 0000000..9e0af90 --- /dev/null +++ b/pkg/types/priority.go @@ -0,0 +1,10 @@ +package types + +type Priority int8 + +const ( + PriorityLow Priority = 1 + PriorityNorm Priority = 60 + PriorityHigh Priority = 120 + Priority127 Priority = 127 +) diff --git a/pkg/types/strategy.go b/pkg/types/strategy.go new file mode 100644 index 0000000..f6b0e00 --- /dev/null +++ b/pkg/types/strategy.go @@ -0,0 +1,30 @@ +package types + +type StrategyName string + +// 下单策略接口 +type Strategy interface { + Meta() StrategyMeta + // init id, 计算完成 publish 时使用id, + // exchange: 封装indicator访问, 封装历史k线访问 + // args: 动态策略参数, 执行时创建 + Init(indId int64, exchange any, args map[string]any) + Indicators() []string // 依赖指标列表, ind -> id(int64) + Emit() // 下单: sell/buy -> 持单,双方向?, score(分数权重) +} + +type StrategyMeta struct { + // Id string `json:"id"` // 策略注册/执行器系统分配 + Name string `json:"name"` + Desc string `json:"desc"` + DescEn string `json:"descEn"` +} + +// KlineServiceClient +// IndicatorClient.Sub(MACD5) stream[Indicator] +// IndicatorClient.Sub(ALMA) stream[Indicator] +// + +// 持单策略接口，动态止盈/止损/加仓/减仓 +type OrderStrategy interface { +} diff --git a/pkg/types/window.go b/pkg/types/window.go new file mode 100644 index 0000000..6f39d7e --- /dev/null +++ b/pkg/types/window.go @@ -0,0 +1,7 @@ +package types + +// 窗口接口 kline, indicator +type Window interface { + WindowGet(cur, index int64) + WindowRange(cur, after, before int64) +} diff --git a/pkg/utils/collect/collect.go b/pkg/utils/collect/collect.go new file mode 100644 index 0000000..c154bdf --- /dev/null +++ b/pkg/utils/collect/collect.go @@ -0,0 +1,178 @@ +package collect + +import "sort" + +func In[T comparable](value T, values ...T) bool { + if len(values) == 0 { + return false + } + for _, v := range values { + if value == v { + return true + } + } + return false +} + +func NotIn[T comparable](value T, values ...T) bool { + return !In(value, values...) +} + +func Count[T any](slice []T, isCount func(int, T) bool) int { + count := 0 + if slice == nil { + return count + } + for i, item := range slice { + if isCount(i, item) { + count++ + } + } + return count +} + +func Filter[T any](slice []T, predicate func(int, T) bool) []T { + var res []T + for i, item := range slice { + if predicate(i, item) { + res = append(res, item) + } + } + return res +} + +// Tail 获取切片尾部元素 +// dv: 空切片默认值 +func Tail[T any](slice []T, dv T) T { + if len(slice) == 0 { + return dv + } + return slice[len(slice)-1] +} + +func MapValues[K comparable, V any](kvs map[K]V) []V { + var values []V + if len(kvs) == 0 { + return values + } + for _, v := range kvs { + values = append(values, v) + } + return values +} + +func Sum[T int32 | int64 | int](nums []T) T { + var sum T = 0 + for _, num := range nums { + sum += num + } + return sum +} + +func Max[T int32 | int64 | int](nums []T, dv T) T { + if len(nums) == 0 { + return dv + } + var max T = nums[0] + for i := 1; i < len(nums); i++ { + if nums[i] > max { + max = nums[i] + } + } + return max +} + +func Slice2Map[T any, K comparable](slice []T, k func(int, T) K) map[K]T { + if slice == nil { + return make(map[K]T, 0) + } + + m := make(map[K]T, len(slice)) + for i, item := range slice { + m[k(i, item)] = item + } + return m +} + +func Slice2MapKv[T any, K comparable, V any](slice []T, mapping func(int, T) (K, V)) map[K]V { + if slice == nil { + return make(map[K]V, 0) + } + + m := make(map[K]V, len(slice)) + for i, item := range slice { + k, v := mapping(i, item) + m[k] = v + } + return m +} + +// SliceEquals compare slice all element equals +func SliceEquals[T comparable](slice1 []T, slice2 []T) bool { + if len(slice1) != len(slice2) { + return false + } + for i, ele := range slice1 { + if ele != slice2[i] { + return false + } + } + return true +} + +func Join(slice []string, on string) (res string) { + length := len(slice) + for i := 0; i < length; i++ { + res += slice[i] + if i < length-1 { + res += on + } + } + return +} + +func CopyMap[K comparable, V any](src map[K]V) (dst map[K]V) { + dst = make(map[K]V, len(src)) + for k, v := range src { + dst[k] = v + } + return +} + +func CopyMap2[K comparable, K2 comparable, V2 any](src map[K]map[K2]V2) (dst map[K]map[K2]V2) { + dst = make(map[K]map[K2]V2, len(src)) + for k, v := range src { + cv := make(map[K2]V2, len(v)) + for k2, v2 := range v { + cv[k2] = v2 + } + dst[k] = cv + } + return +} + +func IndexMapping[T any](slice []T) (index []int) { + index = make([]int, len(slice)) + for i := range slice { + index[i] = i + } + return +} + +func SortIndex[T any](slice []T, less func(i, j int) bool) (sortIndex []int) { + sortIndex = IndexMapping(slice) + sort.Slice(sortIndex, func(i, j int) bool { + return less(sortIndex[i], sortIndex[j]) + }) + return +} + +func SliceMapping[T interface{}, M interface{}](slice []T, mapping func(T) M) (m []M) { + if len(slice) == 0 { + return + } + for _, item := range slice { + m = append(m, mapping(item)) + } + return +} diff --git a/pkg/utils/collect/concurrent_map.go b/pkg/utils/collect/concurrent_map.go new file mode 100644 index 0000000..9df8dd7 --- /dev/null +++ b/pkg/utils/collect/concurrent_map.go @@ -0,0 +1,219 @@ +package collect + +import ( + "hash/fnv" + "sync" +) + +// ConcurrentMap 分段锁 map, 提升并发性 +type ConcurrentMap[K comparable, V any] struct { + hashKeyFunc func(K) string + // equalsFunc func(v1, v2 V) bool + // counter int64 + segments int + segmentsMap []map[K]V + segmentsLock []*sync.RWMutex +} + +// NewConcurrentMap 分段锁并发 map +// segments: 分段数 +// hashKeyFunc: key转string函数 +func NewConcurrentMap[K comparable, V any](concurrencyLevel int, hashKeyFunc func(K) string) *ConcurrentMap[K, V] { + segments := 1 // segments = 2^n + for segments < concurrencyLevel { + segments <<= 1 + } + + m := &ConcurrentMap[K, V]{ + hashKeyFunc: hashKeyFunc, + segments: segments, + segmentsMap: make([]map[K]V, segments), + segmentsLock: make([]*sync.RWMutex, segments), + } + for i := 0; i < segments; i++ { + m.segmentsMap[i] = make(map[K]V, 16) + m.segmentsLock[i] = &sync.RWMutex{} + } + return m +} + +// segment 根据 key 确定分段 +func (m *ConcurrentMap[K, V]) segment(k K) int { + hashK := m.hashKeyFunc(k) + hash := fnv32Hash(hashK) + return int(hash & uint32(m.segments-1)) +} + +func (m *ConcurrentMap[K, V]) update(k K, update func(map[K]V)) { + segment := m.segment(k) + lock := m.segmentsLock[segment] + lock.Lock() + defer lock.Unlock() + update(m.segmentsMap[segment]) +} + +// Store 放置新值 +func (m *ConcurrentMap[K, V]) Store(k K, v V) { + m.update(k, func(segment map[K]V) { + segment[k] = v + }) +} + +func (m *ConcurrentMap[K, V]) Load(k K) (v V, ok bool) { + segment := m.segment(k) + lock := m.segmentsLock[segment] + lock.RLock() + defer lock.RUnlock() + v, ok = m.segmentsMap[segment][k] + return +} + +func (m *ConcurrentMap[K, V]) Delete(k K) { + m.update(k, func(segment map[K]V) { + delete(segment, k) + }) +} + +// Range 遍历时勿修改值造成死锁 +func (m *ConcurrentMap[K, V]) Range(f func(key K, value V) (next bool)) { + for i := 0; i < m.segments; i++ { + lock := m.segmentsLock[i] + func() { + lock.RLock() + defer lock.RUnlock() + + for k, v := range m.segmentsMap[i] { + if !f(k, v) { + i = m.segments // stop range + return + } + } + }() + } +} + +func (m *ConcurrentMap[K, V]) RangeUpdate(f func(key K, value V) (next, remove bool, newV V)) { + for i := 0; i < m.segments; i++ { + lock := m.segmentsLock[i] + func() { + lock.Lock() + defer lock.Unlock() + + for k, v := range m.segmentsMap[i] { + next, remove, newV := f(k, v) + if remove { + delete(m.segmentsMap[i], k) + } else { + m.segmentsMap[i][k] = newV + } + if !next { + i = m.segments // stop range + return + } + } + }() + } +} + +func (m *ConcurrentMap[K, V]) LoadRLock(k K, f func(v V, ok bool)) { + segment := m.segment(k) + lock := m.segmentsLock[segment] + lock.RLock() + defer lock.RUnlock() + v, ok := m.segmentsMap[segment][k] + f(v, ok) +} + +// LoadAndUpdate 更新新值 +func (m *ConcurrentMap[K, V]) LoadAndUpdate(k K, f func(v V) (remove bool, nextV V)) (value V) { + m.update(k, func(segment map[K]V) { + remove, nextV := f(segment[k]) + if remove { + delete(segment, k) + } else { + segment[k] = nextV + value = nextV + } + }) + return +} + +func (m *ConcurrentMap[K, V]) LoadAndDelete(k K) (value V, loaded bool) { + m.update(k, func(segment map[K]V) { + value, loaded = segment[k] + if loaded { + delete(segment, k) + } + }) + return +} + +func (m *ConcurrentMap[K, V]) Swap(k K, v V) (prev V, loaded bool) { + m.update(k, func(segment map[K]V) { + prev, loaded = segment[k] + segment[k] = v + }) + return +} + +func (m *ConcurrentMap[K, V]) Size() (size int) { + for i := 0; i < m.segments; i++ { + lock := m.segmentsLock[i] + func() { + lock.RLock() + defer lock.RUnlock() + size += len(m.segmentsMap[i]) + }() + } + return +} + +// ComputeIfAbsent 加载, 如果值不存在使用 mapping(k) 填充并返回 +// mapped 值是否是 mapping(k) 填充的 +func (m *ConcurrentMap[K, V]) ComputeIfAbsent(k K, mapping func(k K) V) (res V, mapped bool) { + res, _, mapped = m.ComputeIfAbsentE(k, func(k K) (V, error) { + return mapping(k), nil + }) + return +} + +// ComputeIfAbsentE 加载, 如果值不存在使用 mapping(k) 填充并返回 +// mapped 值是否是 mapping(k) 填充的 +func (m *ConcurrentMap[K, V]) ComputeIfAbsentE(k K, mapping func(k K) (V, error)) (res V, err error, mapped bool) { + segment := m.segment(k) + lock := m.segmentsLock[segment] + lock.RLock() + v, ok := m.segmentsMap[segment][k] + lock.RUnlock() + if ok { + res = v + return + } + + lock.Lock() + defer lock.Unlock() + + // double check + if v, ok = m.segmentsMap[segment][k]; ok { + res = v + return + } + + // write mapping value + res, err = mapping(k) + if err != nil { + return + } + m.segmentsMap[segment][k] = res + mapped = true + return +} + +func fnv32Hash(k string) uint32 { + f := fnv.New32() + _, err := f.Write([]byte(k)) + if err != nil { + panic(err) + } + return f.Sum32() +} diff --git a/pkg/utils/collect/concurrent_map_test.go b/pkg/utils/collect/concurrent_map_test.go new file mode 100644 index 0000000..309e5bc --- /dev/null +++ b/pkg/utils/collect/concurrent_map_test.go @@ -0,0 +1,140 @@ +package collect + +import ( + "fmt" + "math/rand" + "strconv" + "sync" + "testing" + "time" +) + +func TestConcurrentMap(t *testing.T) { + cm := NewConcurrentMap[string, string](19, func(k string) string { return k }) + concurrent := 1000 + + wg := sync.WaitGroup{} + wg.Add(concurrent) + + for i := 0; i < concurrent; i++ { + go func(loop int) { + for j := 0; j < concurrent; j++ { + k := fmt.Sprintf("%d-%d", loop, j) + cm.Store(k, k) + } + wg.Done() + }(i) + } + wg.Wait() + + var sum int + cm.RangeUpdate(func(key, value string) (bool, bool, string) { + if key != value { + panic(fmt.Errorf("value load error: %s:%s", key, value)) + } + return true, false, value + "-update" + }) + cm.Range(func(key, value string) bool { + sum++ + if (key + "-update") != value { + panic(fmt.Errorf("value update error: %s:%s", key, value)) + } + return true + }) + if sum != (concurrent * concurrent) { + t.Errorf("count error: %d", sum) + } + + wg.Add(concurrent) + for i := 0; i < concurrent; i++ { + go func() { + r := rand.New(rand.NewSource(time.Now().UnixMilli())) + for i := 0; i < concurrent; i++ { + k := fmt.Sprintf("%d-%d", r.Intn(concurrent), r.Intn(concurrent)) + v, ok := cm.Load(k) + if !ok || (k+"-update") != v { + panic(fmt.Errorf("value load error: %s", k)) + } + } + wg.Done() + }() + } + wg.Wait() +} + +type counter struct { + c int +} + +func (c *counter) increment() { + c.c += 1 +} + +func TestComputeIfAbsent(t *testing.T) { + cm := NewConcurrentMap[int, *counter](16, func(k int) string { return strconv.Itoa(k) }) + concurrent := 1000 + add := 10 + wg := &sync.WaitGroup{} + wg.Add(concurrent) + + for i := 0; i < concurrent; i++ { + go func() { + defer wg.Done() + + r := rand.New(rand.NewSource(time.Now().UnixMilli())) + v, mapped := cm.ComputeIfAbsent(r.Intn(10), func(k int) *counter { + return &counter{} + }) + if !mapped { + return + } + for i := 0; i < add; i++ { + v.increment() + } + }() + } + + wg.Wait() + + cm.Range(func(k int, v *counter) bool { + if v.c != add { + t.Error("error value...") + } + return true + }) +} + +func TestBenchmark(t *testing.T) { + cm := NewConcurrentMap[string, int](16, func(s string) string { return s }) + for i := range 10000 { + cm.Store(strconv.Itoa(i), i) + } + + // sm := &sync.Map{} + // for i := range 10000 { + // sm.Store(strconv.Itoa(i), i) + // } + + // mm := make(map[string]int, 10000) + // for i := range 10000 { + // mm[strconv.Itoa(i)] = i + // } + wg := &sync.WaitGroup{} + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + + now := time.Now() + for range 10000 { + for i := range 10000 { + _, _ = cm.Load(strconv.Itoa(i)) + // _, _ = sm.Load(strconv.Itoa(i)) + // _, _ = mm[strconv.Itoa(i)] + } + } + fmt.Printf("done: use %dms\n", time.Now().UnixMilli()-now.UnixMilli()) + }() + } + wg.Wait() +} diff --git a/pkg/utils/conver/unit_conver.go b/pkg/utils/conver/unit_conver.go new file mode 100644 index 0000000..3c5752a --- /dev/null +++ b/pkg/utils/conver/unit_conver.go @@ -0,0 +1,44 @@ +package conver + +import ( + "fmt" + "time" + + "github.com/dsnet/golib/unitconv" +) + +// ParseDataUnit parse 1Ki -> 1024, 1K -> 1000 +func ParseDataUnit(unit string) (val float64, err error) { + val, err = unitconv.ParsePrefix(unit, unitconv.AutoParse) + return +} + +func ParseDataUnitInt(unit string) (val int, err error) { + v, err := ParseDataUnit(unit) + val = int(v) + return +} + +func MustParseDataUnit(unit string) (val float64) { + val, err := unitconv.ParsePrefix(unit, unitconv.AutoParse) + if err != nil { + panic(fmt.Errorf("error parse data unit %s, %s", unit, err.Error())) + } + return +} + +func MustParseDataUnitInt(unit string) (val int) { + return int(MustParseDataUnit(unit)) +} + +func ParseDuration(s string) (time.Duration, error) { + return time.ParseDuration(s) +} + +func MustParseDuration(s string) time.Duration { + d, err := ParseDuration(s) + if err != nil { + panic(err) + } + return d +} diff --git a/pkg/utils/exit/option.go b/pkg/utils/exit/option.go new file mode 100644 index 0000000..5869682 --- /dev/null +++ b/pkg/utils/exit/option.go @@ -0,0 +1,29 @@ +package exit + +var defaultOptions = Options{ + Order: 1, +} + +type Options struct { + Order int // 0头部, 1中间, 2尾部 +} + +type Option func(opts *Options) + +func WithOrderFront() Option { + return func(opts *Options) { + opts.Order = 0 + } +} + +func WithOrderMiddle() Option { + return func(opts *Options) { + opts.Order = 1 + } +} + +func WithOrderTail() Option { + return func(opts *Options) { + opts.Order = 2 + } +} diff --git a/pkg/utils/exit/signal.go b/pkg/utils/exit/signal.go new file mode 100644 index 0000000..89b2a17 --- /dev/null +++ b/pkg/utils/exit/signal.go @@ -0,0 +1,74 @@ +package exit + +import ( + "log" + "os" + "os/signal" + "sig-pub/pkg/zlog" + "sync" + "sync/atomic" + "syscall" + "time" +) + +var ( + AwaitSeconds = 3 + shutdownHooks [3][]func() // front,middle,back hooks + lock = &sync.Mutex{} + sigChan = make(chan os.Signal, 1) +) + +func AddHook(hook func(), options ...Option) { + opts := defaultOptions + for _, opt := range options { + opt(&opts) + } + + lock.Lock() + defer lock.Unlock() + shutdownHooks[opts.Order] = append(shutdownHooks[opts.Order], hook) +} + +func Await() { + // 监听两个信号: TERM信号(kill + 进程号)触发, 中断信号(ctrl + c)触发 + signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) + s := <-sigChan + // 监听到关闭信号 + zlog.Info("catch exit signal: ", s) + + awaitSeconds := AwaitSeconds + var success, fail int32 + for i, hooks := range shutdownHooks { + for _, hook := range hooks { + func() { + defer func() { + if err := recover(); err != nil { + log.Println("exec shutdown hook panic: ", err) + atomic.AddInt32(&fail, 1) + return + } + atomic.AddInt32(&success, 1) + }() + + hook() + }() + } + + // 间隔1s再执行 + if i < 2 && len(hooks) > 0 { + time.Sleep(time.Second) + awaitSeconds -= 1 + } + } + + if awaitSeconds < 1 { + awaitSeconds = 1 + } + + zlog.Infof("execute %d shutdown hook %d ok, %d failed, exit in %d seconds...\n", success+fail, success, fail, awaitSeconds) + time.Sleep(time.Second * time.Duration(awaitSeconds)) +} + +//func Shutdown() { +// sigChan <- syscall.SIGQUIT +//} diff --git a/pkg/utils/files/files.go b/pkg/utils/files/files.go new file mode 100644 index 0000000..9de37a0 --- /dev/null +++ b/pkg/utils/files/files.go @@ -0,0 +1,33 @@ +package files + +import ( + "os" + "strings" +) + +// ParsePathFlag +func ParsePathFlag(confPath string) (dir, fileName, name, ext string) { + idx := strings.LastIndex(confPath, "/") + dir = confPath[:idx+1] + fileName = confPath[idx+1:] + idx2 := strings.LastIndex(fileName, ".") + name = fileName[:idx2] + ext = fileName[idx2+1:] + return +} + +func GetFilenameExt(filename string) (ext string) { + idx := strings.LastIndex(filename, ".") + ext = filename[idx+1:] + return +} + +// FileExists reports whether the named file or directory exists. +func FileExists(name string) bool { + if _, err := os.Stat(name); err != nil { + if os.IsNotExist(err) { + return false + } + } + return true +} diff --git a/pkg/utils/id/snowflake_id.go b/pkg/utils/id/snowflake_id.go new file mode 100644 index 0000000..36df156 --- /dev/null +++ b/pkg/utils/id/snowflake_id.go @@ -0,0 +1,38 @@ +package id + +import ( + "math" + "strconv" + + "github.com/bwmarrin/snowflake" +) + +const snowStartTime int64 = 1714533741822 + +var snowNode *snowflake.Node + +func InitSnowflakeId(machineID int64) { + var err error + snowflake.Epoch = snowStartTime + snowNode, err = snowflake.NewNode(machineID) + if err != nil { + panic(err) + } +} + +func GenSnowId() int64 { + return snowNode.Generate().Int64() +} + +func GenSnowIdS() string { + return strconv.FormatInt(snowNode.Generate().Int64(), 10) +} + +// 使用位操作的方式转换 +func Int64ToFloat64Bits(i int64) float64 { + return math.Float64frombits(uint64(i)) +} + +func Float64ToInt64Bits(f float64) int64 { + return int64(math.Float64bits(f)) +} diff --git a/pkg/utils/id/snowflake_id_test.go b/pkg/utils/id/snowflake_id_test.go new file mode 100644 index 0000000..11bd000 --- /dev/null +++ b/pkg/utils/id/snowflake_id_test.go @@ -0,0 +1,13 @@ +package id + +import ( + "fmt" + "testing" +) + +func TestSnowflakeId(t *testing.T) { + InitSnowflakeId(1) + for i := 0; i < 10; i++ { + fmt.Println(GenSnowIdS()) + } +} diff --git a/pkg/utils/kvcache/cache_test.go b/pkg/utils/kvcache/cache_test.go new file mode 100644 index 0000000..345ffb2 --- /dev/null +++ b/pkg/utils/kvcache/cache_test.go @@ -0,0 +1,20 @@ +package kvcache + +import ( + "fmt" + "testing" + "time" + + "github.com/fanjindong/go-cache" +) + +func TestCache(t *testing.T) { + c := cache.NewMemCache(cache.WithShards(8), cache.WithClearInterval(time.Minute)) + c.Set("a", 1) + c.Set("b", 1, cache.WithEx(1*time.Second)) + time.Sleep(1 * time.Second) + v, ok := c.Get("a") // 1, true + fmt.Println(v, ok) + v, ok = c.Get("b") // nil, false + fmt.Println(v, ok) +} diff --git a/pkg/utils/kvcache/kvcache.go b/pkg/utils/kvcache/kvcache.go new file mode 100644 index 0000000..1b61b72 --- /dev/null +++ b/pkg/utils/kvcache/kvcache.go @@ -0,0 +1,68 @@ +package kvcache + +import ( + "time" + + "github.com/fanjindong/go-cache" +) + +const ( + NoExpiration time.Duration = -1 +) + +type KVCache[V any] struct { + c cache.ICache + expiration time.Duration +} + +func NewKVCache[V any]() *KVCache[V] { + return NewExpireStore[V](NoExpiration) +} + +func NewExpireStore[V any](defaultExpiration time.Duration, iopts ...cache.ICacheOption) *KVCache[V] { + opts := []cache.ICacheOption{ + cache.WithShards(16), + cache.WithClearInterval(time.Minute), + } + opts = append(opts, iopts...) + c := cache.NewMemCache(opts...) + return &KVCache[V]{ + c: c, + expiration: defaultExpiration, + } +} + +func (s *KVCache[V]) Set(k string, v V) { + if s.expiration == NoExpiration { + s.c.Set(k, v) + return + } + s.c.Set(k, v, cache.WithEx(s.expiration)) +} + +func (s *KVCache[V]) SetEx(k string, v V, ex time.Duration) { + s.c.Set(k, v, cache.WithEx(ex)) +} + +func (s *KVCache[V]) Get(k string) (v V, ok bool) { + val, ok := s.c.Get(k) + if !ok { + return + } + v = val.(V) + return +} + +func (s *KVCache[V]) Delete(keys ...string) { + s.c.Del(keys...) +} + +func (s *KVCache[V]) ForEach(f func(k string, v V)) { + for k, v := range s.c.ToMap() { + f(k, v.(V)) + } +} + +func (s *KVCache[V]) Count() int { + return len(s.c.ToMap()) +} diff --git a/pkg/utils/nets/ip.go b/pkg/utils/nets/ip.go new file mode 100644 index 0000000..183f0d6 --- /dev/null +++ b/pkg/utils/nets/ip.go @@ -0,0 +1,85 @@ +package nets + +import ( + "errors" + "fmt" + "math" + "math/big" + "net" + "strconv" + "strings" +) + +// GetHostIpv4 获取本地内网IP +func GetHostIpv4() (string, error) { + privates, err := getAllIPV4(func(ip net.IP) bool { + return ip.IsPrivate() + }) + if err != nil { + return "", err + } + if len(privates) == 0 { + return "", errors.New("no private ip") + } + return privates[0], nil +} + +func getAllIPV4(filter func(net.IP) bool) (ips []string, err error) { + // 获取所有网卡 + addrs, err := net.InterfaceAddrs() + if err != nil { + return + } + + for _, addr := range addrs { + // 这个网络地址是IP地址: ipv4, ipv6 + ipNet, isIpNet := addr.(*net.IPNet) + if isIpNet && !ipNet.IP.IsLoopback() { + // 跳过IPV6 + if ipNet.IP.To4() != nil { + if filter(ipNet.IP) { + ips = append(ips, ipNet.IP.String()) + } + } + } + } + return +} + +func Address2i64(addr string) (int64, error) { + idx := strings.Index(addr, ":") + if idx < 0 { + return 0, fmt.Errorf("invalid addr %s", addr) + } + ip, port := addr[0:idx], addr[idx+1:] + portI, err := strconv.Atoi(port) + if err != nil { + return 0, err + } + ipI, err := Ip2i64(ip) + if err != nil { + return 0, err + } + return (int64(portI) << 32) | ipI, nil +} + +func I642Address(addr int64) string { + port := addr >> 32 + ip := addr & math.MaxUint32 + return fmt.Sprintf("%s:%d", I642Ip(ip), port) +} + +func Ip2i64(ip string) (int64, error) { + ip4 := net.ParseIP(ip).To4() + if ip4 == nil { + return 0, fmt.Errorf("invalid ip %s", ip) + } + ret := big.NewInt(0) + ret.SetBytes(ip4) + return ret.Int64(), nil +} + +func I642Ip(ip int64) string { + return fmt.Sprintf("%d.%d.%d.%d", + byte(ip>>24), byte(ip>>16), byte(ip>>8), byte(ip)) +} diff --git a/pkg/utils/nets/ip_test.go b/pkg/utils/nets/ip_test.go new file mode 100644 index 0000000..ae9c999 --- /dev/null +++ b/pkg/utils/nets/ip_test.go @@ -0,0 +1,28 @@ +package nets + +import "testing" + +func TestIpConvert(t *testing.T) { + ip := "192.168.1.110" + // ip := "255.255.255.255" + i64, err := Ip2i64(ip) + if err != nil { + t.Error(err) + } + ip2 := I642Ip(int64(int32(i64))) + if ip2 != ip { + t.Error("ip parse error") + } +} + +func TestAddressConvert(t *testing.T) { + addr := "192.168.1.110:8080" + i64, err := Address2i64(addr) + if err != nil { + t.Error(err) + } + addr2 := I642Address(i64) + if addr2 != addr { + t.Error("address parse error") + } +} diff --git a/pkg/utils/resp/resp.go b/pkg/utils/resp/resp.go new file mode 100644 index 0000000..9654c04 --- /dev/null +++ b/pkg/utils/resp/resp.go @@ -0,0 +1,63 @@ +package resp + +import ( + "github.com/bytedance/sonic" +) + +const ( + CodeOK = 200 + CodeFail = 400 + CodeError = 500 +) + +type H map[string]interface{} + +// Response 响应体包装 +type Response struct { + Seq int `json:"seq,omitempty"` + Code int `json:"code,omitempty"` + Msg string `json:"msg,omitempty"` + Data any `json:"data,omitempty"` + Extra map[string]any `json:"extra,omitempty"` +} + +func (resp *Response) Json() ([]byte, error) { + json, err := sonic.Marshal(resp) + if err != nil { + return nil, err + } + return json, nil +} + +func (resp *Response) JsonString() (string, error) { + bytes, err := resp.Json() + if err != nil { + return "", err + } + return string(bytes), nil +} + +func RespSeq(seq int, code int, msg string, data any) *Response { + return &Response{ + Seq: seq, + Code: code, + Msg: msg, + Data: data, + } +} + +func Resp(code int, msg string, data any) *Response { + return RespSeq(0, code, msg, data) +} + +func Success(data any) *Response { + return Resp(CodeOK, "", data) +} + +func Fail(msg string) *Response { + return Resp(CodeFail, msg, nil) +} + +func Error(msg string) *Response { + return Resp(CodeError, msg, nil) +} diff --git a/pkg/utils/strs/random.go b/pkg/utils/strs/random.go new file mode 100644 index 0000000..964f8a9 --- /dev/null +++ b/pkg/utils/strs/random.go @@ -0,0 +1,85 @@ +// reference go-zero/core/stringx/random.go + +package strs + +import ( + crand "crypto/rand" + "fmt" + "math/rand" + "sync" + "time" +) + +const ( + letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + letterIdxBits = 6 // 6 bits to represent a letter index + idLen = 8 + defaultRandLen = 8 + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + + return string(b) +} + +// Seed sets the seed to seed. +func Seed(seed int64) { + src.Seed(seed) +} diff --git a/pkg/utils/strs/variable.go b/pkg/utils/strs/variable.go new file mode 100644 index 0000000..7d15490 --- /dev/null +++ b/pkg/utils/strs/variable.go @@ -0,0 +1,74 @@ +package strs + +import "strings" + +// SnakeString 驼峰转蛇形 +func SnakeString(s string) string { + data := make([]byte, 0, len(s)*2) + j := false + num := len(s) + for i := 0; i < num; i++ { + d := s[i] + // or通过ASCII码进行大小写的转化 + // 65-90（A-Z），97-122（a-z） + //判断如果字母为大写的A-Z就在前面拼接一个_ + if i > 0 && d >= 'A' && d <= 'Z' && j { + data = append(data, '_') + } + if d != '_' { + j = true + } + data = append(data, d) + } + //ToLower把大写字母统一转小写 + return strings.ToLower(string(data[:])) +} + +// CamelString 蛇形转驼峰 +func CamelString(s string) string { + data := make([]byte, 0, len(s)) + j := false + k := false + num := len(s) - 1 + for i := 0; i <= num; i++ { + d := s[i] + if k == false && d >= 'A' && d <= 'Z' { + k = true + } + if d >= 'a' && d <= 'z' && (j || k == false) { + d = d - 32 + j = false + k = true + } + if k && d == '_' && num > i && s[i+1] >= 'a' && s[i+1] <= 'z' { + j = true + continue + } + data = append(data, d) + } + return string(data[:]) +} + +// LowerInitialLetter 首字母小写 +func LowerInitialLetter(s string) string { + if s == "" { + return s + } + letters := []rune(s) + if letters[0] >= 'A' && letters[0] <= 'Z' { + letters[0] += 32 + } + return string(letters) +} + +// UpperInitialLetter 首字母大写 +func UpperInitialLetter(s string) string { + if s == "" { + return s + } + letters := []rune(s) + if letters[0] >= 'a' && letters[0] <= 'z' { + letters[0] -= 32 + } + return string(letters) +} diff --git a/pkg/utils/validator/validator.go b/pkg/utils/validator/validator.go new file mode 100644 index 0000000..d1f00da --- /dev/null +++ b/pkg/utils/validator/validator.go @@ -0,0 +1,12 @@ +package validator + +import "strings" + +func IsAnyBlank(strs ...string) bool { + for _, str := range strs { + if strings.TrimSpace(str) == "" { + return true + } + } + return false +} diff --git a/pkg/zlog/exported.go b/pkg/zlog/exported.go new file mode 100644 index 0000000..6498074 --- /dev/null +++ b/pkg/zlog/exported.go @@ -0,0 +1,38 @@ +package zlog + +func Info(args ...any) { + zSugar.Info(args...) +} +func Infoln(args ...any) { + zSugar.Infoln(args...) +} +func Infof(format string, args ...any) { + zSugar.Infof(format, args...) +} +func Warning(args ...any) { + zSugar.Warn(args...) +} +func Warningln(args ...any) { + zSugar.Warnln(args...) +} +func Warningf(format string, args ...any) { + zSugar.Warnf(format, args...) +} +func Error(args ...any) { + zSugar.Error(args...) +} +func Errorln(args ...any) { + zSugar.Errorln(args...) +} +func Errorf(format string, args ...any) { + zSugar.Errorf(format, args...) +} +func Fatal(args ...any) { + zSugar.Fatal(args...) +} +func Fatalln(args ...any) { + zSugar.Fatalln(args...) +} +func Fatalf(format string, args ...any) { + zSugar.Fatalf(format, args...) +} diff --git a/pkg/zlog/zaplog.go b/pkg/zlog/zaplog.go new file mode 100644 index 0000000..7022837 --- /dev/null +++ b/pkg/zlog/zaplog.go @@ -0,0 +1,56 @@ +package zlog + +import ( + "os" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +var ( + zLogger *zap.Logger + zSugar *zap.SugaredLogger +) + +func init() { + // 默认日志器 + // zLogger, _ = zap.NewProduction() + // zSugar = zLogger.Sugar() + Init() +} + +func Sync() { + _ = zLogger.Sync() +} + +// 初始化日志器 +func Init(coreOptions ...CoreOption) { + encoder := getEncoder() + + // 控制台输出（所有级别） + cores := []zapcore.Core{ + zapcore.NewCore(encoder, zapcore.Lock(os.Stdout), zapcore.DebugLevel), // 控制台输出 + } + + // 输出日志到多个目标(控制台+文件) + for _, option := range coreOptions { + core := option(encoder) + cores = append(cores, core) + } + + // 创建 Core（核心日志器） + core := zapcore.NewTee(cores...) + + zLogger = zap.New(core, zap.AddCaller(), zap.AddCallerSkip(1)) + zSugar = zLogger.Sugar() +} + +func getEncoder() zapcore.Encoder { + encoderConfig := zap.NewProductionEncoderConfig() + encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder // 修改时间编码器 + + // 在日志文件中使用大写字母记录日志级别 + encoderConfig.EncodeLevel = zapcore.CapitalLevelEncoder + // NewConsoleEncoder 打印更符合人们观察的方式 + return zapcore.NewConsoleEncoder(encoderConfig) +} diff --git a/pkg/zlog/zaplog_option.go b/pkg/zlog/zaplog_option.go new file mode 100644 index 0000000..569b217 --- /dev/null +++ b/pkg/zlog/zaplog_option.go @@ -0,0 +1,35 @@ +package zlog + +import ( + "fmt" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" +) + +type CoreOption func(encoder zapcore.Encoder) zapcore.Core + +// WithLogFile 文件输出 +// 使用 lumberjack 进行日志轮转 +func WithLogFile(level zapcore.Level, fileConfig func(*lumberjack.Logger)) CoreOption { + return func(encoder zapcore.Encoder) zapcore.Core { + fileLog := &lumberjack.Logger{ + Filename: fmt.Sprintf("%s.log", level.String()), // level 日志文件 + MaxSize: 10, // 日志文件大小（MB） + MaxBackups: 3, // 保留旧日志文件数量 + MaxAge: 30, // 保留旧日志天数 + } + if fileConfig != nil { + fileConfig(fileLog) + } + fileWriter := zapcore.AddSync(fileLog) + + // 定义日志级别 + levelPriority := zap.LevelEnablerFunc(func(lvl zapcore.Level) bool { + // return lvl >= level // level 及以上级别 + return lvl == level // leve l级别 + }) + return zapcore.NewCore(encoder, fileWriter, levelPriority) + } +} diff --git a/pkg/zlog/zlog_test.go b/pkg/zlog/zlog_test.go new file mode 100644 index 0000000..a09951c --- /dev/null +++ b/pkg/zlog/zlog_test.go @@ -0,0 +1,21 @@ +package zlog + +import ( + "testing" + + "go.uber.org/zap" +) + +func TestLog(t *testing.T) { + // Init(WithLogFile(zapcore.ErrorLevel, func(l *lumberjack.Logger) { + // l.Filename = "./logs/sig_error.log" + // })) + + Init() + defer Sync() + + Info("hello 1info...") + Errorf("hell error...%d", 1803) + + zLogger.Error("error日志", zap.Stack("stacktrace")) +}