github.com/vipernet-xyz/tm@v0.34.24/abci/client/grpc_client.go (about) 1 package abcicli 2 3 import ( 4 "fmt" 5 "net" 6 "sync" 7 "time" 8 9 "golang.org/x/net/context" 10 "google.golang.org/grpc" 11 12 "github.com/vipernet-xyz/tm/abci/types" 13 tmnet "github.com/vipernet-xyz/tm/libs/net" 14 "github.com/vipernet-xyz/tm/libs/service" 15 tmsync "github.com/vipernet-xyz/tm/libs/sync" 16 ) 17 18 var _ Client = (*grpcClient)(nil) 19 20 // A stripped copy of the remoteClient that makes 21 // synchronous calls using grpc 22 type grpcClient struct { 23 service.BaseService 24 mustConnect bool 25 26 client types.ABCIApplicationClient 27 conn *grpc.ClientConn 28 chReqRes chan *ReqRes // dispatches "async" responses to callbacks *in order*, needed by mempool 29 30 mtx tmsync.Mutex 31 addr string 32 err error 33 resCb func(*types.Request, *types.Response) // listens to all callbacks 34 } 35 36 func NewGRPCClient(addr string, mustConnect bool) Client { 37 cli := &grpcClient{ 38 addr: addr, 39 mustConnect: mustConnect, 40 // Buffering the channel is needed to make calls appear asynchronous, 41 // which is required when the caller makes multiple async calls before 42 // processing callbacks (e.g. due to holding locks). 64 means that a 43 // caller can make up to 64 async calls before a callback must be 44 // processed (otherwise it deadlocks). It also means that we can make 64 45 // gRPC calls while processing a slow callback at the channel head. 46 chReqRes: make(chan *ReqRes, 64), 47 } 48 cli.BaseService = *service.NewBaseService(nil, "grpcClient", cli) 49 return cli 50 } 51 52 func dialerFunc(ctx context.Context, addr string) (net.Conn, error) { 53 return tmnet.Connect(addr) 54 } 55 56 func (cli *grpcClient) OnStart() error { 57 if err := cli.BaseService.OnStart(); err != nil { 58 return err 59 } 60 61 // This processes asynchronous request/response messages and dispatches 62 // them to callbacks. 63 go func() { 64 // Use a separate function to use defer for mutex unlocks (this handles panics) 65 callCb := func(reqres *ReqRes) { 66 cli.mtx.Lock() 67 defer cli.mtx.Unlock() 68 69 reqres.Done() 70 71 // Notify client listener if set 72 if cli.resCb != nil { 73 cli.resCb(reqres.Request, reqres.Response) 74 } 75 76 // Notify reqRes listener if set 77 reqres.InvokeCallback() 78 } 79 for reqres := range cli.chReqRes { 80 if reqres != nil { 81 callCb(reqres) 82 } else { 83 cli.Logger.Error("Received nil reqres") 84 } 85 } 86 }() 87 88 RETRY_LOOP: 89 for { 90 //nolint:staticcheck // SA1019 Existing use of deprecated but supported dial option. 91 conn, err := grpc.Dial(cli.addr, grpc.WithInsecure(), grpc.WithContextDialer(dialerFunc)) 92 if err != nil { 93 if cli.mustConnect { 94 return err 95 } 96 cli.Logger.Error(fmt.Sprintf("abci.grpcClient failed to connect to %v. Retrying...\n", cli.addr), "err", err) 97 time.Sleep(time.Second * dialRetryIntervalSeconds) 98 continue RETRY_LOOP 99 } 100 101 cli.Logger.Info("Dialed server. Waiting for echo.", "addr", cli.addr) 102 client := types.NewABCIApplicationClient(conn) 103 cli.conn = conn 104 105 ENSURE_CONNECTED: 106 for { 107 _, err := client.Echo(context.Background(), &types.RequestEcho{Message: "hello"}, grpc.WaitForReady(true)) 108 if err == nil { 109 break ENSURE_CONNECTED 110 } 111 cli.Logger.Error("Echo failed", "err", err) 112 time.Sleep(time.Second * echoRetryIntervalSeconds) 113 } 114 115 cli.client = client 116 return nil 117 } 118 } 119 120 func (cli *grpcClient) OnStop() { 121 cli.BaseService.OnStop() 122 123 if cli.conn != nil { 124 cli.conn.Close() 125 } 126 close(cli.chReqRes) 127 } 128 129 func (cli *grpcClient) StopForError(err error) { 130 if !cli.IsRunning() { 131 return 132 } 133 134 cli.mtx.Lock() 135 if cli.err == nil { 136 cli.err = err 137 } 138 cli.mtx.Unlock() 139 140 cli.Logger.Error(fmt.Sprintf("Stopping abci.grpcClient for error: %v", err.Error())) 141 if err := cli.Stop(); err != nil { 142 cli.Logger.Error("Error stopping abci.grpcClient", "err", err) 143 } 144 } 145 146 func (cli *grpcClient) Error() error { 147 cli.mtx.Lock() 148 defer cli.mtx.Unlock() 149 return cli.err 150 } 151 152 // Set listener for all responses 153 // NOTE: callback may get internally generated flush responses. 154 func (cli *grpcClient) SetResponseCallback(resCb Callback) { 155 cli.mtx.Lock() 156 cli.resCb = resCb 157 cli.mtx.Unlock() 158 } 159 160 //---------------------------------------- 161 // GRPC calls are synchronous, but some callbacks expect to be called asynchronously 162 // (eg. the mempool expects to be able to lock to remove bad txs from cache). 163 // To accommodate, we finish each call in its own go-routine, 164 // which is expensive, but easy - if you want something better, use the socket protocol! 165 // maybe one day, if people really want it, we use grpc streams, 166 // but hopefully not :D 167 168 func (cli *grpcClient) EchoAsync(msg string) *ReqRes { 169 req := types.ToRequestEcho(msg) 170 res, err := cli.client.Echo(context.Background(), req.GetEcho(), grpc.WaitForReady(true)) 171 if err != nil { 172 cli.StopForError(err) 173 } 174 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Echo{Echo: res}}) 175 } 176 177 func (cli *grpcClient) FlushAsync() *ReqRes { 178 req := types.ToRequestFlush() 179 res, err := cli.client.Flush(context.Background(), req.GetFlush(), grpc.WaitForReady(true)) 180 if err != nil { 181 cli.StopForError(err) 182 } 183 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Flush{Flush: res}}) 184 } 185 186 func (cli *grpcClient) InfoAsync(params types.RequestInfo) *ReqRes { 187 req := types.ToRequestInfo(params) 188 res, err := cli.client.Info(context.Background(), req.GetInfo(), grpc.WaitForReady(true)) 189 if err != nil { 190 cli.StopForError(err) 191 } 192 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Info{Info: res}}) 193 } 194 195 func (cli *grpcClient) SetOptionAsync(params types.RequestSetOption) *ReqRes { 196 req := types.ToRequestSetOption(params) 197 res, err := cli.client.SetOption(context.Background(), req.GetSetOption(), grpc.WaitForReady(true)) 198 if err != nil { 199 cli.StopForError(err) 200 } 201 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_SetOption{SetOption: res}}) 202 } 203 204 func (cli *grpcClient) DeliverTxAsync(params types.RequestDeliverTx) *ReqRes { 205 req := types.ToRequestDeliverTx(params) 206 res, err := cli.client.DeliverTx(context.Background(), req.GetDeliverTx(), grpc.WaitForReady(true)) 207 if err != nil { 208 cli.StopForError(err) 209 } 210 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_DeliverTx{DeliverTx: res}}) 211 } 212 213 func (cli *grpcClient) CheckTxAsync(params types.RequestCheckTx) *ReqRes { 214 req := types.ToRequestCheckTx(params) 215 res, err := cli.client.CheckTx(context.Background(), req.GetCheckTx(), grpc.WaitForReady(true)) 216 if err != nil { 217 cli.StopForError(err) 218 } 219 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_CheckTx{CheckTx: res}}) 220 } 221 222 func (cli *grpcClient) QueryAsync(params types.RequestQuery) *ReqRes { 223 req := types.ToRequestQuery(params) 224 res, err := cli.client.Query(context.Background(), req.GetQuery(), grpc.WaitForReady(true)) 225 if err != nil { 226 cli.StopForError(err) 227 } 228 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Query{Query: res}}) 229 } 230 231 func (cli *grpcClient) CommitAsync() *ReqRes { 232 req := types.ToRequestCommit() 233 res, err := cli.client.Commit(context.Background(), req.GetCommit(), grpc.WaitForReady(true)) 234 if err != nil { 235 cli.StopForError(err) 236 } 237 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_Commit{Commit: res}}) 238 } 239 240 func (cli *grpcClient) InitChainAsync(params types.RequestInitChain) *ReqRes { 241 req := types.ToRequestInitChain(params) 242 res, err := cli.client.InitChain(context.Background(), req.GetInitChain(), grpc.WaitForReady(true)) 243 if err != nil { 244 cli.StopForError(err) 245 } 246 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_InitChain{InitChain: res}}) 247 } 248 249 func (cli *grpcClient) BeginBlockAsync(params types.RequestBeginBlock) *ReqRes { 250 req := types.ToRequestBeginBlock(params) 251 res, err := cli.client.BeginBlock(context.Background(), req.GetBeginBlock(), grpc.WaitForReady(true)) 252 if err != nil { 253 cli.StopForError(err) 254 } 255 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_BeginBlock{BeginBlock: res}}) 256 } 257 258 func (cli *grpcClient) EndBlockAsync(params types.RequestEndBlock) *ReqRes { 259 req := types.ToRequestEndBlock(params) 260 res, err := cli.client.EndBlock(context.Background(), req.GetEndBlock(), grpc.WaitForReady(true)) 261 if err != nil { 262 cli.StopForError(err) 263 } 264 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_EndBlock{EndBlock: res}}) 265 } 266 267 func (cli *grpcClient) ListSnapshotsAsync(params types.RequestListSnapshots) *ReqRes { 268 req := types.ToRequestListSnapshots(params) 269 res, err := cli.client.ListSnapshots(context.Background(), req.GetListSnapshots(), grpc.WaitForReady(true)) 270 if err != nil { 271 cli.StopForError(err) 272 } 273 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_ListSnapshots{ListSnapshots: res}}) 274 } 275 276 func (cli *grpcClient) OfferSnapshotAsync(params types.RequestOfferSnapshot) *ReqRes { 277 req := types.ToRequestOfferSnapshot(params) 278 res, err := cli.client.OfferSnapshot(context.Background(), req.GetOfferSnapshot(), grpc.WaitForReady(true)) 279 if err != nil { 280 cli.StopForError(err) 281 } 282 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_OfferSnapshot{OfferSnapshot: res}}) 283 } 284 285 func (cli *grpcClient) LoadSnapshotChunkAsync(params types.RequestLoadSnapshotChunk) *ReqRes { 286 req := types.ToRequestLoadSnapshotChunk(params) 287 res, err := cli.client.LoadSnapshotChunk(context.Background(), req.GetLoadSnapshotChunk(), grpc.WaitForReady(true)) 288 if err != nil { 289 cli.StopForError(err) 290 } 291 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_LoadSnapshotChunk{LoadSnapshotChunk: res}}) 292 } 293 294 func (cli *grpcClient) ApplySnapshotChunkAsync(params types.RequestApplySnapshotChunk) *ReqRes { 295 req := types.ToRequestApplySnapshotChunk(params) 296 res, err := cli.client.ApplySnapshotChunk(context.Background(), req.GetApplySnapshotChunk(), grpc.WaitForReady(true)) 297 if err != nil { 298 cli.StopForError(err) 299 } 300 return cli.finishAsyncCall(req, &types.Response{Value: &types.Response_ApplySnapshotChunk{ApplySnapshotChunk: res}}) 301 } 302 303 // finishAsyncCall creates a ReqRes for an async call, and immediately populates it 304 // with the response. We don't complete it until it's been ordered via the channel. 305 func (cli *grpcClient) finishAsyncCall(req *types.Request, res *types.Response) *ReqRes { 306 reqres := NewReqRes(req) 307 reqres.Response = res 308 cli.chReqRes <- reqres // use channel for async responses, since they must be ordered 309 return reqres 310 } 311 312 // finishSyncCall waits for an async call to complete. It is necessary to call all 313 // sync calls asynchronously as well, to maintain call and response ordering via 314 // the channel, and this method will wait until the async call completes. 315 func (cli *grpcClient) finishSyncCall(reqres *ReqRes) *types.Response { 316 // It's possible that the callback is called twice, since the callback can 317 // be called immediately on SetCallback() in addition to after it has been 318 // set. This is because completing the ReqRes happens in a separate critical 319 // section from the one where the callback is called: there is a race where 320 // SetCallback() is called between completing the ReqRes and dispatching the 321 // callback. 322 // 323 // We also buffer the channel with 1 response, since SetCallback() will be 324 // called synchronously if the reqres is already completed, in which case 325 // it will block on sending to the channel since it hasn't gotten around to 326 // receiving from it yet. 327 // 328 // ReqRes should really handle callback dispatch internally, to guarantee 329 // that it's only called once and avoid the above race conditions. 330 var once sync.Once 331 ch := make(chan *types.Response, 1) 332 reqres.SetCallback(func(res *types.Response) { 333 once.Do(func() { 334 ch <- res 335 }) 336 }) 337 return <-ch 338 } 339 340 //---------------------------------------- 341 342 func (cli *grpcClient) FlushSync() error { 343 reqres := cli.FlushAsync() 344 cli.finishSyncCall(reqres).GetFlush() 345 return cli.Error() 346 } 347 348 func (cli *grpcClient) EchoSync(msg string) (*types.ResponseEcho, error) { 349 reqres := cli.EchoAsync(msg) 350 // StopForError should already have been called if error is set 351 return cli.finishSyncCall(reqres).GetEcho(), cli.Error() 352 } 353 354 func (cli *grpcClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { 355 reqres := cli.InfoAsync(req) 356 return cli.finishSyncCall(reqres).GetInfo(), cli.Error() 357 } 358 359 func (cli *grpcClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) { 360 reqres := cli.SetOptionAsync(req) 361 return reqres.Response.GetSetOption(), cli.Error() 362 } 363 364 func (cli *grpcClient) DeliverTxSync(params types.RequestDeliverTx) (*types.ResponseDeliverTx, error) { 365 reqres := cli.DeliverTxAsync(params) 366 return cli.finishSyncCall(reqres).GetDeliverTx(), cli.Error() 367 } 368 369 func (cli *grpcClient) CheckTxSync(params types.RequestCheckTx) (*types.ResponseCheckTx, error) { 370 reqres := cli.CheckTxAsync(params) 371 return cli.finishSyncCall(reqres).GetCheckTx(), cli.Error() 372 } 373 374 func (cli *grpcClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) { 375 reqres := cli.QueryAsync(req) 376 return cli.finishSyncCall(reqres).GetQuery(), cli.Error() 377 } 378 379 func (cli *grpcClient) CommitSync() (*types.ResponseCommit, error) { 380 reqres := cli.CommitAsync() 381 return cli.finishSyncCall(reqres).GetCommit(), cli.Error() 382 } 383 384 func (cli *grpcClient) InitChainSync(params types.RequestInitChain) (*types.ResponseInitChain, error) { 385 reqres := cli.InitChainAsync(params) 386 return cli.finishSyncCall(reqres).GetInitChain(), cli.Error() 387 } 388 389 func (cli *grpcClient) BeginBlockSync(params types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { 390 reqres := cli.BeginBlockAsync(params) 391 return cli.finishSyncCall(reqres).GetBeginBlock(), cli.Error() 392 } 393 394 func (cli *grpcClient) EndBlockSync(params types.RequestEndBlock) (*types.ResponseEndBlock, error) { 395 reqres := cli.EndBlockAsync(params) 396 return cli.finishSyncCall(reqres).GetEndBlock(), cli.Error() 397 } 398 399 func (cli *grpcClient) ListSnapshotsSync(params types.RequestListSnapshots) (*types.ResponseListSnapshots, error) { 400 reqres := cli.ListSnapshotsAsync(params) 401 return cli.finishSyncCall(reqres).GetListSnapshots(), cli.Error() 402 } 403 404 func (cli *grpcClient) OfferSnapshotSync(params types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) { 405 reqres := cli.OfferSnapshotAsync(params) 406 return cli.finishSyncCall(reqres).GetOfferSnapshot(), cli.Error() 407 } 408 409 func (cli *grpcClient) LoadSnapshotChunkSync( 410 params types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) { 411 reqres := cli.LoadSnapshotChunkAsync(params) 412 return cli.finishSyncCall(reqres).GetLoadSnapshotChunk(), cli.Error() 413 } 414 415 func (cli *grpcClient) ApplySnapshotChunkSync( 416 params types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) { 417 reqres := cli.ApplySnapshotChunkAsync(params) 418 return cli.finishSyncCall(reqres).GetApplySnapshotChunk(), cli.Error() 419 }