github.com/mydexchain/tendermint@v0.0.4/abci/client/socket_client.go (about) 1 package abcicli 2 3 import ( 4 "bufio" 5 "container/list" 6 "errors" 7 "fmt" 8 "io" 9 "net" 10 "reflect" 11 "time" 12 13 "github.com/mydexchain/tendermint/abci/types" 14 tmnet "github.com/mydexchain/tendermint/libs/net" 15 "github.com/mydexchain/tendermint/libs/service" 16 tmsync "github.com/mydexchain/tendermint/libs/sync" 17 "github.com/mydexchain/tendermint/libs/timer" 18 ) 19 20 const ( 21 reqQueueSize = 256 // TODO make configurable 22 flushThrottleMS = 20 // Don't wait longer than... 23 ) 24 25 // This is goroutine-safe, but users should beware that the application in 26 // general is not meant to be interfaced with concurrent callers. 27 type socketClient struct { 28 service.BaseService 29 30 addr string 31 mustConnect bool 32 conn net.Conn 33 34 reqQueue chan *ReqRes 35 flushTimer *timer.ThrottleTimer 36 37 mtx tmsync.Mutex 38 err error 39 reqSent *list.List // list of requests sent, waiting for response 40 resCb func(*types.Request, *types.Response) // called on all requests, if set. 41 } 42 43 var _ Client = (*socketClient)(nil) 44 45 // NewSocketClient creates a new socket client, which connects to a given 46 // address. If mustConnect is true, the client will return an error upon start 47 // if it fails to connect. 48 func NewSocketClient(addr string, mustConnect bool) Client { 49 cli := &socketClient{ 50 reqQueue: make(chan *ReqRes, reqQueueSize), 51 flushTimer: timer.NewThrottleTimer("socketClient", flushThrottleMS), 52 mustConnect: mustConnect, 53 54 addr: addr, 55 reqSent: list.New(), 56 resCb: nil, 57 } 58 cli.BaseService = *service.NewBaseService(nil, "socketClient", cli) 59 return cli 60 } 61 62 // OnStart implements Service by connecting to the server and spawning reading 63 // and writing goroutines. 64 func (cli *socketClient) OnStart() error { 65 var ( 66 err error 67 conn net.Conn 68 ) 69 70 for { 71 conn, err = tmnet.Connect(cli.addr) 72 if err != nil { 73 if cli.mustConnect { 74 return err 75 } 76 cli.Logger.Error(fmt.Sprintf("abci.socketClient failed to connect to %v. Retrying after %vs...", 77 cli.addr, dialRetryIntervalSeconds), "err", err) 78 time.Sleep(time.Second * dialRetryIntervalSeconds) 79 continue 80 } 81 cli.conn = conn 82 83 go cli.sendRequestsRoutine(conn) 84 go cli.recvResponseRoutine(conn) 85 86 return nil 87 } 88 } 89 90 // OnStop implements Service by closing connection and flushing all queues. 91 func (cli *socketClient) OnStop() { 92 if cli.conn != nil { 93 cli.conn.Close() 94 } 95 96 cli.flushQueue() 97 cli.flushTimer.Stop() 98 } 99 100 // Error returns an error if the client was stopped abruptly. 101 func (cli *socketClient) Error() error { 102 cli.mtx.Lock() 103 defer cli.mtx.Unlock() 104 return cli.err 105 } 106 107 // SetResponseCallback sets a callback, which will be executed for each 108 // non-error & non-empty response from the server. 109 // 110 // NOTE: callback may get internally generated flush responses. 111 func (cli *socketClient) SetResponseCallback(resCb Callback) { 112 cli.mtx.Lock() 113 cli.resCb = resCb 114 cli.mtx.Unlock() 115 } 116 117 //---------------------------------------- 118 119 func (cli *socketClient) sendRequestsRoutine(conn io.Writer) { 120 w := bufio.NewWriter(conn) 121 for { 122 select { 123 case reqres := <-cli.reqQueue: 124 // cli.Logger.Debug("Sent request", "requestType", reflect.TypeOf(reqres.Request), "request", reqres.Request) 125 126 cli.willSendReq(reqres) 127 err := types.WriteMessage(reqres.Request, w) 128 if err != nil { 129 cli.stopForError(fmt.Errorf("write to buffer: %w", err)) 130 return 131 } 132 133 // If it's a flush request, flush the current buffer. 134 if _, ok := reqres.Request.Value.(*types.Request_Flush); ok { 135 err = w.Flush() 136 if err != nil { 137 cli.stopForError(fmt.Errorf("flush buffer: %w", err)) 138 return 139 } 140 } 141 case <-cli.flushTimer.Ch: // flush queue 142 select { 143 case cli.reqQueue <- NewReqRes(types.ToRequestFlush()): 144 default: 145 // Probably will fill the buffer, or retry later. 146 } 147 case <-cli.Quit(): 148 return 149 } 150 } 151 } 152 153 func (cli *socketClient) recvResponseRoutine(conn io.Reader) { 154 r := bufio.NewReader(conn) 155 for { 156 var res = &types.Response{} 157 err := types.ReadMessage(r, res) 158 if err != nil { 159 cli.stopForError(fmt.Errorf("read message: %w", err)) 160 return 161 } 162 163 // cli.Logger.Debug("Received response", "responseType", reflect.TypeOf(res), "response", res) 164 165 switch r := res.Value.(type) { 166 case *types.Response_Exception: // app responded with error 167 // XXX After setting cli.err, release waiters (e.g. reqres.Done()) 168 cli.stopForError(errors.New(r.Exception.Error)) 169 return 170 default: 171 err := cli.didRecvResponse(res) 172 if err != nil { 173 cli.stopForError(err) 174 return 175 } 176 } 177 } 178 } 179 180 func (cli *socketClient) willSendReq(reqres *ReqRes) { 181 cli.mtx.Lock() 182 defer cli.mtx.Unlock() 183 cli.reqSent.PushBack(reqres) 184 } 185 186 func (cli *socketClient) didRecvResponse(res *types.Response) error { 187 cli.mtx.Lock() 188 defer cli.mtx.Unlock() 189 190 // Get the first ReqRes. 191 next := cli.reqSent.Front() 192 if next == nil { 193 return fmt.Errorf("unexpected %v when nothing expected", reflect.TypeOf(res.Value)) 194 } 195 196 reqres := next.Value.(*ReqRes) 197 if !resMatchesReq(reqres.Request, res) { 198 return fmt.Errorf("unexpected %v when response to %v expected", 199 reflect.TypeOf(res.Value), reflect.TypeOf(reqres.Request.Value)) 200 } 201 202 reqres.Response = res 203 reqres.Done() // release waiters 204 cli.reqSent.Remove(next) // pop first item from linked list 205 206 // Notify client listener if set (global callback). 207 if cli.resCb != nil { 208 cli.resCb(reqres.Request, res) 209 } 210 211 // Notify reqRes listener if set (request specific callback). 212 // 213 // NOTE: It is possible this callback isn't set on the reqres object. At this 214 // point, in which case it will be called after, when it is set. 215 reqres.InvokeCallback() 216 217 return nil 218 } 219 220 //---------------------------------------- 221 222 func (cli *socketClient) EchoAsync(msg string) *ReqRes { 223 return cli.queueRequest(types.ToRequestEcho(msg)) 224 } 225 226 func (cli *socketClient) FlushAsync() *ReqRes { 227 return cli.queueRequest(types.ToRequestFlush()) 228 } 229 230 func (cli *socketClient) InfoAsync(req types.RequestInfo) *ReqRes { 231 return cli.queueRequest(types.ToRequestInfo(req)) 232 } 233 234 func (cli *socketClient) SetOptionAsync(req types.RequestSetOption) *ReqRes { 235 return cli.queueRequest(types.ToRequestSetOption(req)) 236 } 237 238 func (cli *socketClient) DeliverTxAsync(req types.RequestDeliverTx) *ReqRes { 239 return cli.queueRequest(types.ToRequestDeliverTx(req)) 240 } 241 242 func (cli *socketClient) CheckTxAsync(req types.RequestCheckTx) *ReqRes { 243 return cli.queueRequest(types.ToRequestCheckTx(req)) 244 } 245 246 func (cli *socketClient) QueryAsync(req types.RequestQuery) *ReqRes { 247 return cli.queueRequest(types.ToRequestQuery(req)) 248 } 249 250 func (cli *socketClient) CommitAsync() *ReqRes { 251 return cli.queueRequest(types.ToRequestCommit()) 252 } 253 254 func (cli *socketClient) InitChainAsync(req types.RequestInitChain) *ReqRes { 255 return cli.queueRequest(types.ToRequestInitChain(req)) 256 } 257 258 func (cli *socketClient) BeginBlockAsync(req types.RequestBeginBlock) *ReqRes { 259 return cli.queueRequest(types.ToRequestBeginBlock(req)) 260 } 261 262 func (cli *socketClient) EndBlockAsync(req types.RequestEndBlock) *ReqRes { 263 return cli.queueRequest(types.ToRequestEndBlock(req)) 264 } 265 266 func (cli *socketClient) ListSnapshotsAsync(req types.RequestListSnapshots) *ReqRes { 267 return cli.queueRequest(types.ToRequestListSnapshots(req)) 268 } 269 270 func (cli *socketClient) OfferSnapshotAsync(req types.RequestOfferSnapshot) *ReqRes { 271 return cli.queueRequest(types.ToRequestOfferSnapshot(req)) 272 } 273 274 func (cli *socketClient) LoadSnapshotChunkAsync(req types.RequestLoadSnapshotChunk) *ReqRes { 275 return cli.queueRequest(types.ToRequestLoadSnapshotChunk(req)) 276 } 277 278 func (cli *socketClient) ApplySnapshotChunkAsync(req types.RequestApplySnapshotChunk) *ReqRes { 279 return cli.queueRequest(types.ToRequestApplySnapshotChunk(req)) 280 } 281 282 //---------------------------------------- 283 284 func (cli *socketClient) FlushSync() error { 285 reqRes := cli.queueRequest(types.ToRequestFlush()) 286 if err := cli.Error(); err != nil { 287 return err 288 } 289 reqRes.Wait() // NOTE: if we don't flush the queue, its possible to get stuck here 290 return cli.Error() 291 } 292 293 func (cli *socketClient) EchoSync(msg string) (*types.ResponseEcho, error) { 294 reqres := cli.queueRequest(types.ToRequestEcho(msg)) 295 if err := cli.FlushSync(); err != nil { 296 return nil, err 297 } 298 299 return reqres.Response.GetEcho(), cli.Error() 300 } 301 302 func (cli *socketClient) InfoSync(req types.RequestInfo) (*types.ResponseInfo, error) { 303 reqres := cli.queueRequest(types.ToRequestInfo(req)) 304 if err := cli.FlushSync(); err != nil { 305 return nil, err 306 } 307 308 return reqres.Response.GetInfo(), cli.Error() 309 } 310 311 func (cli *socketClient) SetOptionSync(req types.RequestSetOption) (*types.ResponseSetOption, error) { 312 reqres := cli.queueRequest(types.ToRequestSetOption(req)) 313 if err := cli.FlushSync(); err != nil { 314 return nil, err 315 } 316 317 return reqres.Response.GetSetOption(), cli.Error() 318 } 319 320 func (cli *socketClient) DeliverTxSync(req types.RequestDeliverTx) (*types.ResponseDeliverTx, error) { 321 reqres := cli.queueRequest(types.ToRequestDeliverTx(req)) 322 if err := cli.FlushSync(); err != nil { 323 return nil, err 324 } 325 326 return reqres.Response.GetDeliverTx(), cli.Error() 327 } 328 329 func (cli *socketClient) CheckTxSync(req types.RequestCheckTx) (*types.ResponseCheckTx, error) { 330 reqres := cli.queueRequest(types.ToRequestCheckTx(req)) 331 if err := cli.FlushSync(); err != nil { 332 return nil, err 333 } 334 335 return reqres.Response.GetCheckTx(), cli.Error() 336 } 337 338 func (cli *socketClient) QuerySync(req types.RequestQuery) (*types.ResponseQuery, error) { 339 reqres := cli.queueRequest(types.ToRequestQuery(req)) 340 if err := cli.FlushSync(); err != nil { 341 return nil, err 342 } 343 344 return reqres.Response.GetQuery(), cli.Error() 345 } 346 347 func (cli *socketClient) CommitSync() (*types.ResponseCommit, error) { 348 reqres := cli.queueRequest(types.ToRequestCommit()) 349 if err := cli.FlushSync(); err != nil { 350 return nil, err 351 } 352 353 return reqres.Response.GetCommit(), cli.Error() 354 } 355 356 func (cli *socketClient) InitChainSync(req types.RequestInitChain) (*types.ResponseInitChain, error) { 357 reqres := cli.queueRequest(types.ToRequestInitChain(req)) 358 if err := cli.FlushSync(); err != nil { 359 return nil, err 360 } 361 362 return reqres.Response.GetInitChain(), cli.Error() 363 } 364 365 func (cli *socketClient) BeginBlockSync(req types.RequestBeginBlock) (*types.ResponseBeginBlock, error) { 366 reqres := cli.queueRequest(types.ToRequestBeginBlock(req)) 367 if err := cli.FlushSync(); err != nil { 368 return nil, err 369 } 370 371 return reqres.Response.GetBeginBlock(), cli.Error() 372 } 373 374 func (cli *socketClient) EndBlockSync(req types.RequestEndBlock) (*types.ResponseEndBlock, error) { 375 reqres := cli.queueRequest(types.ToRequestEndBlock(req)) 376 if err := cli.FlushSync(); err != nil { 377 return nil, err 378 } 379 380 return reqres.Response.GetEndBlock(), cli.Error() 381 } 382 383 func (cli *socketClient) ListSnapshotsSync(req types.RequestListSnapshots) (*types.ResponseListSnapshots, error) { 384 reqres := cli.queueRequest(types.ToRequestListSnapshots(req)) 385 if err := cli.FlushSync(); err != nil { 386 return nil, err 387 } 388 389 return reqres.Response.GetListSnapshots(), cli.Error() 390 } 391 392 func (cli *socketClient) OfferSnapshotSync(req types.RequestOfferSnapshot) (*types.ResponseOfferSnapshot, error) { 393 reqres := cli.queueRequest(types.ToRequestOfferSnapshot(req)) 394 if err := cli.FlushSync(); err != nil { 395 return nil, err 396 } 397 398 return reqres.Response.GetOfferSnapshot(), cli.Error() 399 } 400 401 func (cli *socketClient) LoadSnapshotChunkSync( 402 req types.RequestLoadSnapshotChunk) (*types.ResponseLoadSnapshotChunk, error) { 403 reqres := cli.queueRequest(types.ToRequestLoadSnapshotChunk(req)) 404 if err := cli.FlushSync(); err != nil { 405 return nil, err 406 } 407 408 return reqres.Response.GetLoadSnapshotChunk(), cli.Error() 409 } 410 411 func (cli *socketClient) ApplySnapshotChunkSync( 412 req types.RequestApplySnapshotChunk) (*types.ResponseApplySnapshotChunk, error) { 413 reqres := cli.queueRequest(types.ToRequestApplySnapshotChunk(req)) 414 if err := cli.FlushSync(); err != nil { 415 return nil, err 416 } 417 return reqres.Response.GetApplySnapshotChunk(), cli.Error() 418 } 419 420 //---------------------------------------- 421 422 func (cli *socketClient) queueRequest(req *types.Request) *ReqRes { 423 reqres := NewReqRes(req) 424 425 // TODO: set cli.err if reqQueue times out 426 cli.reqQueue <- reqres 427 428 // Maybe auto-flush, or unset auto-flush 429 switch req.Value.(type) { 430 case *types.Request_Flush: 431 cli.flushTimer.Unset() 432 default: 433 cli.flushTimer.Set() 434 } 435 436 return reqres 437 } 438 439 func (cli *socketClient) flushQueue() { 440 cli.mtx.Lock() 441 defer cli.mtx.Unlock() 442 443 // mark all in-flight messages as resolved (they will get cli.Error()) 444 for req := cli.reqSent.Front(); req != nil; req = req.Next() { 445 reqres := req.Value.(*ReqRes) 446 reqres.Done() 447 } 448 449 // mark all queued messages as resolved 450 LOOP: 451 for { 452 select { 453 case reqres := <-cli.reqQueue: 454 reqres.Done() 455 default: 456 break LOOP 457 } 458 } 459 } 460 461 //---------------------------------------- 462 463 func resMatchesReq(req *types.Request, res *types.Response) (ok bool) { 464 switch req.Value.(type) { 465 case *types.Request_Echo: 466 _, ok = res.Value.(*types.Response_Echo) 467 case *types.Request_Flush: 468 _, ok = res.Value.(*types.Response_Flush) 469 case *types.Request_Info: 470 _, ok = res.Value.(*types.Response_Info) 471 case *types.Request_SetOption: 472 _, ok = res.Value.(*types.Response_SetOption) 473 case *types.Request_DeliverTx: 474 _, ok = res.Value.(*types.Response_DeliverTx) 475 case *types.Request_CheckTx: 476 _, ok = res.Value.(*types.Response_CheckTx) 477 case *types.Request_Commit: 478 _, ok = res.Value.(*types.Response_Commit) 479 case *types.Request_Query: 480 _, ok = res.Value.(*types.Response_Query) 481 case *types.Request_InitChain: 482 _, ok = res.Value.(*types.Response_InitChain) 483 case *types.Request_BeginBlock: 484 _, ok = res.Value.(*types.Response_BeginBlock) 485 case *types.Request_EndBlock: 486 _, ok = res.Value.(*types.Response_EndBlock) 487 case *types.Request_ApplySnapshotChunk: 488 _, ok = res.Value.(*types.Response_ApplySnapshotChunk) 489 case *types.Request_LoadSnapshotChunk: 490 _, ok = res.Value.(*types.Response_LoadSnapshotChunk) 491 case *types.Request_ListSnapshots: 492 _, ok = res.Value.(*types.Response_ListSnapshots) 493 case *types.Request_OfferSnapshot: 494 _, ok = res.Value.(*types.Response_OfferSnapshot) 495 } 496 return ok 497 } 498 499 func (cli *socketClient) stopForError(err error) { 500 if !cli.IsRunning() { 501 return 502 } 503 504 cli.mtx.Lock() 505 if cli.err == nil { 506 cli.err = err 507 } 508 cli.mtx.Unlock() 509 510 cli.Logger.Error(fmt.Sprintf("Stopping abci.socketClient for error: %v", err.Error())) 511 if err := cli.Stop(); err != nil { 512 cli.Logger.Error("Error stopping abci.socketClient", "err", err) 513 } 514 }