github.com/altipla-consulting/ravendb-go-client@v0.1.3/subscription_worker.go (about) 1 package ravendb 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "log" 7 "net" 8 "reflect" 9 "strconv" 10 "strings" 11 "sync" 12 "sync/atomic" 13 "time" 14 ) 15 16 var ( 17 // LogSubscriptions allows to monitor read/writes made by SubscriptionWorker to a tcp connection. For debugging. 18 LogSubscriptionWorker func(op string, d []byte) = func(op string, d []byte) { 19 // no-op 20 } 21 ) 22 23 // SubscriptionWorker describes subscription worker 24 type SubscriptionWorker struct { 25 clazz reflect.Type 26 revisions bool 27 logger *log.Logger 28 store *DocumentStore 29 dbName string 30 31 cancellationRequested int32 // atomic, > 0 means cancellation was requested 32 options *SubscriptionWorkerOptions 33 tcpClient atomic.Value // net.Conn 34 parser *json.Decoder 35 disposed int32 // atomic 36 // this channel is closed when worker 37 chDone chan struct{} 38 39 afterAcknowledgment []func(*SubscriptionBatch) 40 onSubscriptionConnectionRetry []func(error) 41 42 redirectNode *ServerNode 43 subscriptionLocalRequestExecutor *RequestExecutor 44 45 lastConnectionFailure time.Time 46 supportedFeatures *supportedFeatures 47 onClosed func(*SubscriptionWorker) 48 49 err atomic.Value // error 50 mu sync.Mutex 51 } 52 53 // Err returns a potential error, available after worker finished 54 func (w *SubscriptionWorker) Err() error { 55 if v := w.err.Load(); v == nil { 56 return nil 57 } else { 58 return v.(error) 59 } 60 } 61 62 func (w *SubscriptionWorker) isCancellationRequested() bool { 63 v := atomic.LoadInt32(&w.cancellationRequested) 64 return v > 0 65 } 66 67 // Cancel requests the worker to finish. It doesn't happen immediately. 68 // To check if the worker finished, use HasFinished 69 // To wait 70 func (w *SubscriptionWorker) Cancel() { 71 atomic.AddInt32(&w.cancellationRequested, 1) 72 // we might be reading from a connection, so break that loop 73 // by closing the connection 74 w.closeTcpClient() 75 } 76 77 // IsDone returns true if the worker has finished 78 func (w *SubscriptionWorker) IsDone() bool { 79 if w.chDone == nil { 80 // not started yet 81 return true 82 } 83 select { 84 case <-w.chDone: 85 return true 86 default: 87 return false 88 } 89 } 90 91 // WaitUntilFinished waits until worker finishes for up to a timeout and 92 // returns an error. 93 // If timeout is 0, it waits indefinitely. 94 func (w *SubscriptionWorker) WaitUntilFinished(timeout time.Duration) error { 95 if w.chDone == nil { 96 // not started yet 97 return newSubscriptionInvalidStateError("SubscriptionWorker has not yet been started with Run()") 98 } 99 100 if timeout == 0 { 101 <-w.chDone 102 return w.Err() 103 } 104 105 select { 106 case <-w.chDone: 107 // no-op, we're here if already finished (channel closed) 108 case <-time.After(timeout): 109 return NewTimeoutError("timed out waiting for subscription worker to finish") 110 } 111 return w.Err() 112 } 113 114 func (w *SubscriptionWorker) getTcpClient() net.Conn { 115 if conn := w.tcpClient.Load(); conn == nil { 116 return nil 117 } else { 118 return conn.(net.Conn) 119 } 120 } 121 122 func (w *SubscriptionWorker) isDisposed() bool { 123 v := atomic.LoadInt32(&w.disposed) 124 return v != 0 125 } 126 127 func (w *SubscriptionWorker) markDisposed() { 128 atomic.StoreInt32(&w.disposed, 1) 129 } 130 131 // AddAfterAcknowledgmentListener adds callback function that will be called after 132 // listener has been acknowledged. 133 // Returns id that can be used in RemoveAfterAcknowledgmentListener 134 func (w *SubscriptionWorker) AddAfterAcknowledgmentListener(handler func(*SubscriptionBatch)) int { 135 w.afterAcknowledgment = append(w.afterAcknowledgment, handler) 136 return len(w.afterAcknowledgment) - 1 137 } 138 139 // RemoveAfterAcknowledgmentListener removes a callback added with AddAfterAcknowledgmentListener 140 func (w *SubscriptionWorker) RemoveAfterAcknowledgmentListener(id int) { 141 w.afterAcknowledgment[id] = nil 142 } 143 144 // AddOnSubscriptionConnectionRetry adds a callback function that will be called 145 // when subscription connection is retried. 146 // Returns id that can be used in RemoveOnSubscriptionConnectionRetry 147 func (w *SubscriptionWorker) AddOnSubscriptionConnectionRetry(handler func(error)) int { 148 w.onSubscriptionConnectionRetry = append(w.onSubscriptionConnectionRetry, handler) 149 return len(w.onSubscriptionConnectionRetry) - 1 150 } 151 152 // RemoveOnSubscriptionConnectionRetry removes a callback added with AddOnSubscriptionConnectionRetry 153 func (w *SubscriptionWorker) RemoveOnSubscriptionConnectionRetry(id int) { 154 w.onSubscriptionConnectionRetry[id] = nil 155 } 156 157 // NewSubscriptionWorker returns new SubscriptionWorker 158 func NewSubscriptionWorker(clazz reflect.Type, options *SubscriptionWorkerOptions, withRevisions bool, documentStore *DocumentStore, dbName string) (*SubscriptionWorker, error) { 159 160 if options.SubscriptionName == "" { 161 return nil, newIllegalArgumentError("SubscriptionConnectionOptions must specify the subscriptionName") 162 } 163 164 if dbName == "" { 165 dbName = documentStore.GetDatabase() 166 } 167 168 res := &SubscriptionWorker{ 169 clazz: clazz, 170 options: options, 171 revisions: withRevisions, 172 store: documentStore, 173 dbName: dbName, 174 } 175 176 return res, nil 177 } 178 179 // Close closes a subscription 180 func (w *SubscriptionWorker) Close() error { 181 return w.close(true) 182 } 183 184 func (w *SubscriptionWorker) close(waitForSubscriptionTask bool) error { 185 if w.isDisposed() { 186 return nil 187 } 188 defer func() { 189 if w.onClosed != nil { 190 w.onClosed(w) 191 } 192 }() 193 w.markDisposed() 194 w.Cancel() 195 196 if waitForSubscriptionTask { 197 _ = w.WaitUntilFinished(0) 198 } 199 200 if w.subscriptionLocalRequestExecutor != nil { 201 w.subscriptionLocalRequestExecutor.Close() 202 } 203 return nil 204 } 205 206 func (w *SubscriptionWorker) Run(cb func(*SubscriptionBatch) error) error { 207 if w.chDone != nil { 208 return newIllegalStateError("The subscription is already running") 209 } 210 211 // unbuffered so that we can ack to the server that the user processed 212 // a batch 213 w.chDone = make(chan struct{}) 214 215 go func() { 216 w.runSubscriptionAsync(cb) 217 }() 218 return nil 219 } 220 221 func (w *SubscriptionWorker) getCurrentNodeTag() string { 222 if w.redirectNode != nil { 223 return w.redirectNode.ClusterTag 224 } 225 return "" 226 } 227 228 func (w *SubscriptionWorker) getSubscriptionName() string { 229 if w.options != nil { 230 return w.options.SubscriptionName 231 } 232 return "" 233 } 234 235 func (w *SubscriptionWorker) connectToServer() (net.Conn, error) { 236 command := NewGetTcpInfoCommand("Subscription/"+w.dbName, w.dbName) 237 requestExecutor := w.store.GetRequestExecutor(w.dbName) 238 239 var err error 240 if w.redirectNode != nil { 241 err = requestExecutor.Execute(w.redirectNode, -1, command, false, nil) 242 if err != nil { 243 w.redirectNode = nil 244 // if we failed to talk to a node, we'll forget about it and let the topology to 245 // redirect us to the current node 246 return nil, newRuntimeError(err.Error()) 247 } 248 } else { 249 if err = requestExecutor.ExecuteCommand(command, nil); err != nil { 250 return nil, err 251 } 252 } 253 254 uri := command.Result.URL 255 var serverCert []byte 256 if command.Result.Certificate != nil { 257 serverCert = []byte(*command.Result.Certificate) 258 } 259 cert := w.store.Certificate 260 tcpClient, err := tcpConnect(uri, serverCert, cert) 261 if err != nil { 262 msg := fmt.Sprintf("failed with %s", err) 263 LogSubscriptionWorker("connect", []byte(msg)) 264 return nil, err 265 } 266 LogSubscriptionWorker("connect", nil) 267 w.tcpClient.Store(tcpClient) 268 databaseName := w.dbName 269 if databaseName == "" { 270 databaseName = w.store.GetDatabase() 271 } 272 273 parameters := &tcpNegotiateParameters{} 274 parameters.database = databaseName 275 parameters.operation = operationSubscription 276 parameters.version = subscriptionTCPVersion 277 fn := func(s string) int { 278 n, _ := w.readServerResponseAndGetVersion(s) 279 return n 280 } 281 parameters.readResponseAndGetVersionCallback = fn 282 parameters.destinationNodeTag = w.getCurrentNodeTag() 283 parameters.destinationUrl = command.Result.URL 284 285 w.supportedFeatures, err = negotiateProtocolVersion(tcpClient, parameters) 286 if err != nil { 287 return nil, err 288 } 289 290 if w.supportedFeatures.protocolVersion <= 0 { 291 return nil, newIllegalStateError(w.options.SubscriptionName + " : TCP negotiation resulted with an invalid protocol version: " + strconv.Itoa(w.supportedFeatures.protocolVersion)) 292 } 293 294 options, err := jsonMarshal(w.options) 295 if err != nil { 296 return nil, err 297 } 298 299 _, err = tcpClient.Write(options) 300 if err != nil { 301 return nil, err 302 } 303 LogSubscriptionWorker("write", options) 304 if w.subscriptionLocalRequestExecutor != nil { 305 w.subscriptionLocalRequestExecutor.Close() 306 } 307 conv := w.store.GetConventions() 308 cert = requestExecutor.Certificate 309 trustStore := requestExecutor.TrustStore 310 uri = command.requestedNode.URL 311 w.subscriptionLocalRequestExecutor = RequestExecutorCreateForSingleNodeWithoutConfigurationUpdates(uri, w.dbName, cert, trustStore, conv) 312 return tcpClient, nil 313 } 314 315 func (w *SubscriptionWorker) ensureParser() { 316 if w.parser == nil { 317 w.parser = json.NewDecoder(w.getTcpClient()) 318 } 319 } 320 321 func (w *SubscriptionWorker) readServerResponseAndGetVersion(url string) (int, error) { 322 //Reading reply from server 323 w.ensureParser() 324 var reply *tcpConnectionHeaderResponse 325 err := w.parser.Decode(&reply) 326 if err != nil { 327 return 0, err 328 } 329 330 { 331 // approximate but better that nothing 332 d, _ := json.Marshal(reply) 333 LogSubscriptionWorker("read", d) 334 } 335 336 switch reply.Status { 337 case tcpConnectionStatusOk: 338 return reply.Version, nil 339 case tcpConnectionStatusAuthorizationFailed: 340 return 0, newAuthorizationError("Cannot access database " + w.dbName + " because " + reply.Message) 341 case tcpConnectionStatusTcpVersionMismatch: 342 if reply.Version != outOfRangeStatus { 343 return reply.Version, nil 344 } 345 // Kindly request the server to drop the connection 346 _ = w.sendDropMessage(reply) 347 return 0, newIllegalStateError("Can't connect to database " + w.dbName + " because: " + reply.Message) 348 } 349 350 return 0, newIllegalStateError("Unknown status '%s'", reply.Status) 351 } 352 353 func (w *SubscriptionWorker) sendDropMessage(reply *tcpConnectionHeaderResponse) error { 354 dropMsg := &tcpConnectionHeaderMessage{} 355 dropMsg.Operation = operationDrop 356 dropMsg.DatabaseName = w.dbName 357 dropMsg.OperationVersion = subscriptionTCPVersion 358 dropMsg.Info = "Couldn't agree on subscription tcp version ours: " + strconv.Itoa(subscriptionTCPVersion) + " theirs: " + strconv.Itoa(reply.Version) 359 header, err := jsonMarshal(dropMsg) 360 if err != nil { 361 return err 362 } 363 tcpClient := w.getTcpClient() 364 if _, err = tcpClient.Write(header); err != nil { 365 return err 366 } 367 LogSubscriptionWorker("write", header) 368 return nil 369 } 370 371 func (w *SubscriptionWorker) assertConnectionState(connectionStatus *subscriptionConnectionServerMessage) error { 372 //fmt.Printf("assertConnectionStatus: %v\n", connectionStatus) 373 if connectionStatus.Type == subscriptionServerMessageError { 374 if strings.Contains(connectionStatus.Exception, "DatabaseDoesNotExistException") { 375 return newDatabaseDoesNotExistError(w.dbName + " does not exists. " + connectionStatus.Message) 376 } 377 } 378 379 if connectionStatus.Type != subscriptionServerMessageConnectionStatus { 380 return newIllegalStateError("Server returned illegal type message when expecting connection status, was:" + connectionStatus.Type) 381 } 382 383 switch connectionStatus.Status { 384 case subscriptionConnectionStatusAccepted: 385 case subscriptionConnectionStatusInUse: 386 return newSubscriptionInUseError("Subscription with id " + w.options.SubscriptionName + " cannot be opened, because it's in use and the connection strategy is " + w.options.Strategy) 387 case subscriptionConnectionStatusClosed: 388 return newSubscriptionClosedError("Subscription with id " + w.options.SubscriptionName + " was closed. " + connectionStatus.Exception) 389 case subscriptionConnectionStatusInvalid: 390 return newSubscriptionInvalidStateError("Subscription with id " + w.options.SubscriptionName + " cannot be opened, because it is in invalid state. " + connectionStatus.Exception) 391 case subscriptionConnectionStatusNotFound: 392 return newSubscriptionDoesNotExistError("Subscription with id " + w.options.SubscriptionName + " cannot be opened, because it does not exist. " + connectionStatus.Exception) 393 case subscriptionConnectionStatusRedirect: 394 data := connectionStatus.Data 395 appropriateNode, _ := jsonGetAsText(data, "RedirectedTag") 396 err := newSubscriptionDoesNotBelongToNodeError("Subscription With id %s cannot be processed by current node, it will be redirected to %s", w.options.SubscriptionName, appropriateNode) 397 err.appropriateNode = appropriateNode 398 return err 399 case subscriptionConnectionStatusConcurrencyReconnect: 400 return newSubscriptionChangeVectorUpdateConcurrencyError(connectionStatus.Message) 401 default: 402 return newIllegalStateError("Subscription " + w.options.SubscriptionName + " could not be opened, reason: " + connectionStatus.Status) 403 } 404 return nil 405 } 406 407 func (w *SubscriptionWorker) processSubscriptionInner(cb func(batch *SubscriptionBatch) error) error { 408 if w.isCancellationRequested() { 409 return throwCancellationRequested() 410 } 411 412 socket, err := w.connectToServer() 413 if err != nil { 414 return err 415 } 416 417 defer func() { 418 _ = socket.Close() 419 }() 420 if w.isCancellationRequested() { 421 return throwCancellationRequested() 422 } 423 424 tcpClientCopy := w.getTcpClient() 425 426 connectionStatus, err := w.readNextObject() 427 if err != nil { 428 return err 429 } 430 431 if w.isCancellationRequested() { 432 return nil 433 } 434 435 if (connectionStatus.Type != subscriptionServerMessageConnectionStatus) || (connectionStatus.Status != subscriptionConnectionStatusAccepted) { 436 if err = w.assertConnectionState(connectionStatus); err != nil { 437 return err 438 } 439 } 440 441 w.lastConnectionFailure = time.Time{} 442 if w.isCancellationRequested() { 443 return nil 444 } 445 446 batch := newSubscriptionBatch(w.clazz, w.revisions, w.subscriptionLocalRequestExecutor, w.store, w.dbName, w.logger) 447 448 for !w.isCancellationRequested() { 449 incomingBatch, err := w.readSingleSubscriptionBatchFromServer(batch) 450 if err != nil { 451 return err 452 } 453 if w.isCancellationRequested() { 454 return throwCancellationRequested() 455 } 456 lastReceivedChangeVector, err := batch.initialize(incomingBatch) 457 if err != nil { 458 return err 459 } 460 461 // send a copy so that the client can safely access it 462 // only copy the fields needed in OpenSession 463 batchCopy := &SubscriptionBatch{ 464 Items: batch.Items, 465 store: batch.store, 466 requestExecutor: batch.requestExecutor, 467 dbName: batch.dbName, 468 } 469 470 err = cb(batchCopy) 471 if err != nil { 472 return err 473 } 474 475 if tcpClientCopy != nil { 476 err = w.sendAck(lastReceivedChangeVector, tcpClientCopy) 477 if err != nil && !w.options.IgnoreSubscriberErrors { 478 return err 479 } 480 } 481 } 482 return nil 483 } 484 485 func (w *SubscriptionWorker) processSubscription(cb func(batch *SubscriptionBatch) error) error { 486 err := w.processSubscriptionInner(cb) 487 if err == nil { 488 return nil 489 } 490 if _, ok := err.(*OperationCancelledError); ok { 491 if !w.isDisposed() { 492 return err 493 } 494 // otherwise this is thrown when shutting down, it 495 // isn't an error, so we don't need to treat 496 // it as such 497 return nil 498 } 499 500 return err 501 } 502 503 func (w *SubscriptionWorker) readSingleSubscriptionBatchFromServer(batch *SubscriptionBatch) ([]*subscriptionConnectionServerMessage, error) { 504 var incomingBatch []*subscriptionConnectionServerMessage 505 endOfBatch := false 506 507 for !endOfBatch && !w.isCancellationRequested() { 508 receivedMessage, err := w.readNextObject() 509 if err != nil { 510 return nil, err 511 } 512 513 if receivedMessage == nil || w.isCancellationRequested() { 514 break 515 } 516 517 switch receivedMessage.Type { 518 case subscriptionServerMessageData: 519 incomingBatch = append(incomingBatch, receivedMessage) 520 case subscriptionServerMessageEndOfBatch: 521 endOfBatch = true 522 case subscriptionServerMessageConfirm: 523 for _, cb := range w.afterAcknowledgment { 524 cb(batch) 525 } 526 incomingBatch = nil 527 //batch.Items = nil 528 case subscriptionServerMessageConnectionStatus: 529 if err = w.assertConnectionState(receivedMessage); err != nil { 530 return nil, err 531 } 532 case subscriptionServerMessageError: 533 return nil, throwSubscriptionError(receivedMessage) 534 default: 535 return nil, throwInvalidServerResponse(receivedMessage) 536 } 537 } 538 539 return incomingBatch, nil 540 } 541 542 func throwInvalidServerResponse(receivedMessage *subscriptionConnectionServerMessage) error { 543 return newIllegalArgumentError("Unrecognized message " + receivedMessage.Type + " type received from server") 544 } 545 546 func throwSubscriptionError(receivedMessage *subscriptionConnectionServerMessage) error { 547 exc := receivedMessage.Exception 548 if exc == "" { 549 exc = "None" 550 } 551 return newIllegalStateError("Connected terminated by server. Exception: " + exc) 552 } 553 554 func (w *SubscriptionWorker) readNextObject() (*subscriptionConnectionServerMessage, error) { 555 if w.isCancellationRequested() || w.isDisposed() { 556 return nil, nil 557 } 558 559 var res *subscriptionConnectionServerMessage 560 err := w.parser.Decode(&res) 561 if err == nil { 562 // approximate but better that nothing. would have to use pass-through reader to monitor the actual bytes 563 d, _ := json.Marshal(res) 564 LogSubscriptionWorker("read", d) 565 566 } 567 return res, err 568 } 569 570 func (w *SubscriptionWorker) sendAck(lastReceivedChangeVector string, networkStream net.Conn) error { 571 msg := &SubscriptionConnectionClientMessage{ 572 ChangeVector: &lastReceivedChangeVector, 573 Type: SubscriptionClientMessageAcknowledge, 574 } 575 ack, err := jsonMarshal(msg) 576 if err != nil { 577 return err 578 } 579 _, err = networkStream.Write(ack) 580 LogSubscriptionWorker("write", ack) 581 return err 582 } 583 584 func (w *SubscriptionWorker) runSubscriptionAsync(cb func(*SubscriptionBatch) error) { 585 586 //fmt.Printf("runSubscription(): %p started\n", w) 587 defer func() { 588 //fmt.Printf("runSubscriptionLoop() %p finished\n", w) 589 close(w.chDone) 590 }() 591 592 for !w.isCancellationRequested() { 593 w.closeTcpClient() 594 595 if w.logger != nil { 596 w.logger.Print("Subscription " + w.options.SubscriptionName + ". Connecting to server...") 597 } 598 599 //fmt.Printf("before w.processSubscription\n") 600 ex := w.processSubscription(cb) 601 //fmt.Printf("after w.processSubscription, ex: %v\n", ex) 602 if ex == nil { 603 continue 604 } 605 606 if w.isCancellationRequested() { 607 if !w.isDisposed() { 608 w.err.Store(ex) 609 return 610 } 611 } 612 shouldReconnect, err := w.shouldTryToReconnect(ex) 613 //fmt.Printf("shouldTryReconnect() returned err='%s'\n", err) 614 if err != nil || !shouldReconnect { 615 if err != nil { 616 w.err.Store(err) 617 } 618 return 619 } 620 time.Sleep(time.Duration(w.options.TimeToWaitBeforeConnectionRetry)) 621 for _, cb := range w.onSubscriptionConnectionRetry { 622 cb(ex) 623 } 624 } 625 } 626 627 func (w *SubscriptionWorker) assertLastConnectionFailure() error { 628 if w.lastConnectionFailure.IsZero() { 629 w.lastConnectionFailure = time.Now() 630 return nil 631 } 632 633 dur := time.Since(w.lastConnectionFailure) 634 635 if dur > time.Duration(w.options.MaxErroneousPeriod) { 636 return newSubscriptionInvalidStateError("Subscription connection was in invalid state for more than %s and therefore will be terminated", time.Duration(w.options.MaxErroneousPeriod)) 637 } 638 return nil 639 } 640 641 func (w *SubscriptionWorker) shouldTryToReconnect(ex error) (bool, error) { 642 //fmt.Printf("shouldTryToReconnect, ex type: %T, ex v: %v, ex str: %s\n", ex, ex, ex) 643 //ex = ExceptionsUtils.unwrapException(ex); 644 if w.isCancellationRequested() { 645 return false, nil 646 } 647 if se, ok := ex.(*SubscriptionDoesNotBelongToNodeError); ok { 648 if err := w.assertLastConnectionFailure(); err != nil { 649 return false, err 650 } 651 652 requestExecutor := w.store.GetRequestExecutor(w.dbName) 653 if se.appropriateNode == "" { 654 return true, nil 655 } 656 657 var nodeToRedirectTo *ServerNode 658 for _, x := range requestExecutor.GetTopologyNodes() { 659 if x.ClusterTag == se.appropriateNode { 660 nodeToRedirectTo = x 661 break 662 } 663 } 664 665 if nodeToRedirectTo == nil { 666 return false, newIllegalStateError("Could not redirect to " + se.appropriateNode + ", because it was not found in local topology, even after retrying") 667 } 668 669 w.redirectNode = nodeToRedirectTo 670 return true, nil 671 } 672 673 if _, ok := ex.(*SubscriptionChangeVectorUpdateConcurrencyError); ok { 674 return true, nil 675 } 676 677 _, ok1 := ex.(*SubscriptionInUseError) 678 _, ok2 := ex.(*SubscriptionDoesNotExistError) 679 _, ok3 := ex.(*SubscriptionClosedError) 680 _, ok4 := ex.(*SubscriptionInvalidStateError) 681 _, ok5 := ex.(*DatabaseDoesNotExistError) 682 _, ok6 := ex.(*AuthorizationError) 683 _, ok7 := ex.(*AllTopologyNodesDownError) 684 _, ok8 := ex.(*SubscriberErrorError) 685 if ok1 || ok2 || ok3 || ok4 || ok5 || ok6 || ok7 || ok8 { 686 w.Cancel() 687 return false, ex 688 } 689 690 if err := w.assertLastConnectionFailure(); err != nil { 691 return false, err 692 } 693 return true, nil 694 } 695 696 func (w *SubscriptionWorker) closeTcpClient() { 697 //w._parser = nil // Note: not necessary and causes data race 698 699 tcpClient := w.getTcpClient() 700 if tcpClient != nil { 701 _ = tcpClient.Close() 702 LogSubscriptionWorker("close", nil) 703 } 704 }