github.com/hechain20/hechain@v0.0.0-20220316014945-b544036ba106/core/ledger/kvledger/txmgmt/statedb/statecouchdb/couchdb.go (about) 1 /* 2 Copyright hechain. All Rights Reserved. 3 4 SPDX-License-Identifier: Apache-2.0 5 */ 6 7 package statecouchdb 8 9 import ( 10 "bytes" 11 "compress/gzip" 12 "context" 13 "encoding/base64" 14 "encoding/json" 15 "fmt" 16 "io" 17 "io/ioutil" 18 "log" 19 "mime" 20 "mime/multipart" 21 "net/http" 22 "net/http/httputil" 23 "net/textproto" 24 "net/url" 25 "regexp" 26 "strconv" 27 "strings" 28 "time" 29 "unicode/utf8" 30 31 "github.com/hechain20/hechain/common/flogging" 32 "github.com/hechain20/hechain/core/ledger" 33 "github.com/pkg/errors" 34 "go.uber.org/zap/zapcore" 35 ) 36 37 var couchdbLogger = flogging.MustGetLogger("couchdb") 38 39 // time between retry attempts in milliseconds 40 const retryWaitTime = 125 41 42 // dbInfo is body for database information. 43 type dbInfo struct { 44 DbName string `json:"db_name"` 45 Sizes struct { 46 File int `json:"file"` 47 External int `json:"external"` 48 Active int `json:"active"` 49 } `json:"sizes"` 50 Other struct { 51 DataSize int `json:"data_size"` 52 } `json:"other"` 53 DocDelCount int `json:"doc_del_count"` 54 DocCount int `json:"doc_count"` 55 DiskSize int `json:"disk_size"` 56 DiskFormatVersion int `json:"disk_format_version"` 57 DataSize int `json:"data_size"` 58 CompactRunning bool `json:"compact_running"` 59 InstanceStartTime string `json:"instance_start_time"` 60 } 61 62 // connectionInfo is a structure for capturing the database info and version 63 type connectionInfo struct { 64 Couchdb string `json:"couchdb"` 65 Version string `json:"version"` 66 Vendor struct { 67 Name string `json:"name"` 68 } `json:"vendor"` 69 } 70 71 // rangeQueryResponse is used for processing REST range query responses from CouchDB 72 type rangeQueryResponse struct { 73 TotalRows int32 `json:"total_rows"` 74 Offset int32 `json:"offset"` 75 Rows []struct { 76 ID string `json:"id"` 77 Key string `json:"key"` 78 Value struct { 79 Rev string `json:"rev"` 80 } `json:"value"` 81 Doc json.RawMessage `json:"doc"` 82 } `json:"rows"` 83 } 84 85 // queryResponse is used for processing REST query responses from CouchDB 86 type queryResponse struct { 87 Warning string `json:"warning"` 88 Docs []json.RawMessage `json:"docs"` 89 Bookmark string `json:"bookmark"` 90 } 91 92 // docMetadata is used for capturing CouchDB document header info, 93 // used to capture id, version, rev and attachments returned in the query from CouchDB 94 type docMetadata struct { 95 ID string `json:"_id"` 96 Rev string `json:"_rev"` 97 Version string `json:"~version"` 98 AttachmentsInfo map[string]*attachmentInfo `json:"_attachments"` 99 } 100 101 // queryResult is used for returning query results from CouchDB 102 type queryResult struct { 103 id string 104 value []byte 105 attachments []*attachmentInfo 106 } 107 108 // couchInstance represents a CouchDB instance 109 type couchInstance struct { 110 conf *ledger.CouchDBConfig 111 client *http.Client // a client to connect to this instance 112 stats *stats 113 } 114 115 // couchDatabase represents a database within a CouchDB instance 116 type couchDatabase struct { 117 couchInstance *couchInstance // connection configuration 118 dbName string 119 } 120 121 // dbReturn contains an error reported by CouchDB 122 type dbReturn struct { 123 StatusCode int `json:"status_code"` 124 Error string `json:"error"` 125 Reason string `json:"reason"` 126 } 127 128 // createIndexResponse contains an the index creation response from CouchDB 129 type createIndexResponse struct { 130 Result string `json:"result"` 131 ID string `json:"id"` 132 Name string `json:"name"` 133 } 134 135 // attachmentInfo contains the definition for an attached file for couchdb 136 type attachmentInfo struct { 137 Name string 138 ContentType string `json:"content_type"` 139 Length uint64 140 AttachmentBytes []byte `json:"data"` 141 } 142 143 func (a *attachmentInfo) len() int { 144 if a == nil { 145 return 0 146 } 147 return len(a.Name) + len(a.ContentType) + len(a.AttachmentBytes) 148 } 149 150 // fileDetails defines the structure needed to send an attachment to couchdb 151 type fileDetails struct { 152 Follows bool `json:"follows"` 153 ContentType string `json:"content_type"` 154 Length int `json:"length"` 155 } 156 157 // batchRetrieveDocMetadataResponse is used for processing REST batch responses from CouchDB 158 type batchRetrieveDocMetadataResponse struct { 159 Rows []struct { 160 ID string `json:"id"` 161 DocMetadata struct { 162 ID string `json:"_id"` 163 Rev string `json:"_rev"` 164 Version string `json:"~version"` 165 } `json:"doc"` 166 } `json:"rows"` 167 } 168 169 // batchUpdateResponse defines a structure for batch update response 170 type batchUpdateResponse struct { 171 ID string `json:"id"` 172 Error string `json:"error"` 173 Reason string `json:"reason"` 174 Ok bool `json:"ok"` 175 Rev string `json:"rev"` 176 } 177 178 // base64Attachment contains the definition for an attached file for couchdb 179 type base64Attachment struct { 180 ContentType string `json:"content_type"` 181 AttachmentData string `json:"data"` 182 } 183 184 // indexResult contains the definition for a couchdb index 185 type indexResult struct { 186 DesignDocument string `json:"designdoc"` 187 Name string `json:"name"` 188 Definition string `json:"definition"` 189 } 190 191 // databaseSecurity contains the definition for CouchDB database security 192 type databaseSecurity struct { 193 Admins struct { 194 Names []string `json:"names"` 195 Roles []string `json:"roles"` 196 } `json:"admins"` 197 Members struct { 198 Names []string `json:"names"` 199 Roles []string `json:"roles"` 200 } `json:"members"` 201 } 202 203 // couchDoc defines the structure for a JSON document value 204 type couchDoc struct { 205 jsonValue []byte 206 attachments []*attachmentInfo 207 } 208 209 func (d *couchDoc) key() (string, error) { 210 m := make(jsonValue) 211 if err := json.Unmarshal(d.jsonValue, &m); err != nil { 212 return "", err 213 } 214 return m[idField].(string), nil 215 } 216 217 func (d *couchDoc) len() int { 218 if d == nil { 219 return 0 220 } 221 size := len(d.jsonValue) 222 for _, a := range d.attachments { 223 size += a.len() 224 } 225 return size 226 } 227 228 // closeResponseBody discards the body and then closes it to enable returning it to 229 // connection pool 230 func closeResponseBody(resp *http.Response) { 231 if resp != nil { 232 io.Copy(ioutil.Discard, resp.Body) // discard whatever is remaining of body 233 resp.Body.Close() 234 } 235 } 236 237 // createDatabaseIfNotExist method provides function to create database 238 func (dbclient *couchDatabase) createDatabaseIfNotExist() error { 239 couchdbLogger.Debugf("[%s] Entering CreateDatabaseIfNotExist()", dbclient.dbName) 240 241 dbInfo, couchDBReturn, err := dbclient.getDatabaseInfo() 242 if err != nil { 243 if couchDBReturn == nil || couchDBReturn.StatusCode != 404 { 244 return err 245 } 246 } 247 248 if dbInfo == nil || couchDBReturn.StatusCode == 404 { 249 couchdbLogger.Debugf("[%s] Database does not exist.", dbclient.dbName) 250 251 connectURL, err := url.Parse(dbclient.couchInstance.url()) 252 if err != nil { 253 couchdbLogger.Errorf("URL parse error: %s", err) 254 return errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 255 } 256 257 // get the number of retries 258 maxRetries := dbclient.couchInstance.conf.MaxRetries 259 260 // process the URL with a PUT, creates the database 261 resp, _, err := dbclient.handleRequest(http.MethodPut, "CreateDatabaseIfNotExist", connectURL, nil, "", "", maxRetries, true, nil) 262 if err != nil { 263 // Check to see if the database exists 264 // Even though handleRequest() returned an error, the 265 // database may have been created and a false error 266 // returned due to a timeout or race condition. 267 // Do a final check to see if the database really got created. 268 dbInfo, couchDBReturn, dbInfoErr := dbclient.getDatabaseInfo() 269 if dbInfoErr != nil || dbInfo == nil || couchDBReturn.StatusCode == 404 { 270 return err 271 } 272 } 273 defer closeResponseBody(resp) 274 couchdbLogger.Infof("Created state database %s", dbclient.dbName) 275 } else { 276 couchdbLogger.Debugf("[%s] Database already exists", dbclient.dbName) 277 } 278 279 if dbclient.dbName != "_users" { 280 errSecurity := dbclient.applyDatabasePermissions() 281 if errSecurity != nil { 282 return errSecurity 283 } 284 } 285 286 couchdbLogger.Debugf("[%s] Exiting CreateDatabaseIfNotExist()", dbclient.dbName) 287 return nil 288 } 289 290 func (dbclient *couchDatabase) applyDatabasePermissions() error { 291 // If the username and password are not set, then skip applying permissions 292 if dbclient.couchInstance.conf.Username == "" && dbclient.couchInstance.conf.Password == "" { 293 return nil 294 } 295 296 securityPermissions := &databaseSecurity{} 297 298 securityPermissions.Admins.Names = append(securityPermissions.Admins.Names, dbclient.couchInstance.conf.Username) 299 securityPermissions.Members.Names = append(securityPermissions.Members.Names, dbclient.couchInstance.conf.Username) 300 301 err := dbclient.applyDatabaseSecurity(securityPermissions) 302 if err != nil { 303 return err 304 } 305 306 return nil 307 } 308 309 // getDatabaseInfo method provides function to retrieve database information 310 func (dbclient *couchDatabase) getDatabaseInfo() (*dbInfo, *dbReturn, error) { 311 connectURL, err := url.Parse(dbclient.couchInstance.url()) 312 if err != nil { 313 couchdbLogger.Errorf("URL parse error: %s", err) 314 return nil, nil, errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 315 } 316 317 // get the number of retries 318 maxRetries := dbclient.couchInstance.conf.MaxRetries 319 320 resp, couchDBReturn, err := dbclient.handleRequest(http.MethodGet, "GetDatabaseInfo", connectURL, nil, "", "", maxRetries, true, nil) 321 if err != nil { 322 return nil, couchDBReturn, err 323 } 324 defer closeResponseBody(resp) 325 326 dbResponse := &dbInfo{} 327 decodeErr := json.NewDecoder(resp.Body).Decode(&dbResponse) 328 if decodeErr != nil { 329 return nil, nil, errors.Wrap(decodeErr, "error decoding response body") 330 } 331 332 // trace the database info response 333 couchdbLogger.Debugw("GetDatabaseInfo()", "dbResponseJSON", dbResponse) 334 335 return dbResponse, couchDBReturn, nil 336 } 337 338 // verifyCouchConfig method provides function to verify the connection information 339 func (couchInstance *couchInstance) verifyCouchConfig() (*connectionInfo, *dbReturn, error) { 340 couchdbLogger.Debugf("Entering VerifyCouchConfig()") 341 defer couchdbLogger.Debugf("Exiting VerifyCouchConfig()") 342 343 connectURL, err := url.Parse(couchInstance.url()) 344 if err != nil { 345 couchdbLogger.Errorf("URL parse error: %s", err) 346 return nil, nil, errors.Wrapf(err, "error parsing couch instance URL: %s", couchInstance.url()) 347 } 348 connectURL.Path = "/" 349 350 // get the number of retries for startup 351 maxRetriesOnStartup := couchInstance.conf.MaxRetriesOnStartup 352 353 resp, couchDBReturn, err := couchInstance.handleRequest(context.Background(), http.MethodGet, "", "VerifyCouchConfig", connectURL, nil, 354 "", "", maxRetriesOnStartup, true, nil) 355 if err != nil { 356 return nil, couchDBReturn, errors.WithMessage(err, "unable to connect to CouchDB, check the hostname and port") 357 } 358 defer closeResponseBody(resp) 359 360 dbResponse := &connectionInfo{} 361 decodeErr := json.NewDecoder(resp.Body).Decode(&dbResponse) 362 if decodeErr != nil { 363 return nil, nil, errors.Wrap(decodeErr, "error decoding response body") 364 } 365 366 // trace the database info response 367 couchdbLogger.Debugw("VerifyConnection()", "dbResponseJSON", dbResponse) 368 369 // check to see if the system databases exist 370 // Verifying the existence of the system database accomplishes two steps 371 // 1. Ensures the system databases are created 372 // 2. Verifies the username password provided in the CouchDB config are valid for system admin 373 err = createSystemDatabasesIfNotExist(couchInstance) 374 if err != nil { 375 couchdbLogger.Errorf("Unable to connect to CouchDB, error: %s. Check the admin username and password.", err) 376 return nil, nil, errors.WithMessage(err, "unable to connect to CouchDB. Check the admin username and password") 377 } 378 379 return dbResponse, couchDBReturn, nil 380 } 381 382 // isEmpty returns false if couchInstance contains any databases 383 // (except couchdb system databases and any database name supplied in the parameter 'databasesToIgnore') 384 func (couchInstance *couchInstance) isEmpty(databasesToIgnore []string) (bool, error) { 385 toIgnore := map[string]bool{} 386 for _, s := range databasesToIgnore { 387 toIgnore[s] = true 388 } 389 applicationDBNames, err := couchInstance.retrieveApplicationDBNames() 390 if err != nil { 391 return false, err 392 } 393 for _, dbName := range applicationDBNames { 394 if !toIgnore[dbName] { 395 return false, nil 396 } 397 } 398 return true, nil 399 } 400 401 // retrieveApplicationDBNames returns all the application database names in the couch instance 402 func (couchInstance *couchInstance) retrieveApplicationDBNames() ([]string, error) { 403 connectURL, err := url.Parse(couchInstance.url()) 404 if err != nil { 405 couchdbLogger.Errorf("URL parse error: %s", err) 406 return nil, errors.Wrapf(err, "error parsing couch instance URL: %s", couchInstance.url()) 407 } 408 connectURL.Path = "/_all_dbs" 409 maxRetries := couchInstance.conf.MaxRetries 410 resp, _, err := couchInstance.handleRequest( 411 context.Background(), 412 http.MethodGet, 413 "", 414 "IsEmpty", 415 connectURL, 416 nil, 417 "", 418 "", 419 maxRetries, 420 true, 421 nil, 422 ) 423 if err != nil { 424 return nil, errors.WithMessage(err, "unable to connect to CouchDB, check the hostname and port") 425 } 426 427 var dbNames []string 428 defer closeResponseBody(resp) 429 if err := json.NewDecoder(resp.Body).Decode(&dbNames); err != nil { 430 return nil, errors.Wrap(err, "error decoding response body") 431 } 432 couchdbLogger.Debugf("dbNames = %s", dbNames) 433 applicationsDBNames := []string{} 434 for _, d := range dbNames { 435 if !isCouchSystemDBName(d) { 436 applicationsDBNames = append(applicationsDBNames, d) 437 } 438 } 439 return applicationsDBNames, nil 440 } 441 442 func isCouchSystemDBName(name string) bool { 443 return strings.HasPrefix(name, "_") 444 } 445 446 // healthCheck checks if the peer is able to communicate with CouchDB 447 func (couchInstance *couchInstance) healthCheck(ctx context.Context) error { 448 connectURL, err := url.Parse(couchInstance.url()) 449 if err != nil { 450 couchdbLogger.Errorf("URL parse error: %s", err) 451 return errors.Wrapf(err, "error parsing CouchDB URL: %s", couchInstance.url()) 452 } 453 _, _, err = couchInstance.handleRequest(ctx, http.MethodHead, "", "HealthCheck", connectURL, nil, "", "", 0, true, nil) 454 if err != nil { 455 return fmt.Errorf("failed to connect to couch db [%s]", err) 456 } 457 return nil 458 } 459 460 // internalQueryLimit returns the maximum number of records to return internally 461 // when querying CouchDB. 462 func (couchInstance *couchInstance) internalQueryLimit() int32 { 463 return int32(couchInstance.conf.InternalQueryLimit) 464 } 465 466 // maxBatchUpdateSize returns the maximum number of records to include in a 467 // bulk update operation. 468 func (couchInstance *couchInstance) maxBatchUpdateSize() int { 469 return couchInstance.conf.MaxBatchUpdateSize 470 } 471 472 // url returns the URL for the CouchDB instance. 473 func (couchInstance *couchInstance) url() string { 474 URL := &url.URL{ 475 Host: couchInstance.conf.Address, 476 Scheme: "http", 477 } 478 return URL.String() 479 } 480 481 // dropDatabase provides method to drop an existing database 482 func (dbclient *couchDatabase) dropDatabase() error { 483 dbName := dbclient.dbName 484 485 couchdbLogger.Debugf("[%s] Entering DropDatabase()", dbName) 486 487 connectURL, err := url.Parse(dbclient.couchInstance.url()) 488 if err != nil { 489 couchdbLogger.Errorf("URL parse error: %s", err) 490 return errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 491 } 492 493 // get the number of retries 494 maxRetries := dbclient.couchInstance.conf.MaxRetries 495 496 resp, couchdbReturn, err := dbclient.handleRequest(http.MethodDelete, "DropDatabase", connectURL, nil, "", "", maxRetries, true, nil) 497 defer closeResponseBody(resp) 498 if couchdbReturn != nil && couchdbReturn.StatusCode == 404 { 499 couchdbLogger.Debugf("[%s] Exiting DropDatabase(), database does not exist", dbclient.dbName) 500 return nil 501 } 502 if err != nil { 503 return err 504 } 505 506 couchdbLogger.Debugf("[%s] Exiting DropDatabase(), database dropped", dbclient.dbName) 507 return nil 508 } 509 510 // saveDoc method provides a function to save a document, id and byte array 511 func (dbclient *couchDatabase) saveDoc(id string, rev string, couchDoc *couchDoc) (string, error) { 512 dbName := dbclient.dbName 513 514 couchdbLogger.Debugf("[%s] Entering SaveDoc() id=[%s]", dbName, id) 515 516 if !utf8.ValidString(id) { 517 return "", errors.Errorf("doc id [%x] not a valid utf8 string", id) 518 } 519 520 saveURL, err := url.Parse(dbclient.couchInstance.url()) 521 if err != nil { 522 couchdbLogger.Errorf("URL parse error: %s", err) 523 return "", errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 524 } 525 526 // Set up a buffer for the data to be pushed to couchdb 527 var data []byte 528 529 // Set up a default boundary for use by multipart if sending attachments 530 defaultBoundary := "" 531 532 // Create a flag for shared connections. This is set to false for zero length attachments 533 keepConnectionOpen := true 534 535 // check to see if attachments is nil, if so, then this is a JSON only 536 if couchDoc.attachments == nil { 537 538 // Test to see if this is a valid JSON 539 if !isJSON(string(couchDoc.jsonValue)) { 540 return "", errors.New("JSON format is not valid") 541 } 542 543 // if there are no attachments, then use the bytes passed in as the JSON 544 data = couchDoc.jsonValue 545 546 } else { // there are attachments 547 548 // attachments are included, create the multipart definition 549 multipartData, multipartBoundary, err3 := createAttachmentPart(couchDoc) 550 if err3 != nil { 551 return "", err3 552 } 553 554 // If there is a zero length attachment, do not keep the connection open 555 for _, attach := range couchDoc.attachments { 556 if attach.Length < 1 { 557 keepConnectionOpen = false 558 } 559 } 560 561 // Set the data buffer to the data from the create multi-part data 562 data = multipartData.Bytes() 563 564 // Set the default boundary to the value generated in the multipart creation 565 defaultBoundary = multipartBoundary 566 567 } 568 569 // get the number of retries 570 maxRetries := dbclient.couchInstance.conf.MaxRetries 571 572 // handle the request for saving document with a retry if there is a revision conflict 573 resp, _, err := dbclient.handleRequestWithRevisionRetry(id, http.MethodPut, dbName, "SaveDoc", saveURL, data, rev, defaultBoundary, maxRetries, keepConnectionOpen, nil) 574 if err != nil { 575 return "", err 576 } 577 defer closeResponseBody(resp) 578 579 // get the revision and return 580 revision, err := getRevisionHeader(resp) 581 if err != nil { 582 return "", err 583 } 584 585 couchdbLogger.Debugf("[%s] Exiting SaveDoc()", dbclient.dbName) 586 587 return revision, nil 588 } 589 590 // getDocumentRevision will return the revision if the document exists, otherwise it will return "" 591 func (dbclient *couchDatabase) getDocumentRevision(id string) string { 592 rev := "" 593 594 // See if the document already exists, we need the rev for saves and deletes 595 _, revdoc, err := dbclient.readDoc(id) 596 if err == nil { 597 // set the revision to the rev returned from the document read 598 rev = revdoc 599 } 600 return rev 601 } 602 603 func createAttachmentPart(couchDoc *couchDoc) (bytes.Buffer, string, error) { 604 // Create a buffer for writing the result 605 writeBuffer := new(bytes.Buffer) 606 607 // read the attachment and save as an attachment 608 writer := multipart.NewWriter(writeBuffer) 609 610 // retrieve the boundary for the multipart 611 defaultBoundary := writer.Boundary() 612 613 fileAttachments := map[string]fileDetails{} 614 615 for _, attachment := range couchDoc.attachments { 616 fileAttachments[attachment.Name] = fileDetails{true, attachment.ContentType, len(attachment.AttachmentBytes)} 617 } 618 619 attachmentJSONMap := map[string]interface{}{ 620 "_attachments": fileAttachments, 621 } 622 623 // Add any data uploaded with the files 624 if couchDoc.jsonValue != nil { 625 626 // create a generic map 627 genericMap := make(map[string]interface{}) 628 629 // unmarshal the data into the generic map 630 decoder := json.NewDecoder(bytes.NewBuffer(couchDoc.jsonValue)) 631 decoder.UseNumber() 632 decodeErr := decoder.Decode(&genericMap) 633 if decodeErr != nil { 634 return *writeBuffer, "", errors.Wrap(decodeErr, "error decoding json data") 635 } 636 637 // add all key/values to the attachmentJSONMap 638 for jsonKey, jsonValue := range genericMap { 639 attachmentJSONMap[jsonKey] = jsonValue 640 } 641 642 } 643 644 filesForUpload, err := json.Marshal(attachmentJSONMap) 645 if err != nil { 646 return *writeBuffer, "", errors.Wrap(err, "error marshalling json data") 647 } 648 649 couchdbLogger.Debugf(string(filesForUpload)) 650 651 // create the header for the JSON 652 header := make(textproto.MIMEHeader) 653 header.Set("Content-Type", "application/json") 654 655 part, err := writer.CreatePart(header) 656 if err != nil { 657 return *writeBuffer, defaultBoundary, errors.Wrap(err, "error creating multipart") 658 } 659 660 part.Write(filesForUpload) 661 662 for _, attachment := range couchDoc.attachments { 663 664 header := make(textproto.MIMEHeader) 665 part, err2 := writer.CreatePart(header) 666 if err2 != nil { 667 return *writeBuffer, defaultBoundary, errors.Wrap(err2, "error creating multipart") 668 } 669 part.Write(attachment.AttachmentBytes) 670 671 } 672 673 err = writer.Close() 674 if err != nil { 675 return *writeBuffer, defaultBoundary, errors.Wrap(err, "error closing multipart writer") 676 } 677 678 return *writeBuffer, defaultBoundary, nil 679 } 680 681 func getRevisionHeader(resp *http.Response) (string, error) { 682 if resp == nil { 683 return "", errors.New("no response received from CouchDB") 684 } 685 686 revision := resp.Header.Get("Etag") 687 688 if revision == "" { 689 return "", errors.New("no revision tag detected") 690 } 691 692 reg := regexp.MustCompile(`"([^"]*)"`) 693 revisionNoQuotes := reg.ReplaceAllString(revision, "${1}") 694 return revisionNoQuotes, nil 695 } 696 697 // readDoc method provides function to retrieve a document and its revision 698 // from the database by id 699 func (dbclient *couchDatabase) readDoc(id string) (*couchDoc, string, error) { 700 var couchDoc couchDoc 701 attachments := []*attachmentInfo{} 702 dbName := dbclient.dbName 703 704 couchdbLogger.Debugf("[%s] Entering ReadDoc() id=[%s]", dbName, id) 705 defer couchdbLogger.Debugf("[%s] Exiting ReadDoc()", dbclient.dbName) 706 707 if !utf8.ValidString(id) { 708 return nil, "", errors.Errorf("doc id [%x] not a valid utf8 string", id) 709 } 710 711 readURL, err := url.Parse(dbclient.couchInstance.url()) 712 if err != nil { 713 couchdbLogger.Errorf("URL parse error: %s", err) 714 return nil, "", errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 715 } 716 717 query := readURL.Query() 718 query.Add("attachments", "true") 719 720 // get the number of retries 721 maxRetries := dbclient.couchInstance.conf.MaxRetries 722 723 resp, couchDBReturn, err := dbclient.handleRequest(http.MethodGet, "ReadDoc", readURL, nil, "", "", maxRetries, true, &query, id) 724 if err != nil { 725 if couchDBReturn != nil && couchDBReturn.StatusCode == 404 { 726 couchdbLogger.Debugf("[%s] Document not found (404), returning nil value instead of 404 error", dbclient.dbName) 727 // non-existent document should return nil value instead of a 404 error 728 // for details see https://github.com/hyperledger-archives/fabric/issues/936 729 return nil, "", nil 730 } 731 couchdbLogger.Debugf("[%s] couchDBReturn=%v\n", dbclient.dbName, couchDBReturn) 732 return nil, "", err 733 } 734 defer closeResponseBody(resp) 735 736 // Get the media type from the Content-Type header 737 mediaType, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type")) 738 if err != nil { 739 log.Fatal(err) 740 } 741 742 // Get the revision from header 743 revision, err := getRevisionHeader(resp) 744 if err != nil { 745 return nil, "", err 746 } 747 748 // Handle as JSON if multipart is NOT detected 749 if !strings.HasPrefix(mediaType, "multipart/") { 750 couchDoc.jsonValue, err = ioutil.ReadAll(resp.Body) 751 if err != nil { 752 return nil, "", errors.Wrap(err, "error reading response body") 753 } 754 755 return &couchDoc, revision, nil 756 } 757 758 // Handle as attachment. 759 // Set up the multipart reader based on the boundary. 760 multipartReader := multipart.NewReader(resp.Body, params["boundary"]) 761 for { 762 p, err := multipartReader.NextPart() 763 if err == io.EOF { 764 break // processed all parts 765 } 766 if err != nil { 767 return nil, "", errors.Wrap(err, "error reading next multipart") 768 } 769 770 defer p.Close() 771 772 couchdbLogger.Debugf("[%s] part header=%s", dbclient.dbName, p.Header) 773 774 if p.Header.Get("Content-Type") == "application/json" { 775 partdata, err := ioutil.ReadAll(p) 776 if err != nil { 777 return nil, "", errors.Wrap(err, "error reading multipart data") 778 } 779 couchDoc.jsonValue = partdata 780 continue 781 } 782 783 // Create an attachment structure and load it. 784 attachment := &attachmentInfo{} 785 attachment.ContentType = p.Header.Get("Content-Type") 786 contentDispositionParts := strings.Split(p.Header.Get("Content-Disposition"), ";") 787 788 if strings.TrimSpace(contentDispositionParts[0]) != "attachment" { 789 continue 790 } 791 792 switch p.Header.Get("Content-Encoding") { 793 case "gzip": // See if the part is gzip encoded 794 795 var respBody []byte 796 797 gr, err := gzip.NewReader(p) 798 if err != nil { 799 return nil, "", errors.Wrap(err, "error creating gzip reader") 800 } 801 respBody, err = ioutil.ReadAll(gr) 802 if err != nil { 803 return nil, "", errors.Wrap(err, "error reading gzip data") 804 } 805 806 couchdbLogger.Debugf("[%s] Retrieved attachment data", dbclient.dbName) 807 attachment.AttachmentBytes = respBody 808 attachment.Length = uint64(len(attachment.AttachmentBytes)) 809 attachment.Name = p.FileName() 810 attachments = append(attachments, attachment) 811 812 default: 813 814 // retrieve the data, this is not gzip 815 partdata, err := ioutil.ReadAll(p) 816 if err != nil { 817 return nil, "", errors.Wrap(err, "error reading multipart data") 818 } 819 couchdbLogger.Debugf("[%s] Retrieved attachment data", dbclient.dbName) 820 attachment.AttachmentBytes = partdata 821 attachment.Length = uint64(len(attachment.AttachmentBytes)) 822 attachment.Name = p.FileName() 823 attachments = append(attachments, attachment) 824 } 825 } 826 827 couchDoc.attachments = attachments 828 return &couchDoc, revision, nil 829 } 830 831 // readDocRange method provides function to a range of documents based on the start and end keys 832 // startKey and endKey can also be empty strings. If startKey and endKey are empty, all documents are returned 833 // This function provides a limit option to specify the max number of entries and is supplied by config. 834 // Skip is reserved for possible future future use. 835 func (dbclient *couchDatabase) readDocRange(startKey, endKey string, limit int32) ([]*queryResult, string, error) { 836 dbName := dbclient.dbName 837 couchdbLogger.Debugf("[%s] Entering ReadDocRange() startKey=%s, endKey=%s", dbName, startKey, endKey) 838 839 var results []*queryResult 840 841 rangeURL, err := url.Parse(dbclient.couchInstance.url()) 842 if err != nil { 843 couchdbLogger.Errorf("URL parse error: %s", err) 844 return nil, "", errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 845 } 846 847 queryParms := rangeURL.Query() 848 // Increment the limit by 1 to see if there are more qualifying records 849 queryParms.Set("limit", strconv.FormatInt(int64(limit+1), 10)) 850 queryParms.Add("include_docs", "true") 851 queryParms.Add("inclusive_end", "false") // endkey should be exclusive to be consistent with goleveldb 852 queryParms.Add("attachments", "true") // get the attachments as well 853 854 // Append the startKey if provided 855 if startKey != "" { 856 if startKey, err = encodeForJSON(startKey); err != nil { 857 return nil, "", err 858 } 859 queryParms.Add("startkey", "\""+startKey+"\"") 860 } 861 862 // Append the endKey if provided 863 if endKey != "" { 864 var err error 865 if endKey, err = encodeForJSON(endKey); err != nil { 866 return nil, "", err 867 } 868 queryParms.Add("endkey", "\""+endKey+"\"") 869 } 870 871 // get the number of retries 872 maxRetries := dbclient.couchInstance.conf.MaxRetries 873 874 resp, _, err := dbclient.handleRequest(http.MethodGet, "RangeDocRange", rangeURL, nil, "", "", maxRetries, true, &queryParms, "_all_docs") 875 if err != nil { 876 return nil, "", err 877 } 878 defer closeResponseBody(resp) 879 880 if couchdbLogger.IsEnabledFor(zapcore.DebugLevel) { 881 dump, err2 := httputil.DumpResponse(resp, false) 882 if err2 != nil { 883 log.Fatal(err2) 884 } 885 // compact debug log by replacing carriage return / line feed with dashes to separate http headers 886 couchdbLogger.Debugf("[%s] HTTP Response: %s", dbclient.dbName, bytes.Replace(dump, []byte{0x0d, 0x0a}, []byte{0x20, 0x7c, 0x20}, -1)) 887 } 888 889 // handle as JSON document 890 jsonResponseRaw, err := ioutil.ReadAll(resp.Body) 891 if err != nil { 892 return nil, "", errors.Wrap(err, "error reading response body") 893 } 894 895 jsonResponse := &rangeQueryResponse{} 896 err2 := json.Unmarshal(jsonResponseRaw, &jsonResponse) 897 if err2 != nil { 898 return nil, "", errors.Wrap(err2, "error unmarshalling json data") 899 } 900 901 // if an additional record is found, then reduce the count by 1 902 // and populate the nextStartKey 903 if jsonResponse.TotalRows > limit { 904 jsonResponse.TotalRows = limit 905 } 906 907 couchdbLogger.Debugf("[%s] Total Rows: %d", dbclient.dbName, jsonResponse.TotalRows) 908 909 // Use the next endKey as the starting default for the nextStartKey 910 nextStartKey := endKey 911 912 for index, row := range jsonResponse.Rows { 913 914 docMetadata := &docMetadata{} 915 err3 := json.Unmarshal(row.Doc, &docMetadata) 916 if err3 != nil { 917 return nil, "", errors.Wrap(err3, "error unmarshalling json data") 918 } 919 920 // if there is an extra row for the nextStartKey, then do not add the row to the result set 921 // and populate the nextStartKey variable 922 if int32(index) >= jsonResponse.TotalRows { 923 nextStartKey = docMetadata.ID 924 continue 925 } 926 927 if docMetadata.AttachmentsInfo != nil { 928 929 couchdbLogger.Debugf("[%s] Adding JSON document and attachments for id: %s", dbclient.dbName, docMetadata.ID) 930 931 attachments := []*attachmentInfo{} 932 for attachmentName, attachment := range docMetadata.AttachmentsInfo { 933 attachment.Name = attachmentName 934 935 attachments = append(attachments, attachment) 936 } 937 938 addDocument := &queryResult{docMetadata.ID, row.Doc, attachments} 939 results = append(results, addDocument) 940 941 } else { 942 943 couchdbLogger.Debugf("[%s] Adding json docment for id: %s", dbclient.dbName, docMetadata.ID) 944 945 addDocument := &queryResult{docMetadata.ID, row.Doc, nil} 946 results = append(results, addDocument) 947 948 } 949 950 } 951 952 couchdbLogger.Debugf("[%s] Exiting ReadDocRange()", dbclient.dbName) 953 954 return results, nextStartKey, nil 955 } 956 957 // deleteDoc method provides function to delete a document from the database by id 958 func (dbclient *couchDatabase) deleteDoc(id, rev string) error { 959 dbName := dbclient.dbName 960 961 couchdbLogger.Debugf("[%s] Entering DeleteDoc() id=%s", dbName, id) 962 963 deleteURL, err := url.Parse(dbclient.couchInstance.url()) 964 if err != nil { 965 couchdbLogger.Errorf("URL parse error: %s", err) 966 return errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 967 } 968 969 // get the number of retries 970 maxRetries := dbclient.couchInstance.conf.MaxRetries 971 972 // handle the request for saving document with a retry if there is a revision conflict 973 resp, couchDBReturn, err := dbclient.handleRequestWithRevisionRetry(id, http.MethodDelete, dbName, "DeleteDoc", 974 deleteURL, nil, "", "", maxRetries, true, nil) 975 if err != nil { 976 if couchDBReturn != nil && couchDBReturn.StatusCode == 404 { 977 couchdbLogger.Debugf("[%s] Document not found (404), returning nil value instead of 404 error", dbclient.dbName) 978 // non-existent document should return nil value instead of a 404 error 979 // for details see https://github.com/hyperledger-archives/fabric/issues/936 980 return nil 981 } 982 return err 983 } 984 defer closeResponseBody(resp) 985 986 couchdbLogger.Debugf("[%s] Exiting DeleteDoc()", dbclient.dbName) 987 988 return nil 989 } 990 991 // queryDocuments method provides function for processing a query 992 func (dbclient *couchDatabase) queryDocuments(query string) ([]*queryResult, string, error) { 993 dbName := dbclient.dbName 994 995 couchdbLogger.Debugf("[%s] Entering QueryDocuments() query=%s", dbName, query) 996 997 var results []*queryResult 998 999 queryURL, err := url.Parse(dbclient.couchInstance.url()) 1000 if err != nil { 1001 couchdbLogger.Errorf("URL parse error: %s", err) 1002 return nil, "", errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1003 } 1004 1005 // get the number of retries 1006 maxRetries := dbclient.couchInstance.conf.MaxRetries 1007 1008 resp, _, err := dbclient.handleRequest(http.MethodPost, "QueryDocuments", queryURL, []byte(query), "", "", maxRetries, true, nil, "_find") 1009 if err != nil { 1010 return nil, "", err 1011 } 1012 defer closeResponseBody(resp) 1013 1014 if couchdbLogger.IsEnabledFor(zapcore.DebugLevel) { 1015 dump, err2 := httputil.DumpResponse(resp, false) 1016 if err2 != nil { 1017 log.Fatal(err2) 1018 } 1019 // compact debug log by replacing carriage return / line feed with dashes to separate http headers 1020 couchdbLogger.Debugf("[%s] HTTP Response: %s", dbclient.dbName, bytes.Replace(dump, []byte{0x0d, 0x0a}, []byte{0x20, 0x7c, 0x20}, -1)) 1021 } 1022 1023 // handle as JSON document 1024 jsonResponseRaw, err := ioutil.ReadAll(resp.Body) 1025 if err != nil { 1026 return nil, "", errors.Wrap(err, "error reading response body") 1027 } 1028 1029 jsonResponse := &queryResponse{} 1030 1031 err2 := json.Unmarshal(jsonResponseRaw, &jsonResponse) 1032 if err2 != nil { 1033 return nil, "", errors.Wrap(err2, "error unmarshalling json data") 1034 } 1035 1036 if jsonResponse.Warning != "" { 1037 couchdbLogger.Warnf("The query [%s] caused the following warning: [%s]", query, jsonResponse.Warning) 1038 } 1039 1040 for _, row := range jsonResponse.Docs { 1041 1042 docMetadata := &docMetadata{} 1043 err3 := json.Unmarshal(row, &docMetadata) 1044 if err3 != nil { 1045 return nil, "", errors.Wrap(err3, "error unmarshalling json data") 1046 } 1047 1048 // JSON Query results never have attachments 1049 // The If block below will never be executed 1050 if docMetadata.AttachmentsInfo != nil { 1051 1052 couchdbLogger.Debugf("[%s] Adding JSON docment and attachments for id: %s", dbclient.dbName, docMetadata.ID) 1053 1054 couchDoc, _, err := dbclient.readDoc(docMetadata.ID) 1055 if err != nil { 1056 return nil, "", err 1057 } 1058 addDocument := &queryResult{id: docMetadata.ID, value: couchDoc.jsonValue, attachments: couchDoc.attachments} 1059 results = append(results, addDocument) 1060 1061 } else { 1062 couchdbLogger.Debugf("[%s] Adding json docment for id: %s", dbclient.dbName, docMetadata.ID) 1063 addDocument := &queryResult{id: docMetadata.ID, value: row, attachments: nil} 1064 1065 results = append(results, addDocument) 1066 1067 } 1068 } 1069 1070 couchdbLogger.Debugf("[%s] Exiting QueryDocuments()", dbclient.dbName) 1071 1072 return results, jsonResponse.Bookmark, nil 1073 } 1074 1075 // listIndex method lists the defined indexes for a database 1076 func (dbclient *couchDatabase) listIndex() ([]*indexResult, error) { 1077 // IndexDefinition contains the definition for a couchdb index 1078 type indexDefinition struct { 1079 DesignDocument string `json:"ddoc"` 1080 Name string `json:"name"` 1081 Type string `json:"type"` 1082 Definition json.RawMessage `json:"def"` 1083 } 1084 1085 // ListIndexResponse contains the definition for listing couchdb indexes 1086 type listIndexResponse struct { 1087 TotalRows int `json:"total_rows"` 1088 Indexes []indexDefinition `json:"indexes"` 1089 } 1090 1091 dbName := dbclient.dbName 1092 couchdbLogger.Debugf("[%s] Entering ListIndex()", dbName) 1093 1094 indexURL, err := url.Parse(dbclient.couchInstance.url()) 1095 if err != nil { 1096 couchdbLogger.Errorf("URL parse error: %s", err) 1097 return nil, errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1098 } 1099 1100 // get the number of retries 1101 maxRetries := dbclient.couchInstance.conf.MaxRetries 1102 1103 resp, _, err := dbclient.handleRequest(http.MethodGet, "ListIndex", indexURL, nil, "", "", maxRetries, true, nil, "_index") 1104 if err != nil { 1105 return nil, err 1106 } 1107 defer closeResponseBody(resp) 1108 1109 // handle as JSON document 1110 jsonResponseRaw, err := ioutil.ReadAll(resp.Body) 1111 if err != nil { 1112 return nil, errors.Wrap(err, "error reading response body") 1113 } 1114 1115 jsonResponse := &listIndexResponse{} 1116 1117 err2 := json.Unmarshal(jsonResponseRaw, jsonResponse) 1118 if err2 != nil { 1119 return nil, errors.Wrap(err2, "error unmarshalling json data") 1120 } 1121 1122 var results []*indexResult 1123 1124 for _, row := range jsonResponse.Indexes { 1125 1126 // if the DesignDocument does not begin with "_design/", then this is a system 1127 // level index and is not meaningful and cannot be edited or deleted 1128 designDoc := row.DesignDocument 1129 s := strings.SplitAfterN(designDoc, "_design/", 2) 1130 if len(s) > 1 { 1131 designDoc = s[1] 1132 1133 // Add the index definition to the results 1134 addIndexResult := &indexResult{DesignDocument: designDoc, Name: row.Name, Definition: fmt.Sprintf("%s", row.Definition)} 1135 results = append(results, addIndexResult) 1136 } 1137 1138 } 1139 1140 couchdbLogger.Debugf("[%s] Exiting ListIndex()", dbclient.dbName) 1141 1142 return results, nil 1143 } 1144 1145 // createIndex method provides a function creating an index 1146 func (dbclient *couchDatabase) createIndex(indexdefinition string) (*createIndexResponse, error) { 1147 dbName := dbclient.dbName 1148 1149 couchdbLogger.Debugf("[%s] Entering CreateIndex() indexdefinition=%s", dbName, indexdefinition) 1150 1151 // Test to see if this is a valid JSON 1152 if !isJSON(indexdefinition) { 1153 return nil, errors.New("JSON format is not valid") 1154 } 1155 1156 indexURL, err := url.Parse(dbclient.couchInstance.url()) 1157 if err != nil { 1158 couchdbLogger.Errorf("URL parse error: %s", err) 1159 return nil, errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1160 } 1161 1162 // get the number of retries 1163 maxRetries := dbclient.couchInstance.conf.MaxRetries 1164 1165 resp, _, err := dbclient.handleRequest(http.MethodPost, "CreateIndex", indexURL, []byte(indexdefinition), "", "", maxRetries, true, nil, "_index") 1166 if err != nil { 1167 return nil, err 1168 } 1169 defer closeResponseBody(resp) 1170 1171 if resp == nil { 1172 return nil, errors.New("invalid response received from CouchDB") 1173 } 1174 1175 // Read the response body 1176 respBody, err := ioutil.ReadAll(resp.Body) 1177 if err != nil { 1178 return nil, errors.Wrap(err, "error reading response body") 1179 } 1180 1181 couchDBReturn := &createIndexResponse{} 1182 1183 jsonBytes := []byte(respBody) 1184 1185 // unmarshal the response 1186 err = json.Unmarshal(jsonBytes, &couchDBReturn) 1187 if err != nil { 1188 return nil, errors.Wrap(err, "error unmarshalling json data") 1189 } 1190 1191 if couchDBReturn.Result == "created" { 1192 1193 couchdbLogger.Infof("Created CouchDB index [%s] in state database [%s] using design document [%s]", couchDBReturn.Name, dbclient.dbName, couchDBReturn.ID) 1194 1195 return couchDBReturn, nil 1196 1197 } 1198 1199 couchdbLogger.Infof("Updated CouchDB index [%s] in state database [%s] using design document [%s]", couchDBReturn.Name, dbclient.dbName, couchDBReturn.ID) 1200 1201 return couchDBReturn, nil 1202 } 1203 1204 // deleteIndex method provides a function deleting an index 1205 func (dbclient *couchDatabase) deleteIndex(designdoc, indexname string) error { 1206 dbName := dbclient.dbName 1207 1208 couchdbLogger.Debugf("[%s] Entering DeleteIndex() designdoc=%s indexname=%s", dbName, designdoc, indexname) 1209 1210 indexURL, err := url.Parse(dbclient.couchInstance.url()) 1211 if err != nil { 1212 couchdbLogger.Errorf("URL parse error: %s", err) 1213 return errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1214 } 1215 1216 // get the number of retries 1217 maxRetries := dbclient.couchInstance.conf.MaxRetries 1218 1219 resp, _, err := dbclient.handleRequest(http.MethodDelete, "DeleteIndex", indexURL, nil, "", "", maxRetries, true, nil, "_index", designdoc, "json", indexname) 1220 if err != nil { 1221 return err 1222 } 1223 defer closeResponseBody(resp) 1224 1225 return nil 1226 } 1227 1228 // getDatabaseSecurity method provides function to retrieve the security config for a database 1229 func (dbclient *couchDatabase) getDatabaseSecurity() (*databaseSecurity, error) { 1230 dbName := dbclient.dbName 1231 1232 couchdbLogger.Debugf("[%s] Entering GetDatabaseSecurity()", dbName) 1233 1234 securityURL, err := url.Parse(dbclient.couchInstance.url()) 1235 if err != nil { 1236 couchdbLogger.Errorf("URL parse error: %s", err) 1237 return nil, errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1238 } 1239 1240 // get the number of retries 1241 maxRetries := dbclient.couchInstance.conf.MaxRetries 1242 1243 resp, _, err := dbclient.handleRequest(http.MethodGet, "GetDatabaseSecurity", securityURL, nil, "", "", maxRetries, true, nil, "_security") 1244 if err != nil { 1245 return nil, err 1246 } 1247 defer closeResponseBody(resp) 1248 1249 // handle as JSON document 1250 jsonResponseRaw, err := ioutil.ReadAll(resp.Body) 1251 if err != nil { 1252 return nil, errors.Wrap(err, "error reading response body") 1253 } 1254 1255 jsonResponse := &databaseSecurity{} 1256 1257 err2 := json.Unmarshal(jsonResponseRaw, jsonResponse) 1258 if err2 != nil { 1259 return nil, errors.Wrap(err2, "error unmarshalling json data") 1260 } 1261 1262 couchdbLogger.Debugf("[%s] Exiting GetDatabaseSecurity()", dbclient.dbName) 1263 1264 return jsonResponse, nil 1265 } 1266 1267 // applyDatabaseSecurity method provides function to update the security config for a database 1268 func (dbclient *couchDatabase) applyDatabaseSecurity(databaseSecurity *databaseSecurity) error { 1269 dbName := dbclient.dbName 1270 1271 couchdbLogger.Debugf("[%s] Entering ApplyDatabaseSecurity()", dbName) 1272 1273 securityURL, err := url.Parse(dbclient.couchInstance.url()) 1274 if err != nil { 1275 couchdbLogger.Errorf("URL parse error: %s", err) 1276 return errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1277 } 1278 1279 // Ensure all of the arrays are initialized to empty arrays instead of nil 1280 if databaseSecurity.Admins.Names == nil { 1281 databaseSecurity.Admins.Names = make([]string, 0) 1282 } 1283 if databaseSecurity.Admins.Roles == nil { 1284 databaseSecurity.Admins.Roles = make([]string, 0) 1285 } 1286 if databaseSecurity.Members.Names == nil { 1287 databaseSecurity.Members.Names = make([]string, 0) 1288 } 1289 if databaseSecurity.Members.Roles == nil { 1290 databaseSecurity.Members.Roles = make([]string, 0) 1291 } 1292 1293 // get the number of retries 1294 maxRetries := dbclient.couchInstance.conf.MaxRetries 1295 1296 databaseSecurityJSON, err := json.Marshal(databaseSecurity) 1297 if err != nil { 1298 return errors.Wrap(err, "error unmarshalling json data") 1299 } 1300 1301 couchdbLogger.Debugf("[%s] Applying security to database: %s", dbclient.dbName, string(databaseSecurityJSON)) 1302 1303 resp, _, err := dbclient.handleRequest(http.MethodPut, "ApplyDatabaseSecurity", securityURL, databaseSecurityJSON, "", "", maxRetries, true, nil, "_security") 1304 if err != nil { 1305 return err 1306 } 1307 defer closeResponseBody(resp) 1308 1309 couchdbLogger.Debugf("[%s] Exiting ApplyDatabaseSecurity()", dbclient.dbName) 1310 1311 return nil 1312 } 1313 1314 // batchRetrieveDocumentMetadata - batch method to retrieve document metadata for a set of keys, 1315 // including ID, couchdb revision number, and ledger version 1316 func (dbclient *couchDatabase) batchRetrieveDocumentMetadata(keys []string) ([]*docMetadata, error) { 1317 couchdbLogger.Debugf("[%s] Entering BatchRetrieveDocumentMetadata() keys=%s", dbclient.dbName, keys) 1318 1319 batchRetrieveURL, err := url.Parse(dbclient.couchInstance.url()) 1320 if err != nil { 1321 couchdbLogger.Errorf("URL parse error: %s", err) 1322 return nil, errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1323 } 1324 1325 queryParms := batchRetrieveURL.Query() 1326 1327 // While BatchRetrieveDocumentMetadata() does not return the entire document, 1328 // for reads/writes, we do need to get document so that we can get the ledger version of the key. 1329 // TODO For blind writes we do not need to get the version, therefore when we bulk get 1330 // the revision numbers for the write keys that were not represented in read set 1331 // (the second time BatchRetrieveDocumentMetadata is called during block processing), 1332 // we could set include_docs to false to optimize the response. 1333 queryParms.Add("include_docs", "true") 1334 1335 keymap := make(map[string]interface{}) 1336 1337 keymap["keys"] = keys 1338 1339 jsonKeys, err := json.Marshal(keymap) 1340 if err != nil { 1341 return nil, errors.Wrap(err, "error marshalling json data") 1342 } 1343 1344 // get the number of retries 1345 maxRetries := dbclient.couchInstance.conf.MaxRetries 1346 1347 resp, _, err := dbclient.handleRequest(http.MethodPost, "BatchRetrieveDocumentMetadata", batchRetrieveURL, jsonKeys, "", "", maxRetries, true, &queryParms, "_all_docs") 1348 if err != nil { 1349 return nil, err 1350 } 1351 defer closeResponseBody(resp) 1352 1353 if couchdbLogger.IsEnabledFor(zapcore.DebugLevel) { 1354 dump, _ := httputil.DumpResponse(resp, false) 1355 // compact debug log by replacing carriage return / line feed with dashes to separate http headers 1356 couchdbLogger.Debugf("[%s] HTTP Response: %s", dbclient.dbName, bytes.Replace(dump, []byte{0x0d, 0x0a}, []byte{0x20, 0x7c, 0x20}, -1)) 1357 } 1358 1359 // handle as JSON document 1360 jsonResponseRaw, err := ioutil.ReadAll(resp.Body) 1361 if err != nil { 1362 return nil, errors.Wrap(err, "error reading response body") 1363 } 1364 1365 jsonResponse := &batchRetrieveDocMetadataResponse{} 1366 1367 err2 := json.Unmarshal(jsonResponseRaw, &jsonResponse) 1368 if err2 != nil { 1369 return nil, errors.Wrap(err2, "error unmarshalling json data") 1370 } 1371 1372 docMetadataArray := []*docMetadata{} 1373 1374 for _, row := range jsonResponse.Rows { 1375 docMetadata := &docMetadata{ID: row.ID, Rev: row.DocMetadata.Rev, Version: row.DocMetadata.Version} 1376 docMetadataArray = append(docMetadataArray, docMetadata) 1377 } 1378 1379 couchdbLogger.Debugf("[%s] Exiting BatchRetrieveDocumentMetadata()", dbclient.dbName) 1380 1381 return docMetadataArray, nil 1382 } 1383 1384 func (dbclient *couchDatabase) insertDocuments(docs []*couchDoc) error { 1385 responses, err := dbclient.batchUpdateDocuments(docs) 1386 if err != nil { 1387 return errors.WithMessage(err, "error while updating docs in bulk") 1388 } 1389 1390 for i, resp := range responses { 1391 if resp.Ok { 1392 continue 1393 } 1394 if _, err := dbclient.saveDoc(resp.ID, "", docs[i]); err != nil { 1395 return errors.WithMessagef(err, "error while storing doc with ID %s", resp.ID) 1396 } 1397 } 1398 return nil 1399 } 1400 1401 // batchUpdateDocuments - batch method to batch update documents 1402 func (dbclient *couchDatabase) batchUpdateDocuments(documents []*couchDoc) ([]*batchUpdateResponse, error) { 1403 dbName := dbclient.dbName 1404 1405 if couchdbLogger.IsEnabledFor(zapcore.DebugLevel) { 1406 documentIdsString, err := printDocumentIds(documents) 1407 if err == nil { 1408 couchdbLogger.Debugf("[%s] Entering BatchUpdateDocuments() document ids=[%s]", dbName, documentIdsString) 1409 } else { 1410 couchdbLogger.Debugf("[%s] Entering BatchUpdateDocuments() Could not print document ids due to error: %+v", dbName, err) 1411 } 1412 } 1413 1414 batchUpdateURL, err := url.Parse(dbclient.couchInstance.url()) 1415 if err != nil { 1416 couchdbLogger.Errorf("URL parse error: %s", err) 1417 return nil, errors.Wrapf(err, "error parsing CouchDB URL: %s", dbclient.couchInstance.url()) 1418 } 1419 1420 documentMap := make(map[string]interface{}) 1421 1422 var jsonDocumentMap []interface{} 1423 1424 for _, jsonDocument := range documents { 1425 1426 // create a document map 1427 document := make(map[string]interface{}) 1428 1429 // unmarshal the JSON component of the couchDoc into the document 1430 err = json.Unmarshal(jsonDocument.jsonValue, &document) 1431 if err != nil { 1432 return nil, errors.Wrap(err, "error unmarshalling json data") 1433 } 1434 1435 // iterate through any attachments 1436 if len(jsonDocument.attachments) > 0 { 1437 1438 // create a file attachment map 1439 fileAttachment := make(map[string]interface{}) 1440 1441 // for each attachment, create a base64Attachment, name the attachment, 1442 // add the content type and base64 encode the attachment 1443 for _, attachment := range jsonDocument.attachments { 1444 fileAttachment[attachment.Name] = base64Attachment{ 1445 attachment.ContentType, 1446 base64.StdEncoding.EncodeToString(attachment.AttachmentBytes), 1447 } 1448 } 1449 1450 // add attachments to the document 1451 document["_attachments"] = fileAttachment 1452 1453 } 1454 1455 // Append the document to the map of documents 1456 jsonDocumentMap = append(jsonDocumentMap, document) 1457 1458 } 1459 1460 // Add the documents to the "docs" item 1461 documentMap["docs"] = jsonDocumentMap 1462 1463 bulkDocsJSON, err := json.Marshal(documentMap) 1464 if err != nil { 1465 return nil, errors.Wrap(err, "error marshalling json data") 1466 } 1467 1468 // get the number of retries 1469 maxRetries := dbclient.couchInstance.conf.MaxRetries 1470 1471 resp, _, err := dbclient.handleRequest(http.MethodPost, "BatchUpdateDocuments", batchUpdateURL, bulkDocsJSON, "", "", maxRetries, true, nil, "_bulk_docs") 1472 if err != nil { 1473 return nil, err 1474 } 1475 defer closeResponseBody(resp) 1476 1477 if couchdbLogger.IsEnabledFor(zapcore.DebugLevel) { 1478 dump, _ := httputil.DumpResponse(resp, false) 1479 // compact debug log by replacing carriage return / line feed with dashes to separate http headers 1480 couchdbLogger.Debugf("[%s] HTTP Response: %s", dbclient.dbName, bytes.Replace(dump, []byte{0x0d, 0x0a}, []byte{0x20, 0x7c, 0x20}, -1)) 1481 } 1482 1483 // handle as JSON document 1484 jsonResponseRaw, err := ioutil.ReadAll(resp.Body) 1485 if err != nil { 1486 return nil, errors.Wrap(err, "error reading response body") 1487 } 1488 1489 jsonResponse := []*batchUpdateResponse{} 1490 err2 := json.Unmarshal(jsonResponseRaw, &jsonResponse) 1491 if err2 != nil { 1492 return nil, errors.Wrap(err2, "error unmarshalling json data") 1493 } 1494 1495 couchdbLogger.Debugf("[%s] Exiting BatchUpdateDocuments() _bulk_docs response=[%s]", dbclient.dbName, string(jsonResponseRaw)) 1496 1497 return jsonResponse, nil 1498 } 1499 1500 // handleRequestWithRevisionRetry method is a generic http request handler with 1501 // a retry for document revision conflict errors, 1502 // which may be detected during saves or deletes that timed out from client http perspective, 1503 // but which eventually succeeded in couchdb 1504 func (dbclient *couchDatabase) handleRequestWithRevisionRetry(id, method, dbName, functionName string, connectURL *url.URL, data []byte, rev string, 1505 multipartBoundary string, maxRetries int, keepConnectionOpen bool, queryParms *url.Values) (*http.Response, *dbReturn, error) { 1506 // Initialize a flag for the revision conflict 1507 revisionConflictDetected := false 1508 var resp *http.Response 1509 var couchDBReturn *dbReturn 1510 var errResp error 1511 1512 // attempt the http request for the max number of retries 1513 // In this case, the retry is to catch problems where a client timeout may miss a 1514 // successful CouchDB update and cause a document revision conflict on a retry in handleRequest 1515 for attempts := 0; attempts <= maxRetries; attempts++ { 1516 1517 // if the revision was not passed in, or if a revision conflict is detected on prior attempt, 1518 // query CouchDB for the document revision 1519 if rev == "" || revisionConflictDetected { 1520 rev = dbclient.getDocumentRevision(id) 1521 } 1522 1523 // handle the request for saving/deleting the couchdb data 1524 resp, couchDBReturn, errResp = dbclient.couchInstance.handleRequest(context.Background(), method, dbName, functionName, connectURL, 1525 data, rev, multipartBoundary, maxRetries, keepConnectionOpen, queryParms, id) 1526 1527 // If there was a 409 conflict error during the save/delete, log it and retry it. 1528 // Otherwise, break out of the retry loop 1529 if couchDBReturn != nil && couchDBReturn.StatusCode == 409 { 1530 couchdbLogger.Warningf("CouchDB document revision conflict detected, retrying. Attempt:%v", attempts+1) 1531 revisionConflictDetected = true 1532 } else { 1533 break 1534 } 1535 } 1536 1537 // return the handleRequest results 1538 return resp, couchDBReturn, errResp 1539 } 1540 1541 func (dbclient *couchDatabase) handleRequest(method, functionName string, connectURL *url.URL, data []byte, rev, multipartBoundary string, 1542 maxRetries int, keepConnectionOpen bool, queryParms *url.Values, pathElements ...string) (*http.Response, *dbReturn, error) { 1543 return dbclient.couchInstance.handleRequest(context.Background(), 1544 method, dbclient.dbName, functionName, connectURL, data, rev, multipartBoundary, 1545 maxRetries, keepConnectionOpen, queryParms, pathElements..., 1546 ) 1547 } 1548 1549 // handleRequest method is a generic http request handler. 1550 // If it returns an error, it ensures that the response body is closed, else it is the 1551 // callee's responsibility to close response correctly. 1552 // Any http error or CouchDB error (4XX or 500) will result in a golang error getting returned 1553 func (couchInstance *couchInstance) handleRequest(ctx context.Context, method, dbName, functionName string, connectURL *url.URL, data []byte, rev string, 1554 multipartBoundary string, maxRetries int, keepConnectionOpen bool, queryParms *url.Values, pathElements ...string) (*http.Response, *dbReturn, error) { 1555 couchdbLogger.Debugf("Entering handleRequest() method=%s url=%v dbName=%s", method, connectURL, dbName) 1556 1557 // create the return objects for couchDB 1558 var resp *http.Response 1559 var errResp error 1560 couchDBReturn := &dbReturn{} 1561 defer couchInstance.recordMetric(time.Now(), dbName, functionName, couchDBReturn) 1562 1563 // set initial wait duration for retries 1564 waitDuration := retryWaitTime * time.Millisecond 1565 1566 if maxRetries < 0 { 1567 return nil, nil, errors.New("number of retries must be zero or greater") 1568 } 1569 1570 requestURL := constructCouchDBUrl(connectURL, dbName, pathElements...) 1571 1572 if queryParms != nil { 1573 requestURL.RawQuery = queryParms.Encode() 1574 } 1575 1576 couchdbLogger.Debugf("Request URL: %s", requestURL) 1577 1578 // attempt the http request for the max number of retries 1579 // if maxRetries is 0, the database creation will be attempted once and will 1580 // return an error if unsuccessful 1581 // if maxRetries is 3 (default), a maximum of 4 attempts (one attempt with 3 retries) 1582 // will be made with warning entries for unsuccessful attempts 1583 for attempts := 0; attempts <= maxRetries; attempts++ { 1584 1585 // Set up a buffer for the payload data 1586 payloadData := new(bytes.Buffer) 1587 1588 payloadData.ReadFrom(bytes.NewReader(data)) 1589 1590 // Create request based on URL for couchdb operation 1591 req, err := http.NewRequestWithContext(ctx, method, requestURL.String(), payloadData) 1592 if err != nil { 1593 return nil, nil, errors.Wrap(err, "error creating http request") 1594 } 1595 1596 // set the request to close on completion if shared connections are not allowSharedConnection 1597 // Current CouchDB has a problem with zero length attachments, do not allow the connection to be reused. 1598 // Apache JIRA item for CouchDB https://issues.apache.org/jira/browse/COUCHDB-3394 1599 if !keepConnectionOpen { 1600 req.Close = true 1601 } 1602 1603 // add content header for PUT 1604 if method == http.MethodPut || method == http.MethodPost || method == http.MethodDelete { 1605 1606 // If the multipartBoundary is not set, then this is a JSON and content-type should be set 1607 // to application/json. Else, this is contains an attachment and needs to be multipart 1608 if multipartBoundary == "" { 1609 req.Header.Set("Content-Type", "application/json") 1610 } else { 1611 req.Header.Set("Content-Type", "multipart/related;boundary=\""+multipartBoundary+"\"") 1612 } 1613 1614 // check to see if the revision is set, if so, pass as a header 1615 if rev != "" { 1616 req.Header.Set("If-Match", rev) 1617 } 1618 } 1619 1620 // add content header for PUT 1621 if method == http.MethodPut || method == http.MethodPost { 1622 req.Header.Set("Accept", "application/json") 1623 } 1624 1625 // add content header for GET 1626 if method == http.MethodGet { 1627 req.Header.Set("Accept", "multipart/related") 1628 } 1629 1630 // If username and password are set the use basic auth 1631 if couchInstance.conf.Username != "" && couchInstance.conf.Password != "" { 1632 // req.Header.Set("Authorization", "Basic YWRtaW46YWRtaW5w") 1633 req.SetBasicAuth(couchInstance.conf.Username, couchInstance.conf.Password) 1634 } 1635 1636 // Execute http request 1637 resp, errResp = couchInstance.client.Do(req) 1638 1639 // check to see if the return from CouchDB is valid 1640 if invalidCouchDBReturn(resp, errResp) { 1641 continue 1642 } 1643 1644 // if there is no golang http error and no CouchDB 500 error, then drop out of the retry 1645 if errResp == nil && resp != nil && resp.StatusCode < 500 { 1646 // if this is an error, then populate the couchDBReturn 1647 if resp.StatusCode >= 400 { 1648 // Read the response body and close it for next attempt 1649 jsonError, err := ioutil.ReadAll(resp.Body) 1650 if err != nil { 1651 return nil, nil, errors.Wrap(err, "error reading response body") 1652 } 1653 defer closeResponseBody(resp) 1654 1655 errorBytes := []byte(jsonError) 1656 // Unmarshal the response 1657 err = json.Unmarshal(errorBytes, &couchDBReturn) 1658 if err != nil { 1659 return nil, nil, errors.Wrap(err, "error unmarshalling json data") 1660 } 1661 } 1662 1663 break 1664 } 1665 1666 // If the maxRetries is greater than 0, then log the retry info 1667 if maxRetries > 0 { 1668 1669 retryMessage := fmt.Sprintf("Retrying couchdb request in %s", waitDuration) 1670 if attempts == maxRetries { 1671 retryMessage = "Retries exhausted" 1672 } 1673 1674 // if this is an unexpected golang http error, log the error and retry 1675 if errResp != nil { 1676 // Log the error with the retry count and continue 1677 couchdbLogger.Warningf("Attempt %d of %d returned error: %s. %s", attempts+1, maxRetries+1, errResp.Error(), retryMessage) 1678 1679 // otherwise this is an unexpected 500 error from CouchDB. Log the error and retry. 1680 } else { 1681 // Read the response body and close it for next attempt 1682 jsonError, err := ioutil.ReadAll(resp.Body) 1683 defer closeResponseBody(resp) 1684 if err != nil { 1685 return nil, nil, errors.Wrap(err, "error reading response body") 1686 } 1687 1688 errorBytes := []byte(jsonError) 1689 // Unmarshal the response 1690 err = json.Unmarshal(errorBytes, &couchDBReturn) 1691 if err != nil { 1692 return nil, nil, errors.Wrap(err, "error unmarshalling json data") 1693 } 1694 1695 // Log the 500 error with the retry count and continue 1696 couchdbLogger.Warningf("Attempt %d of %d returned Couch DB Error:%s, Status Code:%v Reason:%s. %s", 1697 attempts+1, maxRetries+1, couchDBReturn.Error, resp.Status, couchDBReturn.Reason, retryMessage) 1698 1699 } 1700 // if there are more retries remaining, sleep for specified sleep time, then retry 1701 if attempts < maxRetries { 1702 time.Sleep(waitDuration) 1703 } 1704 1705 // backoff, doubling the retry time for next attempt 1706 waitDuration *= 2 1707 1708 } 1709 1710 } // end retry loop 1711 1712 // if a golang http error is still present after retries are exhausted, return the error 1713 if errResp != nil { 1714 return nil, couchDBReturn, errors.Wrap(errResp, "http error calling couchdb") 1715 } 1716 1717 // This situation should not occur according to the golang spec. 1718 // if this error returned (errResp) from an http call, then the resp should be not nil, 1719 // this is a structure and StatusCode is an int 1720 // This is meant to provide a more graceful error if this should occur 1721 if invalidCouchDBReturn(resp, errResp) { 1722 return nil, nil, errors.New("unable to connect to CouchDB, check the hostname and port") 1723 } 1724 1725 // set the return code for the couchDB request 1726 couchDBReturn.StatusCode = resp.StatusCode 1727 1728 // check to see if the status code from couchdb is 400 or higher 1729 // response codes 4XX and 500 will be treated as errors - 1730 // golang error will be created from the couchDBReturn contents and both will be returned 1731 if resp.StatusCode >= 400 { 1732 1733 // if the status code is 400 or greater, log and return an error 1734 couchdbLogger.Debugf("Error handling CouchDB request. Error:%s, Status Code:%v, Reason:%s", 1735 couchDBReturn.Error, resp.StatusCode, couchDBReturn.Reason) 1736 1737 return nil, couchDBReturn, errors.Errorf("error handling CouchDB request. Error:%s, Status Code:%v, Reason:%s", 1738 couchDBReturn.Error, resp.StatusCode, couchDBReturn.Reason) 1739 1740 } 1741 1742 couchdbLogger.Debugf("Exiting handleRequest()") 1743 1744 // If no errors, then return the http response and the couchdb return object 1745 return resp, couchDBReturn, nil 1746 } 1747 1748 func (couchInstance *couchInstance) recordMetric(startTime time.Time, dbName, api string, couchDBReturn *dbReturn) { 1749 couchInstance.stats.observeProcessingTime(startTime, dbName, api, strconv.Itoa(couchDBReturn.StatusCode)) 1750 } 1751 1752 // invalidCouchDBResponse checks to make sure either a valid response or error is returned 1753 func invalidCouchDBReturn(resp *http.Response, errResp error) bool { 1754 if resp == nil && errResp == nil { 1755 return true 1756 } 1757 return false 1758 } 1759 1760 // isJSON tests a string to determine if a valid JSON 1761 func isJSON(s string) bool { 1762 var js map[string]interface{} 1763 return json.Unmarshal([]byte(s), &js) == nil 1764 } 1765 1766 // encodePathElement uses Golang for url path encoding, additionally: 1767 // '/' is replaced by %2F, otherwise path encoding will treat as path separator and ignore it 1768 // '+' is replaced by %2B, otherwise path encoding will ignore it, while CouchDB will unencode the plus as a space 1769 // Note that all other URL special characters have been tested successfully without need for special handling 1770 func encodePathElement(str string) string { 1771 u := &url.URL{} 1772 u.Path = str 1773 encodedStr := u.EscapedPath() // url encode using golang url path encoding rules 1774 encodedStr = strings.Replace(encodedStr, "/", "%2F", -1) 1775 encodedStr = strings.Replace(encodedStr, "+", "%2B", -1) 1776 1777 return encodedStr 1778 } 1779 1780 func encodeForJSON(str string) (string, error) { 1781 buf := &bytes.Buffer{} 1782 encoder := json.NewEncoder(buf) 1783 if err := encoder.Encode(str); err != nil { 1784 return "", errors.Wrap(err, "error encoding json data") 1785 } 1786 // Encode adds double quotes to string and terminates with \n - stripping them as bytes as they are all ascii(0-127) 1787 buffer := buf.Bytes() 1788 return string(buffer[1 : len(buffer)-2]), nil 1789 } 1790 1791 // printDocumentIds is a convenience method to print readable log entries for arrays of pointers 1792 // to couch document IDs 1793 func printDocumentIds(documentPointers []*couchDoc) (string, error) { 1794 documentIds := []string{} 1795 1796 for _, documentPointer := range documentPointers { 1797 docMetadata := &docMetadata{} 1798 err := json.Unmarshal(documentPointer.jsonValue, &docMetadata) 1799 if err != nil { 1800 return "", errors.Wrap(err, "error unmarshalling json data") 1801 } 1802 documentIds = append(documentIds, docMetadata.ID) 1803 } 1804 return strings.Join(documentIds, ","), nil 1805 }