github.com/tenywen/fabric@v1.0.0-beta.0.20170620030522-a5b1ed380643/core/ledger/kvledger/txmgmt/statedb/statecouchdb/statecouchdb.go (about) 1 /* 2 Copyright IBM Corp. 2016, 2017 All Rights Reserved. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package statecouchdb 18 19 import ( 20 "bytes" 21 "encoding/json" 22 "errors" 23 "fmt" 24 "strconv" 25 "strings" 26 "sync" 27 "unicode/utf8" 28 29 "github.com/hyperledger/fabric/common/flogging" 30 "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/statedb" 31 "github.com/hyperledger/fabric/core/ledger/kvledger/txmgmt/version" 32 "github.com/hyperledger/fabric/core/ledger/ledgerconfig" 33 "github.com/hyperledger/fabric/core/ledger/util/couchdb" 34 ) 35 36 var logger = flogging.MustGetLogger("statecouchdb") 37 38 var compositeKeySep = []byte{0x00} 39 var lastKeyIndicator = byte(0x01) 40 41 var binaryWrapper = "valueBytes" 42 43 //querySkip is implemented for future use by query paging 44 //currently defaulted to 0 and is not used 45 var querySkip = 0 46 47 // VersionedDBProvider implements interface VersionedDBProvider 48 type VersionedDBProvider struct { 49 couchInstance *couchdb.CouchInstance 50 databases map[string]*VersionedDB 51 mux sync.Mutex 52 openCounts uint64 53 } 54 55 // NewVersionedDBProvider instantiates VersionedDBProvider 56 func NewVersionedDBProvider() (*VersionedDBProvider, error) { 57 logger.Debugf("constructing CouchDB VersionedDBProvider") 58 couchDBDef := couchdb.GetCouchDBDefinition() 59 couchInstance, err := couchdb.CreateCouchInstance(couchDBDef.URL, couchDBDef.Username, couchDBDef.Password, 60 couchDBDef.MaxRetries, couchDBDef.MaxRetriesOnStartup, couchDBDef.RequestTimeout) 61 if err != nil { 62 return nil, err 63 } 64 65 return &VersionedDBProvider{couchInstance, make(map[string]*VersionedDB), sync.Mutex{}, 0}, nil 66 } 67 68 // GetDBHandle gets the handle to a named database 69 func (provider *VersionedDBProvider) GetDBHandle(dbName string) (statedb.VersionedDB, error) { 70 provider.mux.Lock() 71 defer provider.mux.Unlock() 72 73 vdb := provider.databases[dbName] 74 if vdb == nil { 75 var err error 76 vdb, err = newVersionedDB(provider.couchInstance, dbName) 77 if err != nil { 78 return nil, err 79 } 80 provider.databases[dbName] = vdb 81 } 82 return vdb, nil 83 } 84 85 // Close closes the underlying db instance 86 func (provider *VersionedDBProvider) Close() { 87 // No close needed on Couch 88 } 89 90 // VersionedDB implements VersionedDB interface 91 type VersionedDB struct { 92 db *couchdb.CouchDatabase 93 dbName string 94 } 95 96 // newVersionedDB constructs an instance of VersionedDB 97 func newVersionedDB(couchInstance *couchdb.CouchInstance, dbName string) (*VersionedDB, error) { 98 // CreateCouchDatabase creates a CouchDB database object, as well as the underlying database if it does not exist 99 db, err := couchdb.CreateCouchDatabase(*couchInstance, dbName) 100 if err != nil { 101 return nil, err 102 } 103 return &VersionedDB{db, dbName}, nil 104 } 105 106 // Open implements method in VersionedDB interface 107 func (vdb *VersionedDB) Open() error { 108 // no need to open db since a shared couch instance is used 109 return nil 110 } 111 112 // Close implements method in VersionedDB interface 113 func (vdb *VersionedDB) Close() { 114 // no need to close db since a shared couch instance is used 115 } 116 117 // ValidateKey implements method in VersionedDB interface 118 func (vdb *VersionedDB) ValidateKey(key string) error { 119 if !utf8.ValidString(key) { 120 return fmt.Errorf("Key should be a valid utf8 string: [%x]", key) 121 } 122 return nil 123 } 124 125 // GetState implements method in VersionedDB interface 126 func (vdb *VersionedDB) GetState(namespace string, key string) (*statedb.VersionedValue, error) { 127 logger.Debugf("GetState(). ns=%s, key=%s", namespace, key) 128 129 compositeKey := constructCompositeKey(namespace, key) 130 131 couchDoc, _, err := vdb.db.ReadDoc(string(compositeKey)) 132 if err != nil { 133 return nil, err 134 } 135 if couchDoc == nil { 136 return nil, nil 137 } 138 139 //remove the data wrapper and return the value and version 140 returnValue, returnVersion := removeDataWrapper(couchDoc.JSONValue, couchDoc.Attachments) 141 142 return &statedb.VersionedValue{Value: returnValue, Version: &returnVersion}, nil 143 } 144 145 func removeDataWrapper(wrappedValue []byte, attachments []*couchdb.Attachment) ([]byte, version.Height) { 146 147 //initialize the return value 148 returnValue := []byte{} 149 150 //initialize a default return version 151 returnVersion := version.NewHeight(0, 0) 152 153 //create a generic map for the json 154 jsonResult := make(map[string]interface{}) 155 156 //unmarshal the selected json into the generic map 157 decoder := json.NewDecoder(bytes.NewBuffer(wrappedValue)) 158 decoder.UseNumber() 159 _ = decoder.Decode(&jsonResult) 160 161 // handle binary or json data 162 if jsonResult[dataWrapper] == nil && attachments != nil { // binary attachment 163 // get binary data from attachment 164 for _, attachment := range attachments { 165 if attachment.Name == binaryWrapper { 166 returnValue = attachment.AttachmentBytes 167 } 168 } 169 } else { 170 //place the result json in the data key 171 returnMap := jsonResult[dataWrapper] 172 173 //marshal the mapped data. this wrappers the result in a key named "data" 174 returnValue, _ = json.Marshal(returnMap) 175 176 } 177 178 //create an array containing the blockNum and txNum 179 versionArray := strings.Split(fmt.Sprintf("%s", jsonResult["version"]), ":") 180 181 //convert the blockNum from String to unsigned int 182 blockNum, _ := strconv.ParseUint(versionArray[0], 10, 64) 183 184 //convert the txNum from String to unsigned int 185 txNum, _ := strconv.ParseUint(versionArray[1], 10, 64) 186 187 //create the version based on the blockNum and txNum 188 returnVersion = version.NewHeight(blockNum, txNum) 189 190 return returnValue, *returnVersion 191 192 } 193 194 // GetStateMultipleKeys implements method in VersionedDB interface 195 func (vdb *VersionedDB) GetStateMultipleKeys(namespace string, keys []string) ([]*statedb.VersionedValue, error) { 196 197 vals := make([]*statedb.VersionedValue, len(keys)) 198 for i, key := range keys { 199 val, err := vdb.GetState(namespace, key) 200 if err != nil { 201 return nil, err 202 } 203 vals[i] = val 204 } 205 return vals, nil 206 207 } 208 209 // GetStateRangeScanIterator implements method in VersionedDB interface 210 // startKey is inclusive 211 // endKey is exclusive 212 func (vdb *VersionedDB) GetStateRangeScanIterator(namespace string, startKey string, endKey string) (statedb.ResultsIterator, error) { 213 214 //Get the querylimit from core.yaml 215 queryLimit := ledgerconfig.GetQueryLimit() 216 217 compositeStartKey := constructCompositeKey(namespace, startKey) 218 compositeEndKey := constructCompositeKey(namespace, endKey) 219 if endKey == "" { 220 compositeEndKey[len(compositeEndKey)-1] = lastKeyIndicator 221 } 222 queryResult, err := vdb.db.ReadDocRange(string(compositeStartKey), string(compositeEndKey), queryLimit, querySkip) 223 if err != nil { 224 logger.Debugf("Error calling ReadDocRange(): %s\n", err.Error()) 225 return nil, err 226 } 227 logger.Debugf("Exiting GetStateRangeScanIterator") 228 return newKVScanner(namespace, *queryResult), nil 229 230 } 231 232 // ExecuteQuery implements method in VersionedDB interface 233 func (vdb *VersionedDB) ExecuteQuery(namespace, query string) (statedb.ResultsIterator, error) { 234 235 //Get the querylimit from core.yaml 236 queryLimit := ledgerconfig.GetQueryLimit() 237 238 queryString, err := ApplyQueryWrapper(namespace, query, queryLimit, 0) 239 if err != nil { 240 logger.Debugf("Error calling ApplyQueryWrapper(): %s\n", err.Error()) 241 return nil, err 242 } 243 244 queryResult, err := vdb.db.QueryDocuments(queryString) 245 if err != nil { 246 logger.Debugf("Error calling QueryDocuments(): %s\n", err.Error()) 247 return nil, err 248 } 249 logger.Debugf("Exiting ExecuteQuery") 250 return newQueryScanner(*queryResult), nil 251 } 252 253 // ApplyUpdates implements method in VersionedDB interface 254 func (vdb *VersionedDB) ApplyUpdates(batch *statedb.UpdateBatch, height *version.Height) error { 255 256 namespaces := batch.GetUpdatedNamespaces() 257 for _, ns := range namespaces { 258 updates := batch.GetUpdates(ns) 259 for k, vv := range updates { 260 compositeKey := constructCompositeKey(ns, k) 261 logger.Debugf("Channel [%s]: Applying key=[%#v]", vdb.dbName, compositeKey) 262 263 //convert nils to deletes 264 if vv.Value == nil { 265 266 vdb.db.DeleteDoc(string(compositeKey), "") 267 268 } else { 269 couchDoc := &couchdb.CouchDoc{} 270 271 //Check to see if the value is a valid JSON 272 //If this is not a valid JSON, then store as an attachment 273 if couchdb.IsJSON(string(vv.Value)) { 274 // Handle it as json 275 couchDoc.JSONValue = addVersionAndChainCodeID(vv.Value, ns, vv.Version) 276 } else { // if the data is not JSON, save as binary attachment in Couch 277 278 attachment := &couchdb.Attachment{} 279 attachment.AttachmentBytes = vv.Value 280 attachment.ContentType = "application/octet-stream" 281 attachment.Name = binaryWrapper 282 attachments := append([]*couchdb.Attachment{}, attachment) 283 284 couchDoc.Attachments = attachments 285 couchDoc.JSONValue = addVersionAndChainCodeID(nil, ns, vv.Version) 286 } 287 288 // SaveDoc using couchdb client and use attachment to persist the binary data 289 rev, err := vdb.db.SaveDoc(string(compositeKey), "", couchDoc) 290 if err != nil { 291 logger.Errorf("Error during Commit(): %s\n", err.Error()) 292 return err 293 } 294 if rev != "" { 295 logger.Debugf("Saved document revision number: %s\n", rev) 296 } 297 } 298 } 299 } 300 301 // Record a savepoint at a given height 302 err := vdb.recordSavepoint(height) 303 if err != nil { 304 logger.Errorf("Error during recordSavepoint: %s\n", err.Error()) 305 return err 306 } 307 308 return nil 309 } 310 311 //addVersionAndChainCodeID adds keys for version and chaincodeID to the JSON value 312 func addVersionAndChainCodeID(value []byte, chaincodeID string, version *version.Height) []byte { 313 314 //create a version mapping 315 jsonMap := map[string]interface{}{"version": fmt.Sprintf("%v:%v", version.BlockNum, version.TxNum)} 316 317 //add the chaincodeID 318 jsonMap["chaincodeid"] = chaincodeID 319 320 //Add the wrapped data if the value is not null 321 if value != nil { 322 323 //create a new genericMap 324 rawJSON := (*json.RawMessage)(&value) 325 326 //add the rawJSON to the map 327 jsonMap[dataWrapper] = rawJSON 328 329 } 330 331 //marshal the data to a byte array 332 returnJSON, _ := json.Marshal(jsonMap) 333 334 return returnJSON 335 336 } 337 338 // Savepoint docid (key) for couchdb 339 const savepointDocID = "statedb_savepoint" 340 341 // Savepoint data for couchdb 342 type couchSavepointData struct { 343 BlockNum uint64 `json:"BlockNum"` 344 TxNum uint64 `json:"TxNum"` 345 UpdateSeq string `json:"UpdateSeq"` 346 } 347 348 // recordSavepoint Record a savepoint in statedb. 349 // Couch parallelizes writes in cluster or sharded setup and ordering is not guaranteed. 350 // Hence we need to fence the savepoint with sync. So ensure_full_commit is called before 351 // savepoint to ensure all block writes are flushed. Savepoint itself does not need to be flushed, 352 // it will get flushed with next block if not yet committed. 353 func (vdb *VersionedDB) recordSavepoint(height *version.Height) error { 354 var err error 355 var savepointDoc couchSavepointData 356 // ensure full commit to flush all changes until now to disk 357 dbResponse, err := vdb.db.EnsureFullCommit() 358 if err != nil || dbResponse.Ok != true { 359 logger.Errorf("Failed to perform full commit\n") 360 return errors.New("Failed to perform full commit") 361 } 362 363 // construct savepoint document 364 // UpdateSeq would be useful if we want to get all db changes since a logical savepoint 365 dbInfo, _, err := vdb.db.GetDatabaseInfo() 366 if err != nil { 367 logger.Errorf("Failed to get DB info %s\n", err.Error()) 368 return err 369 } 370 savepointDoc.BlockNum = height.BlockNum 371 savepointDoc.TxNum = height.TxNum 372 savepointDoc.UpdateSeq = dbInfo.UpdateSeq 373 374 savepointDocJSON, err := json.Marshal(savepointDoc) 375 if err != nil { 376 logger.Errorf("Failed to create savepoint data %s\n", err.Error()) 377 return err 378 } 379 380 // SaveDoc using couchdb client and use JSON format 381 _, err = vdb.db.SaveDoc(savepointDocID, "", &couchdb.CouchDoc{JSONValue: savepointDocJSON, Attachments: nil}) 382 if err != nil { 383 logger.Errorf("Failed to save the savepoint to DB %s\n", err.Error()) 384 return err 385 } 386 387 return nil 388 } 389 390 // GetLatestSavePoint implements method in VersionedDB interface 391 func (vdb *VersionedDB) GetLatestSavePoint() (*version.Height, error) { 392 393 var err error 394 couchDoc, _, err := vdb.db.ReadDoc(savepointDocID) 395 if err != nil { 396 logger.Errorf("Failed to read savepoint data %s\n", err.Error()) 397 return nil, err 398 } 399 400 // ReadDoc() not found (404) will result in nil response, in these cases return height nil 401 if couchDoc == nil || couchDoc.JSONValue == nil { 402 return nil, nil 403 } 404 405 savepointDoc := &couchSavepointData{} 406 err = json.Unmarshal(couchDoc.JSONValue, &savepointDoc) 407 if err != nil { 408 logger.Errorf("Failed to unmarshal savepoint data %s\n", err.Error()) 409 return nil, err 410 } 411 412 return &version.Height{BlockNum: savepointDoc.BlockNum, TxNum: savepointDoc.TxNum}, nil 413 } 414 415 func constructCompositeKey(ns string, key string) []byte { 416 compositeKey := []byte(ns) 417 compositeKey = append(compositeKey, compositeKeySep...) 418 compositeKey = append(compositeKey, []byte(key)...) 419 return compositeKey 420 } 421 422 func splitCompositeKey(compositeKey []byte) (string, string) { 423 split := bytes.SplitN(compositeKey, compositeKeySep, 2) 424 return string(split[0]), string(split[1]) 425 } 426 427 type kvScanner struct { 428 cursor int 429 namespace string 430 results []couchdb.QueryResult 431 } 432 433 func newKVScanner(namespace string, queryResults []couchdb.QueryResult) *kvScanner { 434 return &kvScanner{-1, namespace, queryResults} 435 } 436 437 func (scanner *kvScanner) Next() (statedb.QueryResult, error) { 438 439 scanner.cursor++ 440 441 if scanner.cursor >= len(scanner.results) { 442 return nil, nil 443 } 444 445 selectedKV := scanner.results[scanner.cursor] 446 447 _, key := splitCompositeKey([]byte(selectedKV.ID)) 448 449 //remove the data wrapper and return the value and version 450 returnValue, returnVersion := removeDataWrapper(selectedKV.Value, selectedKV.Attachments) 451 452 return &statedb.VersionedKV{ 453 CompositeKey: statedb.CompositeKey{Namespace: scanner.namespace, Key: key}, 454 VersionedValue: statedb.VersionedValue{Value: returnValue, Version: &returnVersion}}, nil 455 } 456 457 func (scanner *kvScanner) Close() { 458 scanner = nil 459 } 460 461 type queryScanner struct { 462 cursor int 463 results []couchdb.QueryResult 464 } 465 466 func newQueryScanner(queryResults []couchdb.QueryResult) *queryScanner { 467 return &queryScanner{-1, queryResults} 468 } 469 470 func (scanner *queryScanner) Next() (statedb.QueryResult, error) { 471 472 scanner.cursor++ 473 474 if scanner.cursor >= len(scanner.results) { 475 return nil, nil 476 } 477 478 selectedResultRecord := scanner.results[scanner.cursor] 479 480 namespace, key := splitCompositeKey([]byte(selectedResultRecord.ID)) 481 482 //remove the data wrapper and return the value and version 483 returnValue, returnVersion := removeDataWrapper(selectedResultRecord.Value, selectedResultRecord.Attachments) 484 485 return &statedb.VersionedKV{ 486 CompositeKey: statedb.CompositeKey{Namespace: namespace, Key: key}, 487 VersionedValue: statedb.VersionedValue{Value: returnValue, Version: &returnVersion}}, nil 488 } 489 490 func (scanner *queryScanner) Close() { 491 scanner = nil 492 }