github.com/deroproject/derosuite@v2.1.6-1.0.20200307070847-0f2e589c7a2b+incompatible/walletapi/db.go (about) 1 package walletapi 2 3 import "os" 4 import "fmt" 5 import "time" 6 import "crypto/rand" 7 import "crypto/sha1" 8 import "sync" 9 import "strings" 10 import "encoding/hex" 11 import "encoding/json" 12 import "encoding/binary" 13 14 import "github.com/romana/rlog" 15 import "github.com/vmihailenco/msgpack" 16 17 import bolt "github.com/coreos/bbolt" 18 import "github.com/blang/semver" 19 import "golang.org/x/crypto/pbkdf2" // // used to encrypt master password ( so user can change his password anytime) 20 21 import "github.com/deroproject/derosuite/crypto" 22 import "github.com/deroproject/derosuite/walletapi/mnemonics" 23 24 const FUNDS_BUCKET = "FUNDS" // stores all incoming funds, key is global output index encrypted form 25 const FUNDS_AVAILABLE = "FUNDS_AVAILABLE" // indices of all funds ready to spend 26 const FUNDS_SPENT = "FUNDS_SPENT" // indices of all funds already spent 27 const FUNDS_SPENT_WHERE = "FUNDS_SPENT_WHERE" // mapping which output -> spent where 28 const FUNDS_BUCKET_OUTGOING = "FUNDS_OUTGOING" // stores all tx where our funds were spent 29 30 const RING_BUCKET = "RING_BUCKET" // to randomly choose ring members when transactions are created 31 const KEYIMAGE_BUCKET = "KEYIMAGE_BUCKET" // used to track which funds have been spent (only on chain ) and which output was consumed 32 const SECRET_KEY_BUCKET = "TXSECRETKEY" // used to keep secret keys for any tx this wallet has created 33 const TX_OUT_DETAILS_BUCKET = "TX_OUT_DETAILS" // used to keep secret keys for any tx this wallet has created 34 35 const HEIGHT_TO_BLOCK_BUCKET = "H2BLOCK_BUCKET" // used to track height to block hash mapping 36 const OUR_TX_BUCKET = "OUR_TX_BUCKET" // this contains all TXs in which we have spent OUR FUNDS 37 38 // the strings are prepended so as there can NEVER be collision between TXID and payment ID 39 // as paymentID are chosen by users 40 const TXID = "TXID" // all TX to output index will have this string prepended 41 const PAYID = "PAYID" // all payment ID to output index will have this string prepended, 42 43 // PAYMENT ID itself is a bucket, TODO COLLISIONS ??? 44 //const FUNDS_BY_PAYMENT_ID_BUCKET ="PAYMENTID_BUCKET" // each payment id is a bucket itself, which stores OUTPUT_INDEX 45 46 //const FUNDS_BY_PAYMENT_ID_BUCKET = 10 // each payment id is a bucket itself 47 const FUNDS_BY_TX_ID_BUCKET = 11 // each tx id is a bucket 48 const FUNDS_BY_BLOCK_HEIGHT_BUCKET = 12 // funds sorted by block height 49 const FUNDS_SPENT_BY_BLOCK_HEIGHT_BUCKET = 13 // funds spent by block height 50 const NOTES_BY_TXID = 14 // any notes attached 51 52 // the db has 3 buckets, one 53 // funds ( funds which have arrived ), they can have attached , notes addresses, funds which payment id are linked to 54 // funds_available, contains only output indices 55 // funds_spent, contains only output indices 56 // searchable by payment id and searhable by txid and searchable by block height 57 58 // fundstate : available, spent, spentinpool, fund state works as state machine 59 // search_by_paymentid ( funds from a specific payment id can be accessed here ) 60 // search_by_txid ( funds from a specific payment id can be accessed here ) 61 // search_by_block_height ( funds from a specific payment id can be accessed here ) 62 63 // address book will have random number based entries 64 65 // see this https://godoc.org/golang.org/x/crypto/pbkdf2 66 type KDF struct { 67 Hashfunction string `json:"hash"` //"SHA1" currently only sha1 is supported 68 Keylen int `json:"keylen"` 69 Iterations int `json:"iterations"` 70 Salt []byte `json:"salt"` 71 } 72 73 // this is stored in disk in encrypted form 74 type Wallet struct { 75 Version semver.Version `json:"version"` // database version 76 Secret []byte `json:"secret"` // actual unlocker to the DB, depends on password from user, stored encrypted 77 // secret key used to encrypt all DB data ( both keys and values ) 78 // this is always in encrypted form 79 80 KDF KDF `json:"kdf"` 81 82 account *Account //`json:"-"` // not serialized, we store an encrypted version // keys, seed language etc settings 83 Account_Encrypted []byte `json:"account_encrypted"` 84 85 pbkdf2_password []byte // used to encrypt metadata on updates 86 master_password []byte // single password which never changes 87 88 Daemon_Endpoint string `json:"-"` // endpoint used to communicate with daemon 89 Daemon_Height uint64 `json:"-"` // used to track daemon height ony if wallet in online 90 Daemon_TopoHeight int64 `json:"-"` // used to track daemon topo height ony if wallet in online 91 92 wallet_online_mode bool // set whether the mode is online or offline 93 // an offline wallet can be converted to online mode, calling. 94 // SetOffline() and vice versa using SetOnline 95 // used to create transaction with this fee rate, 96 //if this is lower than network, then created transaction will be rejected by network 97 dynamic_fees_per_kb uint64 98 quit chan bool // channel to quit any processing go routines 99 100 db *bolt.DB // access to DB 101 102 rpcserver *RPCServer // reference to RPCserver 103 104 id string // first 8 bytes of wallet address , to put into logs to identify different wallets in case many are active 105 106 transfer_mutex sync.Mutex // to avoid races within the transfer 107 //sync.Mutex // used to syncronise access 108 sync.RWMutex 109 } 110 111 const META_BUCKET = "METADATA" // all metadata is stored in this bucket 112 const BLOCKS_BUCKET = "BLOCKS" // stores height to block hash mapping for later on syncing 113 114 var BLOCKCHAIN_UNIVERSE = []byte("BLOCKCHAIN_UNIVERSE") // all main chain txs are stored in this bucket 115 116 // when smart contracts are implemented, each will have it's own universe to track and maintain transactions 117 118 // this file implements the encrypted data store at rest 119 func Create_Encrypted_Wallet(filename string, password string, seed crypto.Key) (w *Wallet, err error) { 120 rlog.Infof("Creating Wallet from recovery seed") 121 w = &Wallet{} 122 w.Version, err = semver.Parse("0.0.1-alpha.preview.github") 123 124 if err != nil { 125 return 126 } 127 128 if _, err = os.Stat(filename); err == nil { 129 err = fmt.Errorf("File '%s' already exists", filename) 130 rlog.Errorf("err creating wallet %s", err) 131 return 132 } 133 134 w.db, err = bolt.Open(filename, 0600, &bolt.Options{Timeout: 1 * time.Second}) 135 136 if err != nil { 137 rlog.Errorf("err opening boltdb file %s", err) 138 return 139 } 140 141 // generate account keys 142 w.account, err = Generate_Account_From_Seed(seed) 143 if err != nil { 144 return 145 } 146 147 // generate a 64 byte key to be used as master Key 148 w.master_password = make([]byte, 32, 32) 149 _, err = rand.Read(w.master_password) 150 if err != nil { 151 return 152 } 153 154 err = w.Set_Encrypted_Wallet_Password(password) // lock the db with the password 155 156 w.quit = make(chan bool) 157 158 w.id = string((w.account.GetAddress().String())[:8]) // set unique id for logs 159 160 rlog.Infof("Successfully created wallet %s", w.id) 161 return 162 } 163 164 // create an encrypted wallet using electrum recovery words 165 func Create_Encrypted_Wallet_From_Recovery_Words(filename string, password string, electrum_seed string) (w *Wallet, err error) { 166 rlog.Infof("Creating Wallet from recovery words") 167 168 language, seed, err := mnemonics.Words_To_Key(electrum_seed) 169 if err != nil { 170 rlog.Errorf("err parsing recovery words %s", err) 171 return 172 } 173 w, err = Create_Encrypted_Wallet(filename, password, seed) 174 175 if err != nil { 176 rlog.Errorf("err creating wallet %s", err) 177 return 178 } 179 180 w.account.SeedLanguage = language 181 rlog.Infof("Successfully created wallet %s", w.id) 182 return 183 } 184 185 // create an encrypted wallet using using random data 186 func Create_Encrypted_Wallet_Random(filename string, password string) (w *Wallet, err error) { 187 rlog.Infof("Creating Wallet Randomly") 188 w, err = Create_Encrypted_Wallet(filename, password, *crypto.RandomScalar()) 189 190 if err != nil { 191 rlog.Errorf("err %s", err) 192 return 193 } 194 // TODO setup seed language, default is already english 195 rlog.Infof("Successfully created wallet %s", w.id) 196 return 197 } 198 199 // create an encrypted wallet using using random data 200 func Create_Encrypted_Wallet_ViewOnly(filename string, password string, viewkey string) (w *Wallet, err error) { 201 202 var public_spend, private_view crypto.Key 203 rlog.Infof("Creating View Only Wallet") 204 view_raw, err := hex.DecodeString(strings.TrimSpace(viewkey)) 205 if len(view_raw) != 64 || err != nil { 206 err = fmt.Errorf("View Only key must be 128 chars hexadecimal chars") 207 rlog.Errorf("err %s", err) 208 return 209 } 210 211 copy(public_spend[:], view_raw[:32]) 212 copy(private_view[:], view_raw[32:64]) 213 214 // create encrypted wallet randomly and then swap the keys 215 w, err = Create_Encrypted_Wallet(filename, password, *crypto.RandomScalar()) 216 217 if err != nil { 218 rlog.Errorf("err %s", err) 219 return 220 } 221 222 // swap the keys 223 w.account.Keys.Spendkey_Public = public_spend 224 w.account.Keys.Viewkey_Secret = private_view 225 w.account.Keys.Viewkey_Public = *(private_view.PublicKey()) 226 w.account.ViewOnly = true 227 228 w.Save_Wallet() // save wallet data 229 rlog.Infof("Successfully created view only wallet %s", w.id) 230 return 231 } 232 233 // create an encrypted wallet using using random data 234 func Create_Encrypted_Wallet_NonDeterministic(filename string, password string, secretkey,viewkey string) (w *Wallet, err error) { 235 236 var secret_spend, secret_view crypto.Key 237 rlog.Infof("Creating View Only Wallet") 238 spend_raw, err := hex.DecodeString(strings.TrimSpace(secretkey)) 239 if len(spend_raw) != 32 || err != nil { 240 err = fmt.Errorf("View Only key must be 64 chars hexadecimal chars") 241 rlog.Errorf("err %s", err) 242 return 243 } 244 245 copy(secret_spend[:], spend_raw[:32]) 246 247 248 view_raw, err := hex.DecodeString(strings.TrimSpace(viewkey)) 249 if len(view_raw) != 32 || err != nil { 250 err = fmt.Errorf("Spend Only key must be 64 chars hexadecimal chars") 251 rlog.Errorf("err %s", err) 252 return 253 } 254 255 copy(secret_view[:], view_raw[:32]) 256 257 // create encrypted wallet randomly and then swap the keys 258 w, err = Create_Encrypted_Wallet(filename, password, *crypto.RandomScalar()) 259 260 if err != nil { 261 rlog.Errorf("err %s", err) 262 return 263 } 264 265 // swap the keys 266 w.account.Keys.Spendkey_Secret = secret_spend 267 w.account.Keys.Spendkey_Public = *(secret_spend.PublicKey()) 268 w.account.Keys.Viewkey_Secret = secret_view 269 w.account.Keys.Viewkey_Public = *(secret_view.PublicKey()) 270 271 w.Save_Wallet() // save wallet data 272 rlog.Infof("Successfully created view only wallet %s", w.id) 273 return 274 } 275 276 // wallet must already be open 277 func (w *Wallet) Set_Encrypted_Wallet_Password(password string) (err error) { 278 279 if w == nil { 280 return 281 } 282 w.Lock() 283 284 // set up KDF structure 285 w.KDF.Salt = make([]byte, 32, 32) 286 _, err = rand.Read(w.KDF.Salt) 287 if err != nil { 288 w.Unlock() 289 return 290 } 291 w.KDF.Keylen = 32 292 w.KDF.Iterations = 262144 293 w.KDF.Hashfunction = "SHA1" 294 295 // lets generate the bcrypted password 296 297 w.pbkdf2_password = Generate_Key(w.KDF, password) 298 299 w.Unlock() 300 w.Save_Wallet() // save wallet data 301 302 return 303 } 304 305 func Open_Encrypted_Wallet(filename string, password string) (w *Wallet, err error) { 306 w = &Wallet{} 307 308 if _, err = os.Stat(filename); os.IsNotExist(err) { 309 err = fmt.Errorf("File '%s' does NOT exists", filename) 310 rlog.Errorf("err opening wallet %s", err) 311 return 312 } 313 314 w.db, err = bolt.Open(filename, 0600, &bolt.Options{Timeout: 1 * time.Second}) 315 316 if err != nil { 317 rlog.Errorf("err opening boltdb %s", err) 318 return 319 } 320 321 // read the metadata from metadat bucket 322 w.db.View(func(tx *bolt.Tx) error { 323 // Assume bucket exists and has keys 324 b := tx.Bucket([]byte(META_BUCKET)) 325 326 v := b.Get([]byte(META_BUCKET)) 327 328 if v == nil || len(v) == 0 { 329 err = fmt.Errorf("Invalid Database, Could not find meta data") 330 rlog.Errorf("err opening wallet %s", err) 331 return err 332 } 333 334 //fmt.Printf("v %+v\n",string(v)) // DO NOT dump account keys 335 336 // deserialize json data 337 err = json.Unmarshal(v, &w) 338 if err != nil { 339 rlog.Errorf("err parsing metabucket %s", err) 340 return err 341 } 342 343 w.quit = make(chan bool) 344 // todo make any routines necessary, such as sync etc 345 346 return nil 347 }) 348 349 // try to deseal password and store it 350 w.pbkdf2_password = Generate_Key(w.KDF, password) 351 352 // try to decrypt the master password with the pbkdf2 353 w.master_password, err = DecryptWithKey(w.pbkdf2_password, w.Secret) // decrypt the master key 354 if err != nil { 355 rlog.Errorf("err opening secret err: %s ", err) 356 err = fmt.Errorf("Invalid Password") 357 w.db.Close() 358 w = nil 359 return 360 } 361 362 // password has been found, open the account 363 364 account_bytes, err := w.Decrypt(w.Account_Encrypted) 365 if err != nil { 366 rlog.Errorf("err opening account err: %s ", err) 367 err = fmt.Errorf("Invalid Password") 368 w.db.Close() 369 w = nil 370 return 371 } 372 373 w.account = &Account{} // allocate a new instance 374 err = json.Unmarshal(account_bytes, w.account) 375 if err != nil { 376 return 377 } 378 379 w.id = string((w.account.GetAddress().String())[:8]) // set unique id for logs 380 381 return 382 383 } 384 385 // check whether the already opened wallet can use this password 386 func (w *Wallet) Check_Password(password string) bool { 387 w.Lock() 388 defer w.Unlock() 389 if w == nil { 390 return false 391 } 392 393 pbkdf2_password := Generate_Key(w.KDF, password) 394 395 // TODO we can compare pbkdf2_password & w.pbkdf2_password, if they are equal password is vaid 396 397 // try to decrypt the master password with the pbkdf2 398 _, err := DecryptWithKey(pbkdf2_password, w.Secret) // decrypt the master key 399 400 if err == nil { 401 return true 402 } 403 rlog.Warnf("%s Invalid Password", w.id) 404 return false 405 406 } 407 408 // save updated copy of wallet 409 func (w *Wallet) Save_Wallet() (err error) { 410 w.Lock() 411 defer w.Unlock() 412 if w == nil { 413 return 414 } 415 416 // encrypted the master password with the pbkdf2 417 w.Secret, err = EncryptWithKey(w.pbkdf2_password[:], w.master_password) // encrypt the master key 418 if err != nil { 419 return 420 } 421 422 // encrypt the account 423 424 account_serialized, err := json.Marshal(w.account) 425 if err != nil { 426 return 427 } 428 w.Account_Encrypted, err = w.Encrypt(account_serialized) 429 if err != nil { 430 return 431 } 432 433 // json marshal wallet data struct, serialize it, encrypt it and store it 434 serialized, err := json.Marshal(&w) 435 if err != nil { 436 return 437 } 438 //fmt.Printf("Serialized %+v\n",serialized) 439 440 // let save the secret to DISK in encrypted form 441 err = w.db.Update(func(tx *bolt.Tx) (err error) { 442 443 bucket, err := tx.CreateBucketIfNotExists([]byte(META_BUCKET)) 444 if err != nil { 445 return 446 } 447 err = bucket.Put([]byte(META_BUCKET), serialized) 448 return 449 450 }) 451 rlog.Infof("Saving wallet %s", w.id) 452 return 453 } 454 455 // close the wallet 456 func (w *Wallet) Close_Encrypted_Wallet() { 457 close(w.quit) 458 459 time.Sleep(time.Second) // give goroutines some time to quit 460 rlog.Infof("Saving and Closing Wallet %s\n", w.id) 461 w.Save_Wallet() 462 w.db.Sync() 463 464 w.db.Close() 465 } 466 467 // generate key from password 468 func Generate_Key(k KDF, password string) (key []byte) { 469 switch k.Hashfunction { 470 case "SHA1": 471 return pbkdf2.Key([]byte(password), k.Salt, k.Iterations, k.Keylen, sha1.New) 472 473 default: 474 return pbkdf2.Key([]byte(password), k.Salt, k.Iterations, k.Keylen, sha1.New) 475 } 476 } 477 478 // check whether a key exists 479 func (w *Wallet) check_key_exists(universe []byte, subbucket []byte, key []byte) (result bool) { 480 481 //fmt.Printf("Checking %s %s %x \n", string(universe), string(subbucket), key) 482 483 w.db.View(func(tx *bolt.Tx) error { 484 universe_bucket := tx.Bucket(w.Key2Key(universe)) //open universe bucket 485 if universe_bucket == nil { 486 return nil 487 } 488 bucket := universe_bucket.Bucket(w.Key2Key(subbucket)) // open subbucket 489 if bucket == nil { 490 return nil 491 } 492 493 v := bucket.Get(w.Key2Key(key)) 494 495 if v != nil { 496 // fmt.Printf("Found\n") 497 result = true 498 } 499 return nil 500 }) 501 return // default is false 502 } 503 504 // delete specified key 505 func (w *Wallet) delete_key(universe []byte, subbucket []byte, key []byte) { 506 rlog.Tracef(1, "Deleting %s %s %x\n", string(universe), string(subbucket), key) 507 508 w.db.Update(func(tx *bolt.Tx) (err error) { 509 universe_bucket, err := tx.CreateBucketIfNotExists(w.Key2Key(universe)) //open universe bucket 510 if err != nil { 511 return 512 } 513 bucket, err := universe_bucket.CreateBucketIfNotExists(w.Key2Key(subbucket)) // open subbucket 514 if err != nil { 515 return 516 } 517 518 err = bucket.Delete(w.Key2Key(key)) 519 520 return err // it will be nil 521 522 }) 523 524 } 525 526 // delete specified key 527 func (w *Wallet) delete_bucket(universe []byte, subbucket []byte) { 528 rlog.Tracef(1, "Deleting bucket %s %s \n", string(universe), string(subbucket)) 529 530 w.db.Update(func(tx *bolt.Tx) (err error) { 531 universe_bucket, err := tx.CreateBucketIfNotExists(w.Key2Key(universe)) //open universe bucket 532 if err != nil { 533 return 534 } 535 err = universe_bucket.DeleteBucket(w.Key2Key(subbucket)) // delete subbucket 536 return err // it will be nil 537 538 }) 539 540 } 541 542 // store a key-value, everything is encrypted 543 func (w *Wallet) store_key_value(universe []byte, subbucket []byte, key []byte, value []byte) error { 544 545 rlog.Tracef(1, "Storing %s %s %x\n", string(universe), string(subbucket), key) 546 547 return w.db.Update(func(tx *bolt.Tx) (err error) { 548 universe_bucket, err := tx.CreateBucketIfNotExists(w.Key2Key(universe)) //open universe bucket 549 if err != nil { 550 return 551 } 552 bucket, err := universe_bucket.CreateBucketIfNotExists(w.Key2Key(subbucket)) // open subbucket 553 if err != nil { 554 return 555 } 556 557 encrypted_value, err := w.Encrypt(value) // encrypt and seal the value 558 if err != nil { 559 return 560 } 561 err = bucket.Put(w.Key2Key(key), encrypted_value) 562 563 return err // it will be nil 564 565 }) 566 } 567 568 func (w *Wallet) load_key_value(universe []byte, subbucket []byte, key []byte) (value []byte, err error) { 569 570 rlog.Tracef(1, "loading %s %s %x\n", string(universe), string(subbucket), key) 571 572 w.db.View(func(tx *bolt.Tx) (err error) { 573 universe_bucket := tx.Bucket(w.Key2Key(universe)) //open universe bucket 574 if universe_bucket == nil { 575 return nil 576 } 577 bucket := universe_bucket.Bucket(w.Key2Key(subbucket)) // open subbucket 578 if bucket == nil { 579 return nil 580 } 581 v := bucket.Get(w.Key2Key(key)) 582 583 if v == nil { 584 return fmt.Errorf("%s %s %x NOT Found", string(universe), string(subbucket), key) 585 } 586 587 // fmt.Printf("length of encrypted value %d\n",len(v)) 588 589 value, err = w.Decrypt(v) 590 591 return err // it will be nil if everything is alright 592 593 }) 594 return 595 } 596 597 // enumerate all keys from the bucket 598 // due to design enumeration is impossible, 599 // however, all the keys of a specific bucket, where necessary are added as values, 600 // for example, we never enumerate our funds, if we donot store them in FUNDS_AVAILABLE or FUNDS_SPENT bucket 601 // so we find all values in FUNDS_AVAILABLE and FUNDS_SPENT bucket, and then decode value to recover the funds 602 // this function should only be called for FUNDS_AVAILABLE or FUNDS_SPENT bucket 603 func (w *Wallet) load_all_values_from_bucket(universe []byte, subbucket []byte) (values [][]byte) { 604 605 w.db.View(func(tx *bolt.Tx) (err error) { 606 universe_bucket := tx.Bucket(w.Key2Key(universe)) //open universe bucket 607 if universe_bucket == nil { 608 609 return nil 610 } 611 bucket := universe_bucket.Bucket(w.Key2Key(subbucket)) // open subbucket 612 if bucket == nil { 613 return nil 614 } 615 616 //fmt.Printf("Enumerating Keys\n") 617 // Iterate over items 618 err = bucket.ForEach(func(k, v []byte) error { 619 // fmt.Printf("Enumerated key\n") 620 value, err := w.Decrypt(v) 621 if err == nil { 622 values = append(values, value) 623 } 624 625 return err 626 }) 627 628 return err // it will be nil if everything is alright 629 630 }) 631 return 632 } 633 634 func (w *Wallet) load_ring_member(index_global uint64) (r Ring_Member, err error) { 635 636 // store all data 637 data_bytes, err := w.load_key_value(BLOCKCHAIN_UNIVERSE, []byte(RING_BUCKET), itob(index_global)) 638 639 if err != nil { 640 return 641 } 642 643 err = msgpack.Unmarshal(data_bytes, &r) 644 645 return 646 } 647 648 // itob returns an 8-byte big endian representation of v. 649 func itob(v uint64) []byte { 650 b := make([]byte, 8) 651 binary.BigEndian.PutUint64(b, uint64(v)) 652 return b 653 }