github.com/artpar/rclone@v1.67.3/backend/hasher/kv.go (about) 1 package hasher 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/gob" 7 "errors" 8 "fmt" 9 "strings" 10 "time" 11 12 "github.com/artpar/rclone/fs" 13 "github.com/artpar/rclone/fs/hash" 14 "github.com/artpar/rclone/fs/operations" 15 "github.com/artpar/rclone/lib/kv" 16 ) 17 18 const ( 19 timeFormat = "2006-01-02T15:04:05.000000000-0700" 20 anyFingerprint = "*" 21 ) 22 23 type hashMap map[hash.Type]string 24 25 type hashRecord struct { 26 Fp string // fingerprint 27 Hashes operations.HashSums 28 Created time.Time 29 } 30 31 func (r *hashRecord) encode(key string) ([]byte, error) { 32 var buf bytes.Buffer 33 if err := gob.NewEncoder(&buf).Encode(r); err != nil { 34 fs.Debugf(key, "hasher encoding %v: %v", r, err) 35 return nil, err 36 } 37 return buf.Bytes(), nil 38 } 39 40 func (r *hashRecord) decode(key string, data []byte) error { 41 if err := gob.NewDecoder(bytes.NewBuffer(data)).Decode(r); err != nil { 42 fs.Debugf(key, "hasher decoding %q failed: %v", data, err) 43 return err 44 } 45 return nil 46 } 47 48 // kvPrune: prune a single hash 49 type kvPrune struct { 50 key string 51 } 52 53 func (op *kvPrune) Do(ctx context.Context, b kv.Bucket) error { 54 return b.Delete([]byte(op.key)) 55 } 56 57 // kvPurge: delete a subtree 58 type kvPurge struct { 59 dir string 60 } 61 62 func (op *kvPurge) Do(ctx context.Context, b kv.Bucket) error { 63 dir := op.dir 64 if !strings.HasSuffix(dir, "/") { 65 dir += "/" 66 } 67 var items []string 68 cur := b.Cursor() 69 bkey, _ := cur.Seek([]byte(dir)) 70 for bkey != nil { 71 key := string(bkey) 72 if !strings.HasPrefix(key, dir) { 73 break 74 } 75 items = append(items, key[len(dir):]) 76 bkey, _ = cur.Next() 77 } 78 nerr := 0 79 for _, sub := range items { 80 if err := b.Delete([]byte(dir + sub)); err != nil { 81 nerr++ 82 } 83 } 84 fs.Debugf(dir, "%d hashes purged, %d failed", len(items)-nerr, nerr) 85 return nil 86 } 87 88 // kvMove: assign hashes to new path 89 type kvMove struct { 90 src string 91 dst string 92 dir bool 93 fs *Fs 94 } 95 96 func (op *kvMove) Do(ctx context.Context, b kv.Bucket) error { 97 src, dst := op.src, op.dst 98 if !op.dir { 99 err := moveHash(b, src, dst) 100 fs.Debugf(op.fs, "moving cached hash %s to %s (err: %v)", src, dst, err) 101 return err 102 } 103 104 if !strings.HasSuffix(src, "/") { 105 src += "/" 106 } 107 if !strings.HasSuffix(dst, "/") { 108 dst += "/" 109 } 110 111 var items []string 112 cur := b.Cursor() 113 bkey, _ := cur.Seek([]byte(src)) 114 for bkey != nil { 115 key := string(bkey) 116 if !strings.HasPrefix(key, src) { 117 break 118 } 119 items = append(items, key[len(src):]) 120 bkey, _ = cur.Next() 121 } 122 123 nerr := 0 124 for _, suffix := range items { 125 srcKey, dstKey := src+suffix, dst+suffix 126 err := moveHash(b, srcKey, dstKey) 127 fs.Debugf(op.fs, "Rename cache record %s -> %s (err: %v)", srcKey, dstKey, err) 128 if err != nil { 129 nerr++ 130 } 131 } 132 fs.Debugf(op.fs, "%d hashes moved, %d failed", len(items)-nerr, nerr) 133 return nil 134 } 135 136 func moveHash(b kv.Bucket, src, dst string) error { 137 data := b.Get([]byte(src)) 138 err := b.Delete([]byte(src)) 139 if err != nil || len(data) == 0 { 140 return err 141 } 142 return b.Put([]byte(dst), data) 143 } 144 145 // kvGet: get single hash from database 146 type kvGet struct { 147 key string 148 fp string 149 hash string 150 val string 151 age time.Duration 152 } 153 154 func (op *kvGet) Do(ctx context.Context, b kv.Bucket) error { 155 data := b.Get([]byte(op.key)) 156 if len(data) == 0 { 157 return errors.New("no record") 158 } 159 var r hashRecord 160 if err := r.decode(op.key, data); err != nil { 161 return errors.New("invalid record") 162 } 163 if !(r.Fp == anyFingerprint || op.fp == anyFingerprint || r.Fp == op.fp) { 164 return errors.New("fingerprint changed") 165 } 166 if time.Since(r.Created) > op.age { 167 return errors.New("record timed out") 168 } 169 if r.Hashes != nil { 170 op.val = r.Hashes[op.hash] 171 } 172 return nil 173 } 174 175 // kvPut: set hashes for an object by key 176 type kvPut struct { 177 key string 178 fp string 179 hashes operations.HashSums 180 age time.Duration 181 } 182 183 func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) { 184 data := b.Get([]byte(op.key)) 185 var r hashRecord 186 if len(data) > 0 { 187 err = r.decode(op.key, data) 188 if err != nil || r.Fp != op.fp || time.Since(r.Created) > op.age { 189 r.Hashes = nil 190 } 191 } 192 if len(r.Hashes) == 0 { 193 r.Created = time.Now() 194 r.Hashes = operations.HashSums{} 195 r.Fp = op.fp 196 } 197 198 for hashType, hashVal := range op.hashes { 199 r.Hashes[hashType] = hashVal 200 } 201 if data, err = r.encode(op.key); err != nil { 202 return fmt.Errorf("marshal failed: %w", err) 203 } 204 if err = b.Put([]byte(op.key), data); err != nil { 205 return fmt.Errorf("put failed: %w", err) 206 } 207 return err 208 } 209 210 // kvDump: dump the database. 211 // Note: long dump can cause concurrent operations to fail. 212 type kvDump struct { 213 full bool 214 root string 215 path string 216 fs *Fs 217 num int 218 total int 219 } 220 221 func (op *kvDump) Do(ctx context.Context, b kv.Bucket) error { 222 f, baseRoot, dbPath := op.fs, op.root, op.path 223 224 if op.full { 225 total := 0 226 num := 0 227 _ = b.ForEach(func(bkey, data []byte) error { 228 total++ 229 key := string(bkey) 230 include := (baseRoot == "" || key == baseRoot || strings.HasPrefix(key, baseRoot+"/")) 231 var r hashRecord 232 if err := r.decode(key, data); err != nil { 233 fs.Errorf(nil, "%s: invalid record: %v", key, err) 234 return nil 235 } 236 fmt.Println(f.dumpLine(&r, key, include, nil)) 237 if include { 238 num++ 239 } 240 return nil 241 }) 242 fs.Infof(dbPath, "%d records out of %d", num, total) 243 op.num, op.total = num, total // for unit tests 244 return nil 245 } 246 247 num := 0 248 cur := b.Cursor() 249 var bkey, data []byte 250 if baseRoot != "" { 251 bkey, data = cur.Seek([]byte(baseRoot)) 252 } else { 253 bkey, data = cur.First() 254 } 255 for bkey != nil { 256 key := string(bkey) 257 if !(baseRoot == "" || key == baseRoot || strings.HasPrefix(key, baseRoot+"/")) { 258 break 259 } 260 var r hashRecord 261 if err := r.decode(key, data); err != nil { 262 fs.Errorf(nil, "%s: invalid record: %v", key, err) 263 continue 264 } 265 if key = strings.TrimPrefix(key[len(baseRoot):], "/"); key == "" { 266 key = "/" 267 } 268 fmt.Println(f.dumpLine(&r, key, true, nil)) 269 num++ 270 bkey, data = cur.Next() 271 } 272 fs.Infof(dbPath, "%d records", num) 273 op.num = num // for unit tests 274 return nil 275 } 276 277 func (f *Fs) dumpLine(r *hashRecord, path string, include bool, err error) string { 278 var status string 279 switch { 280 case !include: 281 status = "ext" 282 case err != nil: 283 status = "bad" 284 case r.Fp == anyFingerprint: 285 status = "stk" 286 default: 287 status = "ok " 288 } 289 290 var hashes []string 291 for _, hashType := range f.keepHashes.Array() { 292 hashName := hashType.String() 293 hashVal := r.Hashes[hashName] 294 if hashVal == "" || err != nil { 295 hashVal = "-" 296 } 297 hashVal = fmt.Sprintf("%-*s", hash.Width(hashType, false), hashVal) 298 hashes = append(hashes, hashName+":"+hashVal) 299 } 300 hashesStr := strings.Join(hashes, " ") 301 302 age := time.Since(r.Created).Round(time.Second) 303 if age > 24*time.Hour { 304 age = age.Round(time.Hour) 305 } 306 if err != nil { 307 age = 0 308 } 309 ageStr := age.String() 310 if strings.HasSuffix(ageStr, "h0m0s") { 311 ageStr = strings.TrimSuffix(ageStr, "0m0s") 312 } 313 314 return fmt.Sprintf("%s %s %9s %s", status, hashesStr, ageStr, path) 315 }