github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/model/vfs/vfsswift/fsck_v3.go (about) 1 package vfsswift 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/hex" 7 "encoding/json" 8 "errors" 9 "path" 10 "strings" 11 12 "github.com/cozy/cozy-stack/model/vfs" 13 "github.com/cozy/cozy-stack/pkg/consts" 14 "github.com/cozy/cozy-stack/pkg/couchdb" 15 "github.com/ncw/swift/v2" 16 ) 17 18 func (sfs *swiftVFSV3) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { 19 entries := make(map[string]*vfs.TreeFile, 1024) 20 tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { 21 if !f.IsDir { 22 entries[f.DocID+"/"+f.InternalID] = f 23 } 24 }) 25 if err != nil { 26 return err 27 } 28 if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { 29 if errors.Is(err, vfs.ErrFsckFailFast) { 30 return nil 31 } 32 return err 33 } 34 return sfs.checkFiles(entries, accumulate, failFast) 35 } 36 37 func (sfs *swiftVFSV3) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { 38 entries := make(map[string]*vfs.TreeFile, 1024) 39 _, err := sfs.BuildTree(func(f *vfs.TreeFile) { 40 if !f.IsDir { 41 entries[f.DocID+"/"+f.InternalID] = f 42 } 43 }) 44 if err != nil { 45 return err 46 } 47 return sfs.checkFiles(entries, accumulate, failFast) 48 } 49 50 func (sfs *swiftVFSV3) checkFiles( 51 entries map[string]*vfs.TreeFile, 52 accumulate func(log *vfs.FsckLog), 53 failFast bool, 54 ) error { 55 versions := make(map[string]*vfs.Version, 1024) 56 err := couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error { 57 v := &vfs.Version{} 58 if erru := json.Unmarshal(data, v); erru != nil { 59 return erru 60 } 61 versions[v.DocID] = v 62 return nil 63 }) 64 if err != nil { 65 return err 66 } 67 68 images := make(map[string]struct{}) 69 err = couchdb.ForeachDocs(sfs, consts.NotesImages, func(_ string, data json.RawMessage) error { 70 img := make(map[string]interface{}) 71 if erru := json.Unmarshal(data, &img); erru != nil { 72 return erru 73 } 74 id, _ := img["_id"].(string) 75 images[id] = struct{}{} 76 return nil 77 }) 78 if err != nil && !couchdb.IsNoDatabaseError(err) { 79 return err 80 } 81 82 fileIDs := make(map[string]struct{}, len(entries)) 83 for _, f := range entries { 84 fileIDs[f.DocID] = struct{}{} 85 } 86 87 opts := &swift.ObjectsOpts{Limit: 5_000} 88 err = sfs.c.ObjectsWalk(sfs.ctx, sfs.container, opts, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) { 89 objs, err := sfs.c.Objects(sfs.ctx, sfs.container, opts) 90 if err != nil { 91 return nil, err 92 } 93 for _, obj := range objs { 94 if strings.HasPrefix(obj.Name, "thumbs/") { 95 objName := strings.TrimPrefix(obj.Name, "thumbs/") 96 idx := strings.LastIndex(objName, "-") 97 objName = objName[0:idx] // Remove -format suffix 98 fileID := makeDocID(objName) 99 if _, ok := fileIDs[fileID]; !ok { 100 if _, ok := images[fileID]; !ok { 101 accumulate(&vfs.FsckLog{ 102 Type: vfs.ThumbnailWithNoFile, 103 IsFile: true, 104 FileDoc: &vfs.TreeFile{ 105 DirOrFileDoc: vfs.DirOrFileDoc{ 106 DirDoc: &vfs.DirDoc{ 107 Type: consts.FileType, 108 DocID: fileID, 109 DocName: obj.Name, 110 }, 111 }, 112 }, 113 }) 114 if failFast { 115 return nil, errFailFast 116 } 117 } 118 } 119 continue 120 } 121 docID, internalID := makeDocIDV3(obj.Name) 122 if v, ok := versions[docID+"/"+internalID]; ok { 123 var md5sum []byte 124 md5sum, err = hex.DecodeString(obj.Hash) 125 if err != nil { 126 return nil, err 127 } 128 if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Bytes { 129 accumulate(&vfs.FsckLog{ 130 Type: vfs.ContentMismatch, 131 IsVersion: true, 132 VersionDoc: v, 133 ContentMismatch: &vfs.FsckContentMismatch{ 134 SizeFile: obj.Bytes, 135 SizeIndex: v.ByteSize, 136 MD5SumFile: md5sum, 137 MD5SumIndex: v.MD5Sum, 138 }, 139 }) 140 if failFast { 141 return nil, errFailFast 142 } 143 } 144 delete(versions, v.DocID) 145 continue 146 } 147 f, ok := entries[docID+"/"+internalID] 148 if !ok { 149 accumulate(&vfs.FsckLog{ 150 Type: vfs.IndexMissing, 151 IsFile: true, 152 FileDoc: objectToFileDocV3(sfs.container, obj), 153 }) 154 if failFast { 155 return nil, errFailFast 156 } 157 } else { 158 var md5sum []byte 159 md5sum, err = hex.DecodeString(obj.Hash) 160 if err != nil { 161 return nil, err 162 } 163 if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Bytes { 164 accumulate(&vfs.FsckLog{ 165 Type: vfs.ContentMismatch, 166 IsFile: true, 167 FileDoc: f, 168 ContentMismatch: &vfs.FsckContentMismatch{ 169 SizeFile: obj.Bytes, 170 SizeIndex: f.ByteSize, 171 MD5SumFile: md5sum, 172 MD5SumIndex: f.MD5Sum, 173 }, 174 }) 175 if failFast { 176 return nil, errFailFast 177 } 178 } 179 delete(entries, docID+"/"+internalID) 180 } 181 } 182 return objs, nil 183 }) 184 if err != nil { 185 if errors.Is(err, errFailFast) { 186 return nil 187 } 188 return err 189 } 190 191 // entries should contain only data that does not contain an associated 192 // index. 193 for _, f := range entries { 194 accumulate(&vfs.FsckLog{ 195 Type: vfs.FSMissing, 196 IsFile: true, 197 FileDoc: f, 198 }) 199 if failFast { 200 return nil 201 } 202 } 203 204 for _, v := range versions { 205 accumulate(&vfs.FsckLog{ 206 Type: vfs.FSMissing, 207 IsVersion: true, 208 VersionDoc: v, 209 }) 210 if failFast { 211 return nil 212 } 213 } 214 215 return nil 216 } 217 218 func objectToFileDocV3(container string, object swift.Object) *vfs.TreeFile { 219 md5sum, _ := hex.DecodeString(object.Hash) 220 name := "unknown" 221 mime, class := vfs.ExtractMimeAndClass(object.ContentType) 222 fileID, internalID := makeDocIDV3(object.Name) 223 return &vfs.TreeFile{ 224 DirOrFileDoc: vfs.DirOrFileDoc{ 225 DirDoc: &vfs.DirDoc{ 226 Type: consts.FileType, 227 DocID: fileID, 228 DocName: name, 229 DirID: "", 230 CreatedAt: object.LastModified, 231 UpdatedAt: object.LastModified, 232 Fullpath: path.Join(vfs.OrphansDirName, name), 233 }, 234 ByteSize: object.Bytes, 235 Mime: mime, 236 Class: class, 237 MD5Sum: md5sum, 238 InternalID: internalID, 239 }, 240 } 241 }