github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/model/vfs/vfsswift/fsck_v3.go (about)

     1  package vfsswift
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"encoding/hex"
     7  	"encoding/json"
     8  	"errors"
     9  	"path"
    10  	"strings"
    11  
    12  	"github.com/cozy/cozy-stack/model/vfs"
    13  	"github.com/cozy/cozy-stack/pkg/consts"
    14  	"github.com/cozy/cozy-stack/pkg/couchdb"
    15  	"github.com/ncw/swift/v2"
    16  )
    17  
    18  func (sfs *swiftVFSV3) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error {
    19  	entries := make(map[string]*vfs.TreeFile, 1024)
    20  	tree, err := sfs.BuildTree(func(f *vfs.TreeFile) {
    21  		if !f.IsDir {
    22  			entries[f.DocID+"/"+f.InternalID] = f
    23  		}
    24  	})
    25  	if err != nil {
    26  		return err
    27  	}
    28  	if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil {
    29  		if errors.Is(err, vfs.ErrFsckFailFast) {
    30  			return nil
    31  		}
    32  		return err
    33  	}
    34  	return sfs.checkFiles(entries, accumulate, failFast)
    35  }
    36  
    37  func (sfs *swiftVFSV3) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error {
    38  	entries := make(map[string]*vfs.TreeFile, 1024)
    39  	_, err := sfs.BuildTree(func(f *vfs.TreeFile) {
    40  		if !f.IsDir {
    41  			entries[f.DocID+"/"+f.InternalID] = f
    42  		}
    43  	})
    44  	if err != nil {
    45  		return err
    46  	}
    47  	return sfs.checkFiles(entries, accumulate, failFast)
    48  }
    49  
    50  func (sfs *swiftVFSV3) checkFiles(
    51  	entries map[string]*vfs.TreeFile,
    52  	accumulate func(log *vfs.FsckLog),
    53  	failFast bool,
    54  ) error {
    55  	versions := make(map[string]*vfs.Version, 1024)
    56  	err := couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error {
    57  		v := &vfs.Version{}
    58  		if erru := json.Unmarshal(data, v); erru != nil {
    59  			return erru
    60  		}
    61  		versions[v.DocID] = v
    62  		return nil
    63  	})
    64  	if err != nil {
    65  		return err
    66  	}
    67  
    68  	images := make(map[string]struct{})
    69  	err = couchdb.ForeachDocs(sfs, consts.NotesImages, func(_ string, data json.RawMessage) error {
    70  		img := make(map[string]interface{})
    71  		if erru := json.Unmarshal(data, &img); erru != nil {
    72  			return erru
    73  		}
    74  		id, _ := img["_id"].(string)
    75  		images[id] = struct{}{}
    76  		return nil
    77  	})
    78  	if err != nil && !couchdb.IsNoDatabaseError(err) {
    79  		return err
    80  	}
    81  
    82  	fileIDs := make(map[string]struct{}, len(entries))
    83  	for _, f := range entries {
    84  		fileIDs[f.DocID] = struct{}{}
    85  	}
    86  
    87  	opts := &swift.ObjectsOpts{Limit: 5_000}
    88  	err = sfs.c.ObjectsWalk(sfs.ctx, sfs.container, opts, func(ctx context.Context, opts *swift.ObjectsOpts) (interface{}, error) {
    89  		objs, err := sfs.c.Objects(sfs.ctx, sfs.container, opts)
    90  		if err != nil {
    91  			return nil, err
    92  		}
    93  		for _, obj := range objs {
    94  			if strings.HasPrefix(obj.Name, "thumbs/") {
    95  				objName := strings.TrimPrefix(obj.Name, "thumbs/")
    96  				idx := strings.LastIndex(objName, "-")
    97  				objName = objName[0:idx] // Remove -format suffix
    98  				fileID := makeDocID(objName)
    99  				if _, ok := fileIDs[fileID]; !ok {
   100  					if _, ok := images[fileID]; !ok {
   101  						accumulate(&vfs.FsckLog{
   102  							Type:   vfs.ThumbnailWithNoFile,
   103  							IsFile: true,
   104  							FileDoc: &vfs.TreeFile{
   105  								DirOrFileDoc: vfs.DirOrFileDoc{
   106  									DirDoc: &vfs.DirDoc{
   107  										Type:    consts.FileType,
   108  										DocID:   fileID,
   109  										DocName: obj.Name,
   110  									},
   111  								},
   112  							},
   113  						})
   114  						if failFast {
   115  							return nil, errFailFast
   116  						}
   117  					}
   118  				}
   119  				continue
   120  			}
   121  			docID, internalID := makeDocIDV3(obj.Name)
   122  			if v, ok := versions[docID+"/"+internalID]; ok {
   123  				var md5sum []byte
   124  				md5sum, err = hex.DecodeString(obj.Hash)
   125  				if err != nil {
   126  					return nil, err
   127  				}
   128  				if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Bytes {
   129  					accumulate(&vfs.FsckLog{
   130  						Type:       vfs.ContentMismatch,
   131  						IsVersion:  true,
   132  						VersionDoc: v,
   133  						ContentMismatch: &vfs.FsckContentMismatch{
   134  							SizeFile:    obj.Bytes,
   135  							SizeIndex:   v.ByteSize,
   136  							MD5SumFile:  md5sum,
   137  							MD5SumIndex: v.MD5Sum,
   138  						},
   139  					})
   140  					if failFast {
   141  						return nil, errFailFast
   142  					}
   143  				}
   144  				delete(versions, v.DocID)
   145  				continue
   146  			}
   147  			f, ok := entries[docID+"/"+internalID]
   148  			if !ok {
   149  				accumulate(&vfs.FsckLog{
   150  					Type:    vfs.IndexMissing,
   151  					IsFile:  true,
   152  					FileDoc: objectToFileDocV3(sfs.container, obj),
   153  				})
   154  				if failFast {
   155  					return nil, errFailFast
   156  				}
   157  			} else {
   158  				var md5sum []byte
   159  				md5sum, err = hex.DecodeString(obj.Hash)
   160  				if err != nil {
   161  					return nil, err
   162  				}
   163  				if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Bytes {
   164  					accumulate(&vfs.FsckLog{
   165  						Type:    vfs.ContentMismatch,
   166  						IsFile:  true,
   167  						FileDoc: f,
   168  						ContentMismatch: &vfs.FsckContentMismatch{
   169  							SizeFile:    obj.Bytes,
   170  							SizeIndex:   f.ByteSize,
   171  							MD5SumFile:  md5sum,
   172  							MD5SumIndex: f.MD5Sum,
   173  						},
   174  					})
   175  					if failFast {
   176  						return nil, errFailFast
   177  					}
   178  				}
   179  				delete(entries, docID+"/"+internalID)
   180  			}
   181  		}
   182  		return objs, nil
   183  	})
   184  	if err != nil {
   185  		if errors.Is(err, errFailFast) {
   186  			return nil
   187  		}
   188  		return err
   189  	}
   190  
   191  	// entries should contain only data that does not contain an associated
   192  	// index.
   193  	for _, f := range entries {
   194  		accumulate(&vfs.FsckLog{
   195  			Type:    vfs.FSMissing,
   196  			IsFile:  true,
   197  			FileDoc: f,
   198  		})
   199  		if failFast {
   200  			return nil
   201  		}
   202  	}
   203  
   204  	for _, v := range versions {
   205  		accumulate(&vfs.FsckLog{
   206  			Type:       vfs.FSMissing,
   207  			IsVersion:  true,
   208  			VersionDoc: v,
   209  		})
   210  		if failFast {
   211  			return nil
   212  		}
   213  	}
   214  
   215  	return nil
   216  }
   217  
   218  func objectToFileDocV3(container string, object swift.Object) *vfs.TreeFile {
   219  	md5sum, _ := hex.DecodeString(object.Hash)
   220  	name := "unknown"
   221  	mime, class := vfs.ExtractMimeAndClass(object.ContentType)
   222  	fileID, internalID := makeDocIDV3(object.Name)
   223  	return &vfs.TreeFile{
   224  		DirOrFileDoc: vfs.DirOrFileDoc{
   225  			DirDoc: &vfs.DirDoc{
   226  				Type:      consts.FileType,
   227  				DocID:     fileID,
   228  				DocName:   name,
   229  				DirID:     "",
   230  				CreatedAt: object.LastModified,
   231  				UpdatedAt: object.LastModified,
   232  				Fullpath:  path.Join(vfs.OrphansDirName, name),
   233  			},
   234  			ByteSize:   object.Bytes,
   235  			Mime:       mime,
   236  			Class:      class,
   237  			MD5Sum:     md5sum,
   238  			InternalID: internalID,
   239  		},
   240  	}
   241  }