github.com/demonoid81/moby@v0.0.0-20200517203328-62dd8e17c460/layer/filestore.go (about) 1 package layer // import "github.com/demonoid81/moby/layer" 2 3 import ( 4 "compress/gzip" 5 "encoding/json" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "os" 10 "path/filepath" 11 "regexp" 12 "strconv" 13 "strings" 14 15 "github.com/docker/distribution" 16 "github.com/demonoid81/moby/pkg/ioutils" 17 digest "github.com/opencontainers/go-digest" 18 "github.com/pkg/errors" 19 "github.com/sirupsen/logrus" 20 ) 21 22 var ( 23 stringIDRegexp = regexp.MustCompile(`^[a-f0-9]{64}(-init)?$`) 24 supportedAlgorithms = []digest.Algorithm{ 25 digest.SHA256, 26 // digest.SHA384, // Currently not used 27 // digest.SHA512, // Currently not used 28 } 29 ) 30 31 type fileMetadataStore struct { 32 root string 33 } 34 35 type fileMetadataTransaction struct { 36 store *fileMetadataStore 37 ws *ioutils.AtomicWriteSet 38 } 39 40 // newFSMetadataStore returns an instance of a metadata store 41 // which is backed by files on disk using the provided root 42 // as the root of metadata files. 43 func newFSMetadataStore(root string) (*fileMetadataStore, error) { 44 if err := os.MkdirAll(root, 0700); err != nil { 45 return nil, err 46 } 47 return &fileMetadataStore{ 48 root: root, 49 }, nil 50 } 51 52 func (fms *fileMetadataStore) getLayerDirectory(layer ChainID) string { 53 dgst := digest.Digest(layer) 54 return filepath.Join(fms.root, string(dgst.Algorithm()), dgst.Hex()) 55 } 56 57 func (fms *fileMetadataStore) getLayerFilename(layer ChainID, filename string) string { 58 return filepath.Join(fms.getLayerDirectory(layer), filename) 59 } 60 61 func (fms *fileMetadataStore) getMountDirectory(mount string) string { 62 return filepath.Join(fms.root, "mounts", mount) 63 } 64 65 func (fms *fileMetadataStore) getMountFilename(mount, filename string) string { 66 return filepath.Join(fms.getMountDirectory(mount), filename) 67 } 68 69 func (fms *fileMetadataStore) StartTransaction() (*fileMetadataTransaction, error) { 70 tmpDir := filepath.Join(fms.root, "tmp") 71 if err := os.MkdirAll(tmpDir, 0755); err != nil { 72 return nil, err 73 } 74 ws, err := ioutils.NewAtomicWriteSet(tmpDir) 75 if err != nil { 76 return nil, err 77 } 78 79 return &fileMetadataTransaction{ 80 store: fms, 81 ws: ws, 82 }, nil 83 } 84 85 func (fm *fileMetadataTransaction) SetSize(size int64) error { 86 content := fmt.Sprintf("%d", size) 87 return fm.ws.WriteFile("size", []byte(content), 0644) 88 } 89 90 func (fm *fileMetadataTransaction) SetParent(parent ChainID) error { 91 return fm.ws.WriteFile("parent", []byte(digest.Digest(parent).String()), 0644) 92 } 93 94 func (fm *fileMetadataTransaction) SetDiffID(diff DiffID) error { 95 return fm.ws.WriteFile("diff", []byte(digest.Digest(diff).String()), 0644) 96 } 97 98 func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error { 99 return fm.ws.WriteFile("cache-id", []byte(cacheID), 0644) 100 } 101 102 func (fm *fileMetadataTransaction) SetDescriptor(ref distribution.Descriptor) error { 103 jsonRef, err := json.Marshal(ref) 104 if err != nil { 105 return err 106 } 107 return fm.ws.WriteFile("descriptor.json", jsonRef, 0644) 108 } 109 110 func (fm *fileMetadataTransaction) TarSplitWriter(compressInput bool) (io.WriteCloser, error) { 111 f, err := fm.ws.FileWriter("tar-split.json.gz", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644) 112 if err != nil { 113 return nil, err 114 } 115 var wc io.WriteCloser 116 if compressInput { 117 wc = gzip.NewWriter(f) 118 } else { 119 wc = f 120 } 121 122 return ioutils.NewWriteCloserWrapper(wc, func() error { 123 wc.Close() 124 return f.Close() 125 }), nil 126 } 127 128 func (fm *fileMetadataTransaction) Commit(layer ChainID) error { 129 finalDir := fm.store.getLayerDirectory(layer) 130 if err := os.MkdirAll(filepath.Dir(finalDir), 0755); err != nil { 131 return err 132 } 133 134 return fm.ws.Commit(finalDir) 135 } 136 137 func (fm *fileMetadataTransaction) Cancel() error { 138 return fm.ws.Cancel() 139 } 140 141 func (fm *fileMetadataTransaction) String() string { 142 return fm.ws.String() 143 } 144 145 func (fms *fileMetadataStore) GetSize(layer ChainID) (int64, error) { 146 content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "size")) 147 if err != nil { 148 return 0, err 149 } 150 151 size, err := strconv.ParseInt(string(content), 10, 64) 152 if err != nil { 153 return 0, err 154 } 155 156 return size, nil 157 } 158 159 func (fms *fileMetadataStore) GetParent(layer ChainID) (ChainID, error) { 160 content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "parent")) 161 if err != nil { 162 if os.IsNotExist(err) { 163 return "", nil 164 } 165 return "", err 166 } 167 168 dgst, err := digest.Parse(strings.TrimSpace(string(content))) 169 if err != nil { 170 return "", err 171 } 172 173 return ChainID(dgst), nil 174 } 175 176 func (fms *fileMetadataStore) GetDiffID(layer ChainID) (DiffID, error) { 177 content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "diff")) 178 if err != nil { 179 return "", err 180 } 181 182 dgst, err := digest.Parse(strings.TrimSpace(string(content))) 183 if err != nil { 184 return "", err 185 } 186 187 return DiffID(dgst), nil 188 } 189 190 func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) { 191 contentBytes, err := ioutil.ReadFile(fms.getLayerFilename(layer, "cache-id")) 192 if err != nil { 193 return "", err 194 } 195 content := strings.TrimSpace(string(contentBytes)) 196 197 if content == "" { 198 return "", errors.Errorf("invalid cache id value") 199 } 200 201 return content, nil 202 } 203 204 func (fms *fileMetadataStore) GetDescriptor(layer ChainID) (distribution.Descriptor, error) { 205 content, err := ioutil.ReadFile(fms.getLayerFilename(layer, "descriptor.json")) 206 if err != nil { 207 if os.IsNotExist(err) { 208 // only return empty descriptor to represent what is stored 209 return distribution.Descriptor{}, nil 210 } 211 return distribution.Descriptor{}, err 212 } 213 214 var ref distribution.Descriptor 215 err = json.Unmarshal(content, &ref) 216 if err != nil { 217 return distribution.Descriptor{}, err 218 } 219 return ref, err 220 } 221 222 func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) { 223 fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz")) 224 if err != nil { 225 return nil, err 226 } 227 f, err := gzip.NewReader(fz) 228 if err != nil { 229 fz.Close() 230 return nil, err 231 } 232 233 return ioutils.NewReadCloserWrapper(f, func() error { 234 f.Close() 235 return fz.Close() 236 }), nil 237 } 238 239 func (fms *fileMetadataStore) SetMountID(mount string, mountID string) error { 240 if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil { 241 return err 242 } 243 return ioutil.WriteFile(fms.getMountFilename(mount, "mount-id"), []byte(mountID), 0644) 244 } 245 246 func (fms *fileMetadataStore) SetInitID(mount string, init string) error { 247 if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil { 248 return err 249 } 250 return ioutil.WriteFile(fms.getMountFilename(mount, "init-id"), []byte(init), 0644) 251 } 252 253 func (fms *fileMetadataStore) SetMountParent(mount string, parent ChainID) error { 254 if err := os.MkdirAll(fms.getMountDirectory(mount), 0755); err != nil { 255 return err 256 } 257 return ioutil.WriteFile(fms.getMountFilename(mount, "parent"), []byte(digest.Digest(parent).String()), 0644) 258 } 259 260 func (fms *fileMetadataStore) GetMountID(mount string) (string, error) { 261 contentBytes, err := ioutil.ReadFile(fms.getMountFilename(mount, "mount-id")) 262 if err != nil { 263 return "", err 264 } 265 content := strings.TrimSpace(string(contentBytes)) 266 267 if !stringIDRegexp.MatchString(content) { 268 return "", errors.New("invalid mount id value") 269 } 270 271 return content, nil 272 } 273 274 func (fms *fileMetadataStore) GetInitID(mount string) (string, error) { 275 contentBytes, err := ioutil.ReadFile(fms.getMountFilename(mount, "init-id")) 276 if err != nil { 277 if os.IsNotExist(err) { 278 return "", nil 279 } 280 return "", err 281 } 282 content := strings.TrimSpace(string(contentBytes)) 283 284 if !stringIDRegexp.MatchString(content) { 285 return "", errors.New("invalid init id value") 286 } 287 288 return content, nil 289 } 290 291 func (fms *fileMetadataStore) GetMountParent(mount string) (ChainID, error) { 292 content, err := ioutil.ReadFile(fms.getMountFilename(mount, "parent")) 293 if err != nil { 294 if os.IsNotExist(err) { 295 return "", nil 296 } 297 return "", err 298 } 299 300 dgst, err := digest.Parse(strings.TrimSpace(string(content))) 301 if err != nil { 302 return "", err 303 } 304 305 return ChainID(dgst), nil 306 } 307 308 func (fms *fileMetadataStore) getOrphan() ([]roLayer, error) { 309 var orphanLayers []roLayer 310 for _, algorithm := range supportedAlgorithms { 311 fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, string(algorithm))) 312 if err != nil { 313 if os.IsNotExist(err) { 314 continue 315 } 316 return nil, err 317 } 318 319 for _, fi := range fileInfos { 320 if !fi.IsDir() || !strings.HasSuffix(fi.Name(), "-removing") { 321 continue 322 } 323 // At this stage, fi.Name value looks like <digest>-<random>-removing 324 // Split on '-' to get the digest value. 325 nameSplit := strings.Split(fi.Name(), "-") 326 dgst := digest.NewDigestFromEncoded(algorithm, nameSplit[0]) 327 if err := dgst.Validate(); err != nil { 328 logrus.WithError(err).WithField("digest", string(algorithm)+":"+nameSplit[0]).Debug("ignoring invalid digest") 329 continue 330 } 331 332 chainFile := filepath.Join(fms.root, string(algorithm), fi.Name(), "cache-id") 333 contentBytes, err := ioutil.ReadFile(chainFile) 334 if err != nil { 335 if !os.IsNotExist(err) { 336 logrus.WithError(err).WithField("digest", dgst).Error("failed to read cache ID") 337 } 338 continue 339 } 340 cacheID := strings.TrimSpace(string(contentBytes)) 341 if cacheID == "" { 342 logrus.Error("invalid cache ID") 343 continue 344 } 345 346 l := &roLayer{ 347 chainID: ChainID(dgst), 348 cacheID: cacheID, 349 } 350 orphanLayers = append(orphanLayers, *l) 351 } 352 } 353 354 return orphanLayers, nil 355 } 356 357 func (fms *fileMetadataStore) List() ([]ChainID, []string, error) { 358 var ids []ChainID 359 for _, algorithm := range supportedAlgorithms { 360 fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, string(algorithm))) 361 if err != nil { 362 if os.IsNotExist(err) { 363 continue 364 } 365 return nil, nil, err 366 } 367 368 for _, fi := range fileInfos { 369 if fi.IsDir() && fi.Name() != "mounts" { 370 dgst := digest.NewDigestFromHex(string(algorithm), fi.Name()) 371 if err := dgst.Validate(); err != nil { 372 logrus.Debugf("Ignoring invalid digest %s:%s", algorithm, fi.Name()) 373 } else { 374 ids = append(ids, ChainID(dgst)) 375 } 376 } 377 } 378 } 379 380 fileInfos, err := ioutil.ReadDir(filepath.Join(fms.root, "mounts")) 381 if err != nil { 382 if os.IsNotExist(err) { 383 return ids, []string{}, nil 384 } 385 return nil, nil, err 386 } 387 388 var mounts []string 389 for _, fi := range fileInfos { 390 if fi.IsDir() { 391 mounts = append(mounts, fi.Name()) 392 } 393 } 394 395 return ids, mounts, nil 396 } 397 398 // Remove layerdb folder if that is marked for removal 399 func (fms *fileMetadataStore) Remove(layer ChainID, cache string) error { 400 dgst := digest.Digest(layer) 401 files, err := ioutil.ReadDir(filepath.Join(fms.root, string(dgst.Algorithm()))) 402 if err != nil { 403 return err 404 } 405 for _, f := range files { 406 if !strings.HasSuffix(f.Name(), "-removing") || !strings.HasPrefix(f.Name(), dgst.Encoded()) { 407 continue 408 } 409 410 // Make sure that we only remove layerdb folder which points to 411 // requested cacheID 412 dir := filepath.Join(fms.root, string(dgst.Algorithm()), f.Name()) 413 chainFile := filepath.Join(dir, "cache-id") 414 contentBytes, err := ioutil.ReadFile(chainFile) 415 if err != nil { 416 logrus.WithError(err).WithField("file", chainFile).Error("cannot get cache ID") 417 continue 418 } 419 cacheID := strings.TrimSpace(string(contentBytes)) 420 if cacheID != cache { 421 continue 422 } 423 logrus.Debugf("Removing folder: %s", dir) 424 err = os.RemoveAll(dir) 425 if err != nil && !os.IsNotExist(err) { 426 logrus.WithError(err).WithField("name", f.Name()).Error("cannot remove layer") 427 continue 428 } 429 } 430 return nil 431 } 432 433 func (fms *fileMetadataStore) RemoveMount(mount string) error { 434 return os.RemoveAll(fms.getMountDirectory(mount)) 435 }