github.com/moby/docker@v26.1.3+incompatible/layer/filestore.go (about) 1 package layer // import "github.com/docker/docker/layer" 2 3 import ( 4 "compress/gzip" 5 "context" 6 "encoding/json" 7 "io" 8 "os" 9 "path/filepath" 10 "regexp" 11 "strconv" 12 "strings" 13 14 "github.com/containerd/log" 15 "github.com/docker/distribution" 16 "github.com/docker/docker/pkg/ioutils" 17 "github.com/opencontainers/go-digest" 18 "github.com/pkg/errors" 19 ) 20 21 var ( 22 stringIDRegexp = regexp.MustCompile(`^[a-f0-9]{64}(-init)?$`) 23 supportedAlgorithms = []digest.Algorithm{ 24 digest.SHA256, 25 // digest.SHA384, // Currently not used 26 // digest.SHA512, // Currently not used 27 } 28 ) 29 30 type fileMetadataStore struct { 31 root string 32 } 33 34 type fileMetadataTransaction struct { 35 store *fileMetadataStore 36 ws *ioutils.AtomicWriteSet 37 } 38 39 // newFSMetadataStore returns an instance of a metadata store 40 // which is backed by files on disk using the provided root 41 // as the root of metadata files. 42 func newFSMetadataStore(root string) (*fileMetadataStore, error) { 43 if err := os.MkdirAll(root, 0o700); err != nil { 44 return nil, err 45 } 46 return &fileMetadataStore{ 47 root: root, 48 }, nil 49 } 50 51 func (fms *fileMetadataStore) getLayerDirectory(layer ChainID) string { 52 dgst := digest.Digest(layer) 53 return filepath.Join(fms.root, string(dgst.Algorithm()), dgst.Encoded()) 54 } 55 56 func (fms *fileMetadataStore) getLayerFilename(layer ChainID, filename string) string { 57 return filepath.Join(fms.getLayerDirectory(layer), filename) 58 } 59 60 func (fms *fileMetadataStore) getMountDirectory(mount string) string { 61 return filepath.Join(fms.root, "mounts", mount) 62 } 63 64 func (fms *fileMetadataStore) getMountFilename(mount, filename string) string { 65 return filepath.Join(fms.getMountDirectory(mount), filename) 66 } 67 68 func (fms *fileMetadataStore) StartTransaction() (*fileMetadataTransaction, error) { 69 tmpDir := filepath.Join(fms.root, "tmp") 70 if err := os.MkdirAll(tmpDir, 0o755); err != nil { 71 return nil, err 72 } 73 ws, err := ioutils.NewAtomicWriteSet(tmpDir) 74 if err != nil { 75 return nil, err 76 } 77 78 return &fileMetadataTransaction{ 79 store: fms, 80 ws: ws, 81 }, nil 82 } 83 84 func (fm *fileMetadataTransaction) SetSize(size int64) error { 85 return fm.ws.WriteFile("size", []byte(strconv.FormatInt(size, 10)), 0o644) 86 } 87 88 func (fm *fileMetadataTransaction) SetParent(parent ChainID) error { 89 return fm.ws.WriteFile("parent", []byte(digest.Digest(parent).String()), 0o644) 90 } 91 92 func (fm *fileMetadataTransaction) SetDiffID(diff DiffID) error { 93 return fm.ws.WriteFile("diff", []byte(digest.Digest(diff).String()), 0o644) 94 } 95 96 func (fm *fileMetadataTransaction) SetCacheID(cacheID string) error { 97 return fm.ws.WriteFile("cache-id", []byte(cacheID), 0o644) 98 } 99 100 func (fm *fileMetadataTransaction) SetDescriptor(ref distribution.Descriptor) error { 101 jsonRef, err := json.Marshal(ref) 102 if err != nil { 103 return err 104 } 105 return fm.ws.WriteFile("descriptor.json", jsonRef, 0o644) 106 } 107 108 func (fm *fileMetadataTransaction) TarSplitWriter(compressInput bool) (io.WriteCloser, error) { 109 f, err := fm.ws.FileWriter("tar-split.json.gz", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0o644) 110 if err != nil { 111 return nil, err 112 } 113 var wc io.WriteCloser 114 if compressInput { 115 wc = gzip.NewWriter(f) 116 } else { 117 wc = f 118 } 119 120 return ioutils.NewWriteCloserWrapper(wc, func() error { 121 wc.Close() 122 return f.Close() 123 }), nil 124 } 125 126 func (fm *fileMetadataTransaction) Commit(layer ChainID) error { 127 finalDir := fm.store.getLayerDirectory(layer) 128 if err := os.MkdirAll(filepath.Dir(finalDir), 0o755); err != nil { 129 return err 130 } 131 132 return fm.ws.Commit(finalDir) 133 } 134 135 func (fm *fileMetadataTransaction) Cancel() error { 136 return fm.ws.Cancel() 137 } 138 139 func (fm *fileMetadataTransaction) String() string { 140 return fm.ws.String() 141 } 142 143 func (fms *fileMetadataStore) GetSize(layer ChainID) (int64, error) { 144 content, err := os.ReadFile(fms.getLayerFilename(layer, "size")) 145 if err != nil { 146 return 0, err 147 } 148 149 size, err := strconv.ParseInt(string(content), 10, 64) 150 if err != nil { 151 return 0, err 152 } 153 154 return size, nil 155 } 156 157 func (fms *fileMetadataStore) GetParent(layer ChainID) (ChainID, error) { 158 content, err := os.ReadFile(fms.getLayerFilename(layer, "parent")) 159 if err != nil { 160 if os.IsNotExist(err) { 161 return "", nil 162 } 163 return "", err 164 } 165 166 dgst, err := digest.Parse(strings.TrimSpace(string(content))) 167 if err != nil { 168 return "", err 169 } 170 171 return ChainID(dgst), nil 172 } 173 174 func (fms *fileMetadataStore) GetDiffID(layer ChainID) (DiffID, error) { 175 content, err := os.ReadFile(fms.getLayerFilename(layer, "diff")) 176 if err != nil { 177 return "", err 178 } 179 180 dgst, err := digest.Parse(strings.TrimSpace(string(content))) 181 if err != nil { 182 return "", err 183 } 184 185 return DiffID(dgst), nil 186 } 187 188 func (fms *fileMetadataStore) GetCacheID(layer ChainID) (string, error) { 189 contentBytes, err := os.ReadFile(fms.getLayerFilename(layer, "cache-id")) 190 if err != nil { 191 return "", err 192 } 193 content := strings.TrimSpace(string(contentBytes)) 194 195 if content == "" { 196 return "", errors.Errorf("invalid cache id value") 197 } 198 199 return content, nil 200 } 201 202 func (fms *fileMetadataStore) GetDescriptor(layer ChainID) (distribution.Descriptor, error) { 203 content, err := os.ReadFile(fms.getLayerFilename(layer, "descriptor.json")) 204 if err != nil { 205 if os.IsNotExist(err) { 206 // only return empty descriptor to represent what is stored 207 return distribution.Descriptor{}, nil 208 } 209 return distribution.Descriptor{}, err 210 } 211 212 var ref distribution.Descriptor 213 err = json.Unmarshal(content, &ref) 214 if err != nil { 215 return distribution.Descriptor{}, err 216 } 217 return ref, err 218 } 219 220 func (fms *fileMetadataStore) TarSplitReader(layer ChainID) (io.ReadCloser, error) { 221 fz, err := os.Open(fms.getLayerFilename(layer, "tar-split.json.gz")) 222 if err != nil { 223 return nil, err 224 } 225 f, err := gzip.NewReader(fz) 226 if err != nil { 227 fz.Close() 228 return nil, err 229 } 230 231 return ioutils.NewReadCloserWrapper(f, func() error { 232 f.Close() 233 return fz.Close() 234 }), nil 235 } 236 237 func (fms *fileMetadataStore) SetMountID(mount string, mountID string) error { 238 if err := os.MkdirAll(fms.getMountDirectory(mount), 0o755); err != nil { 239 return err 240 } 241 return os.WriteFile(fms.getMountFilename(mount, "mount-id"), []byte(mountID), 0o644) 242 } 243 244 func (fms *fileMetadataStore) SetInitID(mount string, init string) error { 245 if err := os.MkdirAll(fms.getMountDirectory(mount), 0o755); err != nil { 246 return err 247 } 248 return os.WriteFile(fms.getMountFilename(mount, "init-id"), []byte(init), 0o644) 249 } 250 251 func (fms *fileMetadataStore) SetMountParent(mount string, parent ChainID) error { 252 if err := os.MkdirAll(fms.getMountDirectory(mount), 0o755); err != nil { 253 return err 254 } 255 return os.WriteFile(fms.getMountFilename(mount, "parent"), []byte(digest.Digest(parent).String()), 0o644) 256 } 257 258 func (fms *fileMetadataStore) GetMountID(mount string) (string, error) { 259 contentBytes, err := os.ReadFile(fms.getMountFilename(mount, "mount-id")) 260 if err != nil { 261 return "", err 262 } 263 content := strings.TrimSpace(string(contentBytes)) 264 265 if !stringIDRegexp.MatchString(content) { 266 return "", errors.New("invalid mount id value") 267 } 268 269 return content, nil 270 } 271 272 func (fms *fileMetadataStore) GetInitID(mount string) (string, error) { 273 contentBytes, err := os.ReadFile(fms.getMountFilename(mount, "init-id")) 274 if err != nil { 275 if os.IsNotExist(err) { 276 return "", nil 277 } 278 return "", err 279 } 280 content := strings.TrimSpace(string(contentBytes)) 281 282 if !stringIDRegexp.MatchString(content) { 283 return "", errors.New("invalid init id value") 284 } 285 286 return content, nil 287 } 288 289 func (fms *fileMetadataStore) GetMountParent(mount string) (ChainID, error) { 290 content, err := os.ReadFile(fms.getMountFilename(mount, "parent")) 291 if err != nil { 292 if os.IsNotExist(err) { 293 return "", nil 294 } 295 return "", err 296 } 297 298 dgst, err := digest.Parse(strings.TrimSpace(string(content))) 299 if err != nil { 300 return "", err 301 } 302 303 return ChainID(dgst), nil 304 } 305 306 func (fms *fileMetadataStore) getOrphan() ([]roLayer, error) { 307 var orphanLayers []roLayer 308 for _, algorithm := range supportedAlgorithms { 309 fileInfos, err := os.ReadDir(filepath.Join(fms.root, string(algorithm))) 310 if err != nil { 311 if os.IsNotExist(err) { 312 continue 313 } 314 return nil, err 315 } 316 317 for _, fi := range fileInfos { 318 if !fi.IsDir() || !strings.HasSuffix(fi.Name(), "-removing") { 319 continue 320 } 321 // At this stage, fi.Name value looks like <digest>-<random>-removing 322 // Split on '-' to get the digest value. 323 nameSplit := strings.Split(fi.Name(), "-") 324 dgst := digest.NewDigestFromEncoded(algorithm, nameSplit[0]) 325 if err := dgst.Validate(); err != nil { 326 log.G(context.TODO()).WithError(err).WithField("digest", string(algorithm)+":"+nameSplit[0]).Debug("ignoring invalid digest") 327 continue 328 } 329 330 chainFile := filepath.Join(fms.root, string(algorithm), fi.Name(), "cache-id") 331 contentBytes, err := os.ReadFile(chainFile) 332 if err != nil { 333 if !os.IsNotExist(err) { 334 log.G(context.TODO()).WithError(err).WithField("digest", dgst).Error("failed to read cache ID") 335 } 336 continue 337 } 338 cacheID := strings.TrimSpace(string(contentBytes)) 339 if cacheID == "" { 340 log.G(context.TODO()).Error("invalid cache ID") 341 continue 342 } 343 344 l := &roLayer{ 345 chainID: ChainID(dgst), 346 cacheID: cacheID, 347 } 348 orphanLayers = append(orphanLayers, *l) 349 } 350 } 351 352 return orphanLayers, nil 353 } 354 355 func (fms *fileMetadataStore) List() ([]ChainID, []string, error) { 356 var ids []ChainID 357 for _, algorithm := range supportedAlgorithms { 358 fileInfos, err := os.ReadDir(filepath.Join(fms.root, string(algorithm))) 359 if err != nil { 360 if os.IsNotExist(err) { 361 continue 362 } 363 return nil, nil, err 364 } 365 366 for _, fi := range fileInfos { 367 if fi.IsDir() && fi.Name() != "mounts" { 368 dgst := digest.NewDigestFromEncoded(algorithm, fi.Name()) 369 if err := dgst.Validate(); err != nil { 370 log.G(context.TODO()).Debugf("Ignoring invalid digest %s:%s", algorithm, fi.Name()) 371 } else { 372 ids = append(ids, ChainID(dgst)) 373 } 374 } 375 } 376 } 377 378 fileInfos, err := os.ReadDir(filepath.Join(fms.root, "mounts")) 379 if err != nil { 380 if os.IsNotExist(err) { 381 return ids, []string{}, nil 382 } 383 return nil, nil, err 384 } 385 386 var mounts []string 387 for _, fi := range fileInfos { 388 if fi.IsDir() { 389 mounts = append(mounts, fi.Name()) 390 } 391 } 392 393 return ids, mounts, nil 394 } 395 396 // Remove layerdb folder if that is marked for removal 397 func (fms *fileMetadataStore) Remove(layer ChainID, cache string) error { 398 dgst := digest.Digest(layer) 399 files, err := os.ReadDir(filepath.Join(fms.root, string(dgst.Algorithm()))) 400 if err != nil { 401 return err 402 } 403 for _, f := range files { 404 if !strings.HasSuffix(f.Name(), "-removing") || !strings.HasPrefix(f.Name(), dgst.Encoded()) { 405 continue 406 } 407 408 // Make sure that we only remove layerdb folder which points to 409 // requested cacheID 410 dir := filepath.Join(fms.root, string(dgst.Algorithm()), f.Name()) 411 chainFile := filepath.Join(dir, "cache-id") 412 contentBytes, err := os.ReadFile(chainFile) 413 if err != nil { 414 log.G(context.TODO()).WithError(err).WithField("file", chainFile).Error("cannot get cache ID") 415 continue 416 } 417 cacheID := strings.TrimSpace(string(contentBytes)) 418 if cacheID != cache { 419 continue 420 } 421 log.G(context.TODO()).Debugf("Removing folder: %s", dir) 422 err = os.RemoveAll(dir) 423 if err != nil && !os.IsNotExist(err) { 424 log.G(context.TODO()).WithError(err).WithField("name", f.Name()).Error("cannot remove layer") 425 continue 426 } 427 } 428 return nil 429 } 430 431 func (fms *fileMetadataStore) RemoveMount(mount string) error { 432 return os.RemoveAll(fms.getMountDirectory(mount)) 433 }