github.com/docker/docker@v299999999.0.0-20200612211812-aaf470eca7b5+incompatible/builder/builder-next/adapters/snapshot/snapshot.go (about) 1 package snapshot 2 3 import ( 4 "context" 5 "path/filepath" 6 "strconv" 7 "strings" 8 "sync" 9 10 "github.com/containerd/containerd/leases" 11 "github.com/containerd/containerd/mount" 12 "github.com/containerd/containerd/snapshots" 13 "github.com/docker/docker/daemon/graphdriver" 14 "github.com/docker/docker/layer" 15 "github.com/docker/docker/pkg/idtools" 16 "github.com/moby/buildkit/identity" 17 "github.com/moby/buildkit/snapshot" 18 digest "github.com/opencontainers/go-digest" 19 "github.com/pkg/errors" 20 bolt "go.etcd.io/bbolt" 21 ) 22 23 var keyParent = []byte("parent") 24 var keyCommitted = []byte("committed") 25 var keyIsCommitted = []byte("iscommitted") 26 var keyChainID = []byte("chainid") 27 var keySize = []byte("size") 28 29 // Opt defines options for creating the snapshotter 30 type Opt struct { 31 GraphDriver graphdriver.Driver 32 LayerStore layer.Store 33 Root string 34 IdentityMapping *idtools.IdentityMapping 35 } 36 37 type graphIDRegistrar interface { 38 RegisterByGraphID(string, layer.ChainID, layer.DiffID, string, int64) (layer.Layer, error) 39 Release(layer.Layer) ([]layer.Metadata, error) 40 checksumCalculator 41 } 42 43 type checksumCalculator interface { 44 ChecksumForGraphID(id, parent, oldTarDataPath, newTarDataPath string) (diffID layer.DiffID, size int64, err error) 45 } 46 47 type snapshotter struct { 48 opt Opt 49 50 refs map[string]layer.Layer 51 db *bolt.DB 52 mu sync.Mutex 53 reg graphIDRegistrar 54 } 55 56 // NewSnapshotter creates a new snapshotter 57 func NewSnapshotter(opt Opt, prevLM leases.Manager) (snapshot.Snapshotter, leases.Manager, error) { 58 dbPath := filepath.Join(opt.Root, "snapshots.db") 59 db, err := bolt.Open(dbPath, 0600, nil) 60 if err != nil { 61 return nil, nil, errors.Wrapf(err, "failed to open database file %s", dbPath) 62 } 63 64 reg, ok := opt.LayerStore.(graphIDRegistrar) 65 if !ok { 66 return nil, nil, errors.Errorf("layerstore doesn't support graphID registration") 67 } 68 69 s := &snapshotter{ 70 opt: opt, 71 db: db, 72 refs: map[string]layer.Layer{}, 73 reg: reg, 74 } 75 76 lm := newLeaseManager(s, prevLM) 77 78 ll, err := lm.List(context.TODO()) 79 if err != nil { 80 return nil, nil, err 81 } 82 for _, l := range ll { 83 rr, err := lm.ListResources(context.TODO(), l) 84 if err != nil { 85 return nil, nil, err 86 } 87 for _, r := range rr { 88 if r.Type == "snapshots/default" { 89 lm.addRef(l.ID, r.ID) 90 } 91 } 92 } 93 94 return s, lm, nil 95 } 96 97 func (s *snapshotter) Name() string { 98 return "default" 99 } 100 101 func (s *snapshotter) IdentityMapping() *idtools.IdentityMapping { 102 return s.opt.IdentityMapping 103 } 104 105 func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) error { 106 origParent := parent 107 if parent != "" { 108 if l, err := s.getLayer(parent, false); err != nil { 109 return errors.Wrapf(err, "failed to get parent layer %s", parent) 110 } else if l != nil { 111 parent, err = getGraphID(l) 112 if err != nil { 113 return errors.Wrapf(err, "failed to get parent graphid %s", l.ChainID()) 114 } 115 } else { 116 parent, _ = s.getGraphDriverID(parent) 117 } 118 } 119 if err := s.opt.GraphDriver.Create(key, parent, nil); err != nil { 120 return err 121 } 122 return s.db.Update(func(tx *bolt.Tx) error { 123 b, err := tx.CreateBucketIfNotExists([]byte(key)) 124 if err != nil { 125 return err 126 } 127 return b.Put(keyParent, []byte(origParent)) 128 }) 129 } 130 131 func (s *snapshotter) chainID(key string) (layer.ChainID, bool) { 132 if strings.HasPrefix(key, "sha256:") { 133 dgst, err := digest.Parse(key) 134 if err != nil { 135 return "", false 136 } 137 return layer.ChainID(dgst), true 138 } 139 return "", false 140 } 141 142 func (s *snapshotter) GetLayer(key string) (layer.Layer, error) { 143 return s.getLayer(key, true) 144 } 145 146 func (s *snapshotter) getLayer(key string, withCommitted bool) (layer.Layer, error) { 147 s.mu.Lock() 148 l, ok := s.refs[key] 149 if !ok { 150 id, ok := s.chainID(key) 151 if !ok { 152 if !withCommitted { 153 s.mu.Unlock() 154 return nil, nil 155 } 156 if err := s.db.View(func(tx *bolt.Tx) error { 157 b := tx.Bucket([]byte(key)) 158 if b == nil { 159 return nil 160 } 161 v := b.Get(keyChainID) 162 if v != nil { 163 id = layer.ChainID(v) 164 } 165 return nil 166 }); err != nil { 167 s.mu.Unlock() 168 return nil, errors.WithStack(err) 169 } 170 s.mu.Unlock() 171 if id == "" { 172 return nil, nil 173 } 174 return s.getLayer(string(id), withCommitted) 175 } 176 var err error 177 l, err = s.opt.LayerStore.Get(id) 178 if err != nil { 179 s.mu.Unlock() 180 return nil, errors.WithStack(err) 181 } 182 s.refs[key] = l 183 if err := s.db.Update(func(tx *bolt.Tx) error { 184 _, err := tx.CreateBucketIfNotExists([]byte(key)) 185 return errors.WithStack(err) 186 }); err != nil { 187 s.mu.Unlock() 188 return nil, err 189 } 190 } 191 s.mu.Unlock() 192 193 return l, nil 194 } 195 196 func (s *snapshotter) getGraphDriverID(key string) (string, bool) { 197 var gdID string 198 if err := s.db.View(func(tx *bolt.Tx) error { 199 b := tx.Bucket([]byte(key)) 200 if b == nil { 201 return errors.Errorf("not found") // TODO: typed 202 } 203 v := b.Get(keyCommitted) 204 if v != nil { 205 gdID = string(v) 206 } 207 return nil 208 }); err != nil || gdID == "" { 209 return key, false 210 } 211 return gdID, true 212 } 213 214 func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { 215 inf := snapshots.Info{ 216 Kind: snapshots.KindActive, 217 } 218 219 l, err := s.getLayer(key, false) 220 if err != nil { 221 return snapshots.Info{}, err 222 } 223 if l != nil { 224 if p := l.Parent(); p != nil { 225 inf.Parent = p.ChainID().String() 226 } 227 inf.Kind = snapshots.KindCommitted 228 inf.Name = key 229 return inf, nil 230 } 231 232 l, err = s.getLayer(key, true) 233 if err != nil { 234 return snapshots.Info{}, err 235 } 236 237 id, committed := s.getGraphDriverID(key) 238 if committed { 239 inf.Kind = snapshots.KindCommitted 240 } 241 242 if err := s.db.View(func(tx *bolt.Tx) error { 243 b := tx.Bucket([]byte(id)) 244 if b == nil && l == nil { 245 return errors.Errorf("snapshot %s not found", id) // TODO: typed 246 } 247 inf.Name = key 248 if b != nil { 249 v := b.Get(keyParent) 250 if v != nil { 251 inf.Parent = string(v) 252 return nil 253 } 254 } 255 if l != nil { 256 if p := l.Parent(); p != nil { 257 inf.Parent = p.ChainID().String() 258 } 259 inf.Kind = snapshots.KindCommitted 260 } 261 return nil 262 }); err != nil { 263 return snapshots.Info{}, err 264 } 265 return inf, nil 266 } 267 268 func (s *snapshotter) Mounts(ctx context.Context, key string) (snapshot.Mountable, error) { 269 l, err := s.getLayer(key, true) 270 if err != nil { 271 return nil, err 272 } 273 if l != nil { 274 id := identity.NewID() 275 var rwlayer layer.RWLayer 276 return &mountable{ 277 idmap: s.opt.IdentityMapping, 278 acquire: func() ([]mount.Mount, func() error, error) { 279 rwlayer, err = s.opt.LayerStore.CreateRWLayer(id, l.ChainID(), nil) 280 if err != nil { 281 return nil, nil, err 282 } 283 rootfs, err := rwlayer.Mount("") 284 if err != nil { 285 return nil, nil, err 286 } 287 return []mount.Mount{{ 288 Source: rootfs.Path(), 289 Type: "bind", 290 Options: []string{"rbind"}, 291 }}, func() error { 292 _, err := s.opt.LayerStore.ReleaseRWLayer(rwlayer) 293 return err 294 }, nil 295 }, 296 }, nil 297 } 298 299 id, _ := s.getGraphDriverID(key) 300 301 return &mountable{ 302 idmap: s.opt.IdentityMapping, 303 acquire: func() ([]mount.Mount, func() error, error) { 304 rootfs, err := s.opt.GraphDriver.Get(id, "") 305 if err != nil { 306 return nil, nil, err 307 } 308 return []mount.Mount{{ 309 Source: rootfs.Path(), 310 Type: "bind", 311 Options: []string{"rbind"}, 312 }}, func() error { 313 return s.opt.GraphDriver.Put(id) 314 }, nil 315 }, 316 }, nil 317 } 318 319 func (s *snapshotter) Remove(ctx context.Context, key string) error { 320 return errors.Errorf("calling snapshot.remove is forbidden") 321 } 322 323 func (s *snapshotter) remove(ctx context.Context, key string) error { 324 l, err := s.getLayer(key, true) 325 if err != nil { 326 return err 327 } 328 329 id, _ := s.getGraphDriverID(key) 330 331 var found bool 332 var alreadyCommitted bool 333 if err := s.db.Update(func(tx *bolt.Tx) error { 334 b := tx.Bucket([]byte(key)) 335 found = b != nil 336 337 if b != nil { 338 if b.Get(keyIsCommitted) != nil { 339 alreadyCommitted = true 340 return nil 341 } 342 } 343 if found { 344 tx.DeleteBucket([]byte(key)) 345 if id != key { 346 tx.DeleteBucket([]byte(id)) 347 } 348 } 349 return nil 350 }); err != nil { 351 return err 352 } 353 354 if alreadyCommitted { 355 return nil 356 } 357 358 if l != nil { 359 s.mu.Lock() 360 delete(s.refs, key) 361 s.mu.Unlock() 362 _, err := s.opt.LayerStore.Release(l) 363 return err 364 } 365 366 if !found { // this happens when removing views 367 return nil 368 } 369 370 return s.opt.GraphDriver.Remove(id) 371 } 372 373 func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { 374 return s.db.Update(func(tx *bolt.Tx) error { 375 b, err := tx.CreateBucketIfNotExists([]byte(name)) 376 if err != nil { 377 return err 378 } 379 if err := b.Put(keyCommitted, []byte(key)); err != nil { 380 return err 381 } 382 b, err = tx.CreateBucketIfNotExists([]byte(key)) 383 if err != nil { 384 return err 385 } 386 if err := b.Put(keyIsCommitted, []byte{}); err != nil { 387 return err 388 } 389 return nil 390 }) 391 } 392 393 func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) (snapshot.Mountable, error) { 394 return s.Mounts(ctx, parent) 395 } 396 397 func (s *snapshotter) Walk(context.Context, snapshots.WalkFunc, ...string) error { 398 return errors.Errorf("not-implemented") 399 } 400 401 func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { 402 // not implemented 403 return s.Stat(ctx, info.Name) 404 } 405 406 func (s *snapshotter) Usage(ctx context.Context, key string) (us snapshots.Usage, retErr error) { 407 usage := snapshots.Usage{} 408 if l, err := s.getLayer(key, true); err != nil { 409 return usage, err 410 } else if l != nil { 411 s, err := l.DiffSize() 412 if err != nil { 413 return usage, err 414 } 415 usage.Size = s 416 return usage, nil 417 } 418 419 size := int64(-1) 420 if err := s.db.View(func(tx *bolt.Tx) error { 421 b := tx.Bucket([]byte(key)) 422 if b == nil { 423 return nil 424 } 425 v := b.Get(keySize) 426 if v != nil { 427 s, err := strconv.Atoi(string(v)) 428 if err != nil { 429 return err 430 } 431 size = int64(s) 432 } 433 return nil 434 }); err != nil { 435 return usage, err 436 } 437 438 if size != -1 { 439 usage.Size = size 440 return usage, nil 441 } 442 443 id, _ := s.getGraphDriverID(key) 444 445 info, err := s.Stat(ctx, key) 446 if err != nil { 447 return usage, err 448 } 449 var parent string 450 if info.Parent != "" { 451 if l, err := s.getLayer(info.Parent, false); err != nil { 452 return usage, err 453 } else if l != nil { 454 parent, err = getGraphID(l) 455 if err != nil { 456 return usage, err 457 } 458 } else { 459 parent, _ = s.getGraphDriverID(info.Parent) 460 } 461 } 462 463 diffSize, err := s.opt.GraphDriver.DiffSize(id, parent) 464 if err != nil { 465 return usage, err 466 } 467 468 if err := s.db.Update(func(tx *bolt.Tx) error { 469 b, err := tx.CreateBucketIfNotExists([]byte(key)) 470 if err != nil { 471 return err 472 } 473 return b.Put(keySize, []byte(strconv.Itoa(int(diffSize)))) 474 }); err != nil { 475 return usage, err 476 } 477 usage.Size = diffSize 478 return usage, nil 479 } 480 481 func (s *snapshotter) Close() error { 482 return s.db.Close() 483 } 484 485 type mountable struct { 486 mu sync.Mutex 487 mounts []mount.Mount 488 acquire func() ([]mount.Mount, func() error, error) 489 release func() error 490 refCount int 491 idmap *idtools.IdentityMapping 492 } 493 494 func (m *mountable) Mount() ([]mount.Mount, func() error, error) { 495 m.mu.Lock() 496 defer m.mu.Unlock() 497 498 if m.mounts != nil { 499 m.refCount++ 500 return m.mounts, m.releaseMount, nil 501 } 502 503 mounts, release, err := m.acquire() 504 if err != nil { 505 return nil, nil, err 506 } 507 m.mounts = mounts 508 m.release = release 509 m.refCount = 1 510 511 return m.mounts, m.releaseMount, nil 512 } 513 514 func (m *mountable) releaseMount() error { 515 m.mu.Lock() 516 defer m.mu.Unlock() 517 518 if m.refCount > 1 { 519 m.refCount-- 520 return nil 521 } 522 523 m.refCount = 0 524 if m.release == nil { 525 return nil 526 } 527 528 m.mounts = nil 529 defer func() { 530 m.release = nil 531 }() 532 return m.release() 533 } 534 535 func (m *mountable) IdentityMapping() *idtools.IdentityMapping { 536 return m.idmap 537 }