github.com/moby/docker@v26.1.3+incompatible/builder/builder-next/controller.go (about) 1 package buildkit 2 3 import ( 4 "context" 5 "net/http" 6 "os" 7 "path/filepath" 8 "runtime" 9 "time" 10 11 ctd "github.com/containerd/containerd" 12 "github.com/containerd/containerd/content/local" 13 ctdmetadata "github.com/containerd/containerd/metadata" 14 "github.com/containerd/containerd/platforms" 15 "github.com/containerd/containerd/snapshots" 16 "github.com/containerd/log" 17 "github.com/docker/docker/api/types" 18 "github.com/docker/docker/api/types/filters" 19 "github.com/docker/docker/builder/builder-next/adapters/containerimage" 20 "github.com/docker/docker/builder/builder-next/adapters/localinlinecache" 21 "github.com/docker/docker/builder/builder-next/adapters/snapshot" 22 "github.com/docker/docker/builder/builder-next/exporter/mobyexporter" 23 "github.com/docker/docker/builder/builder-next/imagerefchecker" 24 mobyworker "github.com/docker/docker/builder/builder-next/worker" 25 wlabel "github.com/docker/docker/builder/builder-next/worker/label" 26 "github.com/docker/docker/daemon/config" 27 "github.com/docker/docker/daemon/graphdriver" 28 units "github.com/docker/go-units" 29 "github.com/moby/buildkit/cache" 30 "github.com/moby/buildkit/cache/metadata" 31 "github.com/moby/buildkit/cache/remotecache" 32 "github.com/moby/buildkit/cache/remotecache/gha" 33 inlineremotecache "github.com/moby/buildkit/cache/remotecache/inline" 34 localremotecache "github.com/moby/buildkit/cache/remotecache/local" 35 registryremotecache "github.com/moby/buildkit/cache/remotecache/registry" 36 "github.com/moby/buildkit/client" 37 bkconfig "github.com/moby/buildkit/cmd/buildkitd/config" 38 "github.com/moby/buildkit/control" 39 "github.com/moby/buildkit/frontend" 40 dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder" 41 "github.com/moby/buildkit/frontend/gateway" 42 "github.com/moby/buildkit/frontend/gateway/forwarder" 43 containerdsnapshot "github.com/moby/buildkit/snapshot/containerd" 44 "github.com/moby/buildkit/solver" 45 "github.com/moby/buildkit/solver/bboltcachestorage" 46 "github.com/moby/buildkit/util/archutil" 47 "github.com/moby/buildkit/util/entitlements" 48 "github.com/moby/buildkit/util/network/netproviders" 49 "github.com/moby/buildkit/util/tracing/detect" 50 "github.com/moby/buildkit/worker" 51 "github.com/moby/buildkit/worker/containerd" 52 "github.com/moby/buildkit/worker/label" 53 "github.com/pkg/errors" 54 "go.etcd.io/bbolt" 55 bolt "go.etcd.io/bbolt" 56 "go.opentelemetry.io/otel/sdk/trace" 57 58 "github.com/moby/buildkit/solver/pb" 59 "github.com/moby/buildkit/util/apicaps" 60 ) 61 62 func newController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { 63 if opt.UseSnapshotter { 64 return newSnapshotterController(ctx, rt, opt) 65 } 66 return newGraphDriverController(ctx, rt, opt) 67 } 68 69 func getTraceExporter(ctx context.Context) trace.SpanExporter { 70 span, _, err := detect.Exporter() 71 if err != nil { 72 log.G(ctx).WithError(err).Error("Failed to detect trace exporter for buildkit controller") 73 } 74 return span 75 } 76 77 func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { 78 if err := os.MkdirAll(opt.Root, 0o711); err != nil { 79 return nil, err 80 } 81 82 historyDB, historyConf, err := openHistoryDB(opt.Root, opt.BuilderConfig.History) 83 if err != nil { 84 return nil, err 85 } 86 87 cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(opt.Root, "cache.db")) 88 if err != nil { 89 return nil, err 90 } 91 92 nc := netproviders.Opt{ 93 Mode: "host", 94 } 95 96 // HACK! Windows doesn't have 'host' mode networking. 97 if runtime.GOOS == "windows" { 98 nc = netproviders.Opt{ 99 Mode: "auto", 100 } 101 } 102 103 dns := getDNSConfig(opt.DNSConfig) 104 105 wo, err := containerd.NewWorkerOpt(opt.Root, opt.ContainerdAddress, opt.Snapshotter, opt.ContainerdNamespace, 106 opt.Rootless, map[string]string{ 107 label.Snapshotter: opt.Snapshotter, 108 }, dns, nc, opt.ApparmorProfile, false, nil, "", nil, ctd.WithTimeout(60*time.Second), 109 ) 110 if err != nil { 111 return nil, err 112 } 113 114 policy, err := getGCPolicy(opt.BuilderConfig, opt.Root) 115 if err != nil { 116 return nil, err 117 } 118 119 // make sure platforms are normalized moby/buildkit#4391 120 for i, p := range wo.Platforms { 121 wo.Platforms[i] = platforms.Normalize(p) 122 } 123 124 wo.GCPolicy = policy 125 wo.RegistryHosts = opt.RegistryHosts 126 wo.Labels = getLabels(opt, wo.Labels) 127 128 exec, err := newExecutor(opt.Root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile) 129 if err != nil { 130 return nil, err 131 } 132 wo.Executor = exec 133 134 w, err := mobyworker.NewContainerdWorker(ctx, wo) 135 if err != nil { 136 return nil, err 137 } 138 139 wc := &worker.Controller{} 140 141 err = wc.Add(w) 142 if err != nil { 143 return nil, err 144 } 145 frontends := map[string]frontend.Frontend{ 146 "dockerfile.v0": forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build), 147 "gateway.v0": gateway.NewGatewayFrontend(wc.Infos()), 148 } 149 150 return control.NewController(control.Opt{ 151 SessionManager: opt.SessionManager, 152 WorkerController: wc, 153 Frontends: frontends, 154 CacheManager: solver.NewCacheManager(ctx, "local", cacheStorage, worker.NewCacheResultStorage(wc)), 155 CacheStore: cacheStorage, 156 ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{ 157 "gha": gha.ResolveCacheImporterFunc(), 158 "local": localremotecache.ResolveCacheImporterFunc(opt.SessionManager), 159 "registry": registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, wo.ContentStore, opt.RegistryHosts), 160 }, 161 ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{ 162 "gha": gha.ResolveCacheExporterFunc(), 163 "inline": inlineremotecache.ResolveCacheExporterFunc(), 164 "local": localremotecache.ResolveCacheExporterFunc(opt.SessionManager), 165 "registry": registryremotecache.ResolveCacheExporterFunc(opt.SessionManager, opt.RegistryHosts), 166 }, 167 Entitlements: getEntitlements(opt.BuilderConfig), 168 HistoryDB: historyDB, 169 HistoryConfig: historyConf, 170 LeaseManager: wo.LeaseManager, 171 ContentStore: wo.ContentStore, 172 TraceCollector: getTraceExporter(ctx), 173 }) 174 } 175 176 func openHistoryDB(root string, cfg *config.BuilderHistoryConfig) (*bolt.DB, *bkconfig.HistoryConfig, error) { 177 db, err := bbolt.Open(filepath.Join(root, "history.db"), 0o600, nil) 178 if err != nil { 179 return nil, nil, err 180 } 181 182 var conf *bkconfig.HistoryConfig 183 if cfg != nil { 184 conf = &bkconfig.HistoryConfig{ 185 MaxAge: cfg.MaxAge, 186 MaxEntries: cfg.MaxEntries, 187 } 188 } 189 190 return db, conf, nil 191 } 192 193 func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { 194 if err := os.MkdirAll(opt.Root, 0o711); err != nil { 195 return nil, err 196 } 197 198 dist := opt.Dist 199 root := opt.Root 200 201 pb.Caps.Init(apicaps.Cap{ 202 ID: pb.CapMergeOp, 203 Enabled: false, 204 DisabledReasonMsg: "only enabled with containerd image store backend", 205 }) 206 207 pb.Caps.Init(apicaps.Cap{ 208 ID: pb.CapDiffOp, 209 Enabled: false, 210 DisabledReasonMsg: "only enabled with containerd image store backend", 211 }) 212 213 var driver graphdriver.Driver 214 if ls, ok := dist.LayerStore.(interface { 215 Driver() graphdriver.Driver 216 }); ok { 217 driver = ls.Driver() 218 } else { 219 return nil, errors.Errorf("could not access graphdriver") 220 } 221 222 innerStore, err := local.NewStore(filepath.Join(root, "content")) 223 if err != nil { 224 return nil, err 225 } 226 227 db, err := bolt.Open(filepath.Join(root, "containerdmeta.db"), 0o644, nil) 228 if err != nil { 229 return nil, errors.WithStack(err) 230 } 231 232 mdb := ctdmetadata.NewDB(db, innerStore, map[string]snapshots.Snapshotter{}) 233 234 store := containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit") 235 236 snapshotter, lm, err := snapshot.NewSnapshotter(snapshot.Opt{ 237 GraphDriver: driver, 238 LayerStore: dist.LayerStore, 239 Root: root, 240 IdentityMapping: opt.IdentityMapping, 241 }, ctdmetadata.NewLeaseManager(mdb), "buildkit") 242 if err != nil { 243 return nil, err 244 } 245 246 if err := cache.MigrateV2(context.Background(), filepath.Join(root, "metadata.db"), filepath.Join(root, "metadata_v2.db"), store, snapshotter, lm); err != nil { 247 return nil, err 248 } 249 250 md, err := metadata.NewStore(filepath.Join(root, "metadata_v2.db")) 251 if err != nil { 252 return nil, err 253 } 254 255 layerGetter, ok := snapshotter.(imagerefchecker.LayerGetter) 256 if !ok { 257 return nil, errors.Errorf("snapshotter does not implement layergetter") 258 } 259 260 refChecker := imagerefchecker.New(imagerefchecker.Opt{ 261 ImageStore: dist.ImageStore, 262 LayerGetter: layerGetter, 263 }) 264 265 cm, err := cache.NewManager(cache.ManagerOpt{ 266 Snapshotter: snapshotter, 267 MetadataStore: md, 268 PruneRefChecker: refChecker, 269 LeaseManager: lm, 270 ContentStore: store, 271 GarbageCollect: mdb.GarbageCollect, 272 }) 273 if err != nil { 274 return nil, err 275 } 276 277 src, err := containerimage.NewSource(containerimage.SourceOpt{ 278 CacheAccessor: cm, 279 ContentStore: store, 280 DownloadManager: dist.DownloadManager, 281 MetadataStore: dist.V2MetadataService, 282 ImageStore: dist.ImageStore, 283 ReferenceStore: dist.ReferenceStore, 284 RegistryHosts: opt.RegistryHosts, 285 LayerStore: dist.LayerStore, 286 LeaseManager: lm, 287 GarbageCollect: mdb.GarbageCollect, 288 }) 289 if err != nil { 290 return nil, err 291 } 292 293 dns := getDNSConfig(opt.DNSConfig) 294 295 exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile) 296 if err != nil { 297 return nil, err 298 } 299 300 differ, ok := snapshotter.(mobyexporter.Differ) 301 if !ok { 302 return nil, errors.Errorf("snapshotter doesn't support differ") 303 } 304 305 exp, err := mobyexporter.New(mobyexporter.Opt{ 306 ImageStore: dist.ImageStore, 307 ContentStore: store, 308 Differ: differ, 309 ImageTagger: opt.ImageTagger, 310 LeaseManager: lm, 311 }) 312 if err != nil { 313 return nil, err 314 } 315 316 cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(opt.Root, "cache.db")) 317 if err != nil { 318 return nil, err 319 } 320 321 historyDB, historyConf, err := openHistoryDB(opt.Root, opt.BuilderConfig.History) 322 if err != nil { 323 return nil, err 324 } 325 326 gcPolicy, err := getGCPolicy(opt.BuilderConfig, root) 327 if err != nil { 328 return nil, errors.Wrap(err, "could not get builder GC policy") 329 } 330 331 layers, ok := snapshotter.(mobyworker.LayerAccess) 332 if !ok { 333 return nil, errors.Errorf("snapshotter doesn't support differ") 334 } 335 336 leases, err := lm.List(ctx, `labels."buildkit/lease.temporary"`) 337 if err != nil { 338 return nil, err 339 } 340 for _, l := range leases { 341 lm.Delete(ctx, l) 342 } 343 344 wopt := mobyworker.Opt{ 345 ID: opt.EngineID, 346 ContentStore: store, 347 CacheManager: cm, 348 GCPolicy: gcPolicy, 349 Snapshotter: snapshotter, 350 Executor: exec, 351 ImageSource: src, 352 DownloadManager: dist.DownloadManager, 353 V2MetadataService: dist.V2MetadataService, 354 Exporter: exp, 355 Transport: rt, 356 Layers: layers, 357 Platforms: archutil.SupportedPlatforms(true), 358 LeaseManager: lm, 359 Labels: getLabels(opt, nil), 360 } 361 362 wc := &worker.Controller{} 363 w, err := mobyworker.NewWorker(wopt) 364 if err != nil { 365 return nil, err 366 } 367 wc.Add(w) 368 369 frontends := map[string]frontend.Frontend{ 370 "dockerfile.v0": forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build), 371 "gateway.v0": gateway.NewGatewayFrontend(wc.Infos()), 372 } 373 374 return control.NewController(control.Opt{ 375 SessionManager: opt.SessionManager, 376 WorkerController: wc, 377 Frontends: frontends, 378 CacheManager: solver.NewCacheManager(ctx, "local", cacheStorage, worker.NewCacheResultStorage(wc)), 379 CacheStore: cacheStorage, 380 ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{ 381 "registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.RegistryHosts, store, dist.ReferenceStore, dist.ImageStore), 382 "local": localremotecache.ResolveCacheImporterFunc(opt.SessionManager), 383 }, 384 ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{ 385 "inline": inlineremotecache.ResolveCacheExporterFunc(), 386 }, 387 Entitlements: getEntitlements(opt.BuilderConfig), 388 LeaseManager: lm, 389 ContentStore: store, 390 HistoryDB: historyDB, 391 HistoryConfig: historyConf, 392 TraceCollector: getTraceExporter(ctx), 393 }) 394 } 395 396 func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) { 397 var gcPolicy []client.PruneInfo 398 if conf.GC.Enabled { 399 var ( 400 defaultKeepStorage int64 401 err error 402 ) 403 404 if conf.GC.DefaultKeepStorage != "" { 405 defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage) 406 if err != nil { 407 return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage) 408 } 409 } 410 411 if conf.GC.Policy == nil { 412 gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage) 413 } else { 414 gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy)) 415 for i, p := range conf.GC.Policy { 416 b, err := units.RAMInBytes(p.KeepStorage) 417 if err != nil { 418 return nil, err 419 } 420 if b == 0 { 421 b = defaultKeepStorage 422 } 423 gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{ 424 All: p.All, 425 KeepStorage: b, 426 Filters: filters.Args(p.Filter), 427 }) 428 if err != nil { 429 return nil, err 430 } 431 } 432 } 433 } 434 return gcPolicy, nil 435 } 436 437 func getEntitlements(conf config.BuilderConfig) []string { 438 var ents []string 439 // Incase of no config settings, NetworkHost should be enabled & SecurityInsecure must be disabled. 440 if conf.Entitlements.NetworkHost == nil || *conf.Entitlements.NetworkHost { 441 ents = append(ents, string(entitlements.EntitlementNetworkHost)) 442 } 443 if conf.Entitlements.SecurityInsecure != nil && *conf.Entitlements.SecurityInsecure { 444 ents = append(ents, string(entitlements.EntitlementSecurityInsecure)) 445 } 446 return ents 447 } 448 449 func getLabels(opt Opt, labels map[string]string) map[string]string { 450 if labels == nil { 451 labels = make(map[string]string) 452 } 453 labels[wlabel.HostGatewayIP] = opt.DNSConfig.HostGatewayIP.String() 454 return labels 455 }