github.com/rawahars/moby@v24.0.4+incompatible/builder/builder-next/controller.go (about) 1 package buildkit 2 3 import ( 4 "context" 5 "net/http" 6 "os" 7 "path/filepath" 8 "time" 9 10 ctd "github.com/containerd/containerd" 11 "github.com/containerd/containerd/content/local" 12 ctdmetadata "github.com/containerd/containerd/metadata" 13 "github.com/containerd/containerd/snapshots" 14 "github.com/docker/docker/api/types" 15 "github.com/docker/docker/api/types/filters" 16 "github.com/docker/docker/builder/builder-next/adapters/containerimage" 17 "github.com/docker/docker/builder/builder-next/adapters/localinlinecache" 18 "github.com/docker/docker/builder/builder-next/adapters/snapshot" 19 "github.com/docker/docker/builder/builder-next/exporter/mobyexporter" 20 "github.com/docker/docker/builder/builder-next/imagerefchecker" 21 mobyworker "github.com/docker/docker/builder/builder-next/worker" 22 wlabel "github.com/docker/docker/builder/builder-next/worker/label" 23 "github.com/docker/docker/daemon/config" 24 "github.com/docker/docker/daemon/graphdriver" 25 units "github.com/docker/go-units" 26 "github.com/moby/buildkit/cache" 27 "github.com/moby/buildkit/cache/metadata" 28 "github.com/moby/buildkit/cache/remotecache" 29 "github.com/moby/buildkit/cache/remotecache/gha" 30 inlineremotecache "github.com/moby/buildkit/cache/remotecache/inline" 31 localremotecache "github.com/moby/buildkit/cache/remotecache/local" 32 registryremotecache "github.com/moby/buildkit/cache/remotecache/registry" 33 "github.com/moby/buildkit/client" 34 bkconfig "github.com/moby/buildkit/cmd/buildkitd/config" 35 "github.com/moby/buildkit/control" 36 "github.com/moby/buildkit/frontend" 37 dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder" 38 "github.com/moby/buildkit/frontend/gateway" 39 "github.com/moby/buildkit/frontend/gateway/forwarder" 40 containerdsnapshot "github.com/moby/buildkit/snapshot/containerd" 41 "github.com/moby/buildkit/solver/bboltcachestorage" 42 "github.com/moby/buildkit/util/archutil" 43 "github.com/moby/buildkit/util/entitlements" 44 "github.com/moby/buildkit/util/leaseutil" 45 "github.com/moby/buildkit/util/network/netproviders" 46 "github.com/moby/buildkit/worker" 47 "github.com/moby/buildkit/worker/containerd" 48 "github.com/moby/buildkit/worker/label" 49 "github.com/pkg/errors" 50 "go.etcd.io/bbolt" 51 bolt "go.etcd.io/bbolt" 52 53 "github.com/moby/buildkit/solver/pb" 54 "github.com/moby/buildkit/util/apicaps" 55 ) 56 57 func newController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { 58 if opt.UseSnapshotter { 59 return newSnapshotterController(ctx, rt, opt) 60 } 61 return newGraphDriverController(ctx, rt, opt) 62 } 63 64 func newSnapshotterController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { 65 if err := os.MkdirAll(opt.Root, 0o711); err != nil { 66 return nil, err 67 } 68 69 historyDB, historyConf, err := openHistoryDB(opt.Root, opt.BuilderConfig.History) 70 if err != nil { 71 return nil, err 72 } 73 74 cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(opt.Root, "cache.db")) 75 if err != nil { 76 return nil, err 77 } 78 79 nc := netproviders.Opt{ 80 Mode: "host", 81 } 82 dns := getDNSConfig(opt.DNSConfig) 83 84 wo, err := containerd.NewWorkerOpt(opt.Root, opt.ContainerdAddress, opt.Snapshotter, opt.ContainerdNamespace, 85 opt.Rootless, map[string]string{ 86 label.Snapshotter: opt.Snapshotter, 87 }, dns, nc, opt.ApparmorProfile, false, nil, "", ctd.WithTimeout(60*time.Second)) 88 if err != nil { 89 return nil, err 90 } 91 92 policy, err := getGCPolicy(opt.BuilderConfig, opt.Root) 93 if err != nil { 94 return nil, err 95 } 96 97 wo.GCPolicy = policy 98 wo.RegistryHosts = opt.RegistryHosts 99 wo.Labels = getLabels(opt, wo.Labels) 100 101 exec, err := newExecutor(opt.Root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile) 102 if err != nil { 103 return nil, err 104 } 105 wo.Executor = exec 106 107 w, err := mobyworker.NewContainerdWorker(ctx, wo) 108 if err != nil { 109 return nil, err 110 } 111 112 wc := &worker.Controller{} 113 114 err = wc.Add(w) 115 if err != nil { 116 return nil, err 117 } 118 frontends := map[string]frontend.Frontend{ 119 "dockerfile.v0": forwarder.NewGatewayForwarder(wc, dockerfile.Build), 120 "gateway.v0": gateway.NewGatewayFrontend(wc), 121 } 122 123 return control.NewController(control.Opt{ 124 SessionManager: opt.SessionManager, 125 WorkerController: wc, 126 Frontends: frontends, 127 CacheKeyStorage: cacheStorage, 128 ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{ 129 "gha": gha.ResolveCacheImporterFunc(), 130 "local": localremotecache.ResolveCacheImporterFunc(opt.SessionManager), 131 "registry": registryremotecache.ResolveCacheImporterFunc(opt.SessionManager, wo.ContentStore, opt.RegistryHosts), 132 }, 133 ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{ 134 "gha": gha.ResolveCacheExporterFunc(), 135 "inline": inlineremotecache.ResolveCacheExporterFunc(), 136 "local": localremotecache.ResolveCacheExporterFunc(opt.SessionManager), 137 "registry": registryremotecache.ResolveCacheExporterFunc(opt.SessionManager, opt.RegistryHosts), 138 }, 139 Entitlements: getEntitlements(opt.BuilderConfig), 140 HistoryDB: historyDB, 141 HistoryConfig: historyConf, 142 LeaseManager: wo.LeaseManager, 143 ContentStore: wo.ContentStore, 144 }) 145 } 146 147 func openHistoryDB(root string, cfg *config.BuilderHistoryConfig) (*bolt.DB, *bkconfig.HistoryConfig, error) { 148 db, err := bbolt.Open(filepath.Join(root, "history.db"), 0o600, nil) 149 if err != nil { 150 return nil, nil, err 151 } 152 153 var conf *bkconfig.HistoryConfig 154 if cfg != nil { 155 conf = &bkconfig.HistoryConfig{ 156 MaxAge: cfg.MaxAge, 157 MaxEntries: cfg.MaxEntries, 158 } 159 } 160 161 return db, conf, nil 162 } 163 164 func newGraphDriverController(ctx context.Context, rt http.RoundTripper, opt Opt) (*control.Controller, error) { 165 if err := os.MkdirAll(opt.Root, 0711); err != nil { 166 return nil, err 167 } 168 169 dist := opt.Dist 170 root := opt.Root 171 172 pb.Caps.Init(apicaps.Cap{ 173 ID: pb.CapMergeOp, 174 Enabled: false, 175 DisabledReasonMsg: "only enabled with containerd image store backend", 176 }) 177 178 pb.Caps.Init(apicaps.Cap{ 179 ID: pb.CapDiffOp, 180 Enabled: false, 181 DisabledReasonMsg: "only enabled with containerd image store backend", 182 }) 183 184 var driver graphdriver.Driver 185 if ls, ok := dist.LayerStore.(interface { 186 Driver() graphdriver.Driver 187 }); ok { 188 driver = ls.Driver() 189 } else { 190 return nil, errors.Errorf("could not access graphdriver") 191 } 192 193 store, err := local.NewStore(filepath.Join(root, "content")) 194 if err != nil { 195 return nil, err 196 } 197 198 db, err := bolt.Open(filepath.Join(root, "containerdmeta.db"), 0644, nil) 199 if err != nil { 200 return nil, errors.WithStack(err) 201 } 202 203 mdb := ctdmetadata.NewDB(db, store, map[string]snapshots.Snapshotter{}) 204 205 store = containerdsnapshot.NewContentStore(mdb.ContentStore(), "buildkit") 206 207 lm := leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit") 208 209 snapshotter, lm, err := snapshot.NewSnapshotter(snapshot.Opt{ 210 GraphDriver: driver, 211 LayerStore: dist.LayerStore, 212 Root: root, 213 IdentityMapping: opt.IdentityMapping, 214 }, lm) 215 if err != nil { 216 return nil, err 217 } 218 219 if err := cache.MigrateV2(context.Background(), filepath.Join(root, "metadata.db"), filepath.Join(root, "metadata_v2.db"), store, snapshotter, lm); err != nil { 220 return nil, err 221 } 222 223 md, err := metadata.NewStore(filepath.Join(root, "metadata_v2.db")) 224 if err != nil { 225 return nil, err 226 } 227 228 layerGetter, ok := snapshotter.(imagerefchecker.LayerGetter) 229 if !ok { 230 return nil, errors.Errorf("snapshotter does not implement layergetter") 231 } 232 233 refChecker := imagerefchecker.New(imagerefchecker.Opt{ 234 ImageStore: dist.ImageStore, 235 LayerGetter: layerGetter, 236 }) 237 238 cm, err := cache.NewManager(cache.ManagerOpt{ 239 Snapshotter: snapshotter, 240 MetadataStore: md, 241 PruneRefChecker: refChecker, 242 LeaseManager: lm, 243 ContentStore: store, 244 GarbageCollect: mdb.GarbageCollect, 245 }) 246 if err != nil { 247 return nil, err 248 } 249 250 src, err := containerimage.NewSource(containerimage.SourceOpt{ 251 CacheAccessor: cm, 252 ContentStore: store, 253 DownloadManager: dist.DownloadManager, 254 MetadataStore: dist.V2MetadataService, 255 ImageStore: dist.ImageStore, 256 ReferenceStore: dist.ReferenceStore, 257 RegistryHosts: opt.RegistryHosts, 258 LayerStore: dist.LayerStore, 259 LeaseManager: lm, 260 GarbageCollect: mdb.GarbageCollect, 261 }) 262 if err != nil { 263 return nil, err 264 } 265 266 dns := getDNSConfig(opt.DNSConfig) 267 268 exec, err := newExecutor(root, opt.DefaultCgroupParent, opt.NetworkController, dns, opt.Rootless, opt.IdentityMapping, opt.ApparmorProfile) 269 if err != nil { 270 return nil, err 271 } 272 273 differ, ok := snapshotter.(mobyexporter.Differ) 274 if !ok { 275 return nil, errors.Errorf("snapshotter doesn't support differ") 276 } 277 278 exp, err := mobyexporter.New(mobyexporter.Opt{ 279 ImageStore: dist.ImageStore, 280 Differ: differ, 281 ImageTagger: opt.ImageTagger, 282 }) 283 if err != nil { 284 return nil, err 285 } 286 287 cacheStorage, err := bboltcachestorage.NewStore(filepath.Join(opt.Root, "cache.db")) 288 if err != nil { 289 return nil, err 290 } 291 292 historyDB, historyConf, err := openHistoryDB(opt.Root, opt.BuilderConfig.History) 293 if err != nil { 294 return nil, err 295 } 296 297 gcPolicy, err := getGCPolicy(opt.BuilderConfig, root) 298 if err != nil { 299 return nil, errors.Wrap(err, "could not get builder GC policy") 300 } 301 302 layers, ok := snapshotter.(mobyworker.LayerAccess) 303 if !ok { 304 return nil, errors.Errorf("snapshotter doesn't support differ") 305 } 306 307 leases, err := lm.List(ctx, "labels.\"buildkit/lease.temporary\"") 308 if err != nil { 309 return nil, err 310 } 311 for _, l := range leases { 312 lm.Delete(ctx, l) 313 } 314 315 wopt := mobyworker.Opt{ 316 ID: opt.EngineID, 317 ContentStore: store, 318 CacheManager: cm, 319 GCPolicy: gcPolicy, 320 Snapshotter: snapshotter, 321 Executor: exec, 322 ImageSource: src, 323 DownloadManager: dist.DownloadManager, 324 V2MetadataService: dist.V2MetadataService, 325 Exporter: exp, 326 Transport: rt, 327 Layers: layers, 328 Platforms: archutil.SupportedPlatforms(true), 329 LeaseManager: lm, 330 Labels: getLabels(opt, nil), 331 } 332 333 wc := &worker.Controller{} 334 w, err := mobyworker.NewWorker(wopt) 335 if err != nil { 336 return nil, err 337 } 338 wc.Add(w) 339 340 frontends := map[string]frontend.Frontend{ 341 "dockerfile.v0": forwarder.NewGatewayForwarder(wc, dockerfile.Build), 342 "gateway.v0": gateway.NewGatewayFrontend(wc), 343 } 344 345 return control.NewController(control.Opt{ 346 SessionManager: opt.SessionManager, 347 WorkerController: wc, 348 Frontends: frontends, 349 CacheKeyStorage: cacheStorage, 350 ResolveCacheImporterFuncs: map[string]remotecache.ResolveCacheImporterFunc{ 351 "registry": localinlinecache.ResolveCacheImporterFunc(opt.SessionManager, opt.RegistryHosts, store, dist.ReferenceStore, dist.ImageStore), 352 "local": localremotecache.ResolveCacheImporterFunc(opt.SessionManager), 353 }, 354 ResolveCacheExporterFuncs: map[string]remotecache.ResolveCacheExporterFunc{ 355 "inline": inlineremotecache.ResolveCacheExporterFunc(), 356 }, 357 Entitlements: getEntitlements(opt.BuilderConfig), 358 LeaseManager: lm, 359 ContentStore: store, 360 HistoryDB: historyDB, 361 HistoryConfig: historyConf, 362 }) 363 } 364 365 func getGCPolicy(conf config.BuilderConfig, root string) ([]client.PruneInfo, error) { 366 var gcPolicy []client.PruneInfo 367 if conf.GC.Enabled { 368 var ( 369 defaultKeepStorage int64 370 err error 371 ) 372 373 if conf.GC.DefaultKeepStorage != "" { 374 defaultKeepStorage, err = units.RAMInBytes(conf.GC.DefaultKeepStorage) 375 if err != nil { 376 return nil, errors.Wrapf(err, "could not parse '%s' as Builder.GC.DefaultKeepStorage config", conf.GC.DefaultKeepStorage) 377 } 378 } 379 380 if conf.GC.Policy == nil { 381 gcPolicy = mobyworker.DefaultGCPolicy(root, defaultKeepStorage) 382 } else { 383 gcPolicy = make([]client.PruneInfo, len(conf.GC.Policy)) 384 for i, p := range conf.GC.Policy { 385 b, err := units.RAMInBytes(p.KeepStorage) 386 if err != nil { 387 return nil, err 388 } 389 if b == 0 { 390 b = defaultKeepStorage 391 } 392 gcPolicy[i], err = toBuildkitPruneInfo(types.BuildCachePruneOptions{ 393 All: p.All, 394 KeepStorage: b, 395 Filters: filters.Args(p.Filter), 396 }) 397 if err != nil { 398 return nil, err 399 } 400 } 401 } 402 } 403 return gcPolicy, nil 404 } 405 406 func getEntitlements(conf config.BuilderConfig) []string { 407 var ents []string 408 // Incase of no config settings, NetworkHost should be enabled & SecurityInsecure must be disabled. 409 if conf.Entitlements.NetworkHost == nil || *conf.Entitlements.NetworkHost { 410 ents = append(ents, string(entitlements.EntitlementNetworkHost)) 411 } 412 if conf.Entitlements.SecurityInsecure != nil && *conf.Entitlements.SecurityInsecure { 413 ents = append(ents, string(entitlements.EntitlementSecurityInsecure)) 414 } 415 return ents 416 } 417 418 func getLabels(opt Opt, labels map[string]string) map[string]string { 419 if labels == nil { 420 labels = make(map[string]string) 421 } 422 labels[wlabel.HostGatewayIP] = opt.DNSConfig.HostGatewayIP.String() 423 return labels 424 }