gitlab.com/jfprevost/gitlab-runner-notlscheck@v11.11.4+incompatible/executors/docker/executor_docker.go (about) 1 package docker 2 3 import ( 4 "bytes" 5 "context" 6 "errors" 7 "fmt" 8 "io" 9 "os" 10 "path" 11 "path/filepath" 12 "regexp" 13 "strconv" 14 "strings" 15 "sync" 16 "time" 17 18 "github.com/docker/distribution/reference" 19 "github.com/docker/docker/api/types" 20 "github.com/docker/docker/api/types/container" 21 "github.com/docker/docker/pkg/stdcopy" 22 "github.com/kardianos/osext" 23 "github.com/mattn/go-zglob" 24 "github.com/sirupsen/logrus" 25 26 "gitlab.com/gitlab-org/gitlab-runner/common" 27 "gitlab.com/gitlab-org/gitlab-runner/executors" 28 "gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes" 29 "gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser" 30 "gitlab.com/gitlab-org/gitlab-runner/helpers" 31 docker_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/docker" 32 "gitlab.com/gitlab-org/gitlab-runner/helpers/docker/helperimage" 33 "gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags" 34 ) 35 36 const ( 37 DockerExecutorStagePrepare common.ExecutorStage = "docker_prepare" 38 DockerExecutorStageRun common.ExecutorStage = "docker_run" 39 DockerExecutorStageCleanup common.ExecutorStage = "docker_cleanup" 40 41 DockerExecutorStageCreatingBuildVolumes common.ExecutorStage = "docker_creating_build_volumes" 42 DockerExecutorStageCreatingServices common.ExecutorStage = "docker_creating_services" 43 DockerExecutorStageCreatingUserVolumes common.ExecutorStage = "docker_creating_user_volumes" 44 DockerExecutorStagePullingImage common.ExecutorStage = "docker_pulling_image" 45 ) 46 47 var DockerPrebuiltImagesPaths []string 48 49 var neverRestartPolicy = container.RestartPolicy{Name: "no"} 50 51 var errVolumesManagerUndefined = errors.New("volumesManager is undefined") 52 53 type executor struct { 54 executors.AbstractExecutor 55 client docker_helpers.Client 56 volumeParser parser.Parser 57 info types.Info 58 59 temporary []string // IDs of containers that should be removed 60 61 builds []string // IDs of successfully created build containers 62 services []*types.Container 63 64 links []string 65 66 devices []container.DeviceMapping 67 68 helperImageInfo helperimage.Info 69 70 usedImages map[string]string 71 usedImagesLock sync.RWMutex 72 73 volumesManager volumes.Manager 74 } 75 76 func init() { 77 runnerFolder, err := osext.ExecutableFolder() 78 if err != nil { 79 logrus.Errorln("Docker executor: unable to detect gitlab-runner folder, prebuilt image helpers will be loaded from DockerHub.", err) 80 } 81 82 DockerPrebuiltImagesPaths = []string{ 83 filepath.Join(runnerFolder, "helper-images"), 84 filepath.Join(runnerFolder, "out/helper-images"), 85 } 86 } 87 88 func (e *executor) getServiceVariables() []string { 89 return e.Build.GetAllVariables().PublicOrInternal().StringList() 90 } 91 92 func (e *executor) getUserAuthConfiguration(indexName string) *types.AuthConfig { 93 if e.Build == nil { 94 return nil 95 } 96 97 buf := bytes.NewBufferString(e.Build.GetDockerAuthConfig()) 98 authConfigs, _ := docker_helpers.ReadAuthConfigsFromReader(buf) 99 if authConfigs != nil { 100 return docker_helpers.ResolveDockerAuthConfig(indexName, authConfigs) 101 } 102 return nil 103 } 104 105 func (e *executor) getBuildAuthConfiguration(indexName string) *types.AuthConfig { 106 if e.Build == nil { 107 return nil 108 } 109 110 authConfigs := make(map[string]types.AuthConfig) 111 112 for _, credentials := range e.Build.Credentials { 113 if credentials.Type != "registry" { 114 continue 115 } 116 117 authConfigs[credentials.URL] = types.AuthConfig{ 118 Username: credentials.Username, 119 Password: credentials.Password, 120 ServerAddress: credentials.URL, 121 } 122 } 123 124 if authConfigs != nil { 125 return docker_helpers.ResolveDockerAuthConfig(indexName, authConfigs) 126 } 127 return nil 128 } 129 130 func (e *executor) getHomeDirAuthConfiguration(indexName string) *types.AuthConfig { 131 authConfigs, _ := docker_helpers.ReadDockerAuthConfigsFromHomeDir(e.Shell().User) 132 if authConfigs != nil { 133 return docker_helpers.ResolveDockerAuthConfig(indexName, authConfigs) 134 } 135 return nil 136 } 137 138 func (e *executor) getAuthConfig(imageName string) *types.AuthConfig { 139 indexName, _ := docker_helpers.SplitDockerImageName(imageName) 140 141 authConfig := e.getUserAuthConfiguration(indexName) 142 if authConfig == nil { 143 authConfig = e.getHomeDirAuthConfiguration(indexName) 144 } 145 if authConfig == nil { 146 authConfig = e.getBuildAuthConfiguration(indexName) 147 } 148 149 if authConfig != nil { 150 e.Debugln("Using", authConfig.Username, "to connect to", authConfig.ServerAddress, 151 "in order to resolve", imageName, "...") 152 return authConfig 153 } 154 155 e.Debugln(fmt.Sprintf("No credentials found for %v", indexName)) 156 return nil 157 } 158 159 func (e *executor) pullDockerImage(imageName string, ac *types.AuthConfig) (*types.ImageInspect, error) { 160 e.SetCurrentStage(DockerExecutorStagePullingImage) 161 e.Println("Pulling docker image", imageName, "...") 162 163 ref := imageName 164 // Add :latest to limit the download results 165 if !strings.ContainsAny(ref, ":@") { 166 ref += ":latest" 167 } 168 169 options := types.ImagePullOptions{} 170 if ac != nil { 171 options.RegistryAuth, _ = docker_helpers.EncodeAuthConfig(ac) 172 } 173 174 errorRegexp := regexp.MustCompile("(repository does not exist|not found)") 175 if err := e.client.ImagePullBlocking(e.Context, ref, options); err != nil { 176 if errorRegexp.MatchString(err.Error()) { 177 return nil, &common.BuildError{Inner: err} 178 } 179 return nil, err 180 } 181 182 image, _, err := e.client.ImageInspectWithRaw(e.Context, imageName) 183 return &image, err 184 } 185 186 func (e *executor) getDockerImage(imageName string) (image *types.ImageInspect, err error) { 187 pullPolicy, err := e.Config.Docker.PullPolicy.Get() 188 if err != nil { 189 return nil, err 190 } 191 192 authConfig := e.getAuthConfig(imageName) 193 194 e.Debugln("Looking for image", imageName, "...") 195 existingImage, _, err := e.client.ImageInspectWithRaw(e.Context, imageName) 196 197 // Return early if we already used that image 198 if err == nil && e.wasImageUsed(imageName, existingImage.ID) { 199 return &existingImage, nil 200 } 201 202 defer func() { 203 if err == nil { 204 e.markImageAsUsed(imageName, image.ID) 205 } 206 }() 207 208 // If never is specified then we return what inspect did return 209 if pullPolicy == common.PullPolicyNever { 210 return &existingImage, err 211 } 212 213 if err == nil { 214 // Don't pull image that is passed by ID 215 if existingImage.ID == imageName { 216 return &existingImage, nil 217 } 218 219 // If not-present is specified 220 if pullPolicy == common.PullPolicyIfNotPresent { 221 e.Println("Using locally found image version due to if-not-present pull policy") 222 return &existingImage, err 223 } 224 } 225 226 return e.pullDockerImage(imageName, authConfig) 227 } 228 229 func (e *executor) expandAndGetDockerImage(imageName string, allowedImages []string) (*types.ImageInspect, error) { 230 imageName, err := e.expandImageName(imageName, allowedImages) 231 if err != nil { 232 return nil, err 233 } 234 235 image, err := e.getDockerImage(imageName) 236 if err != nil { 237 return nil, err 238 } 239 240 return image, nil 241 } 242 243 func (e *executor) loadPrebuiltImage(path, ref, tag string) (*types.ImageInspect, error) { 244 file, err := os.OpenFile(path, os.O_RDONLY, 0600) 245 if err != nil { 246 if os.IsNotExist(err) { 247 return nil, err 248 } 249 250 return nil, fmt.Errorf("Cannot load prebuilt image: %s: %q", path, err.Error()) 251 } 252 defer file.Close() 253 254 e.Debugln("Loading prebuilt image...") 255 256 source := types.ImageImportSource{ 257 Source: file, 258 SourceName: "-", 259 } 260 options := types.ImageImportOptions{Tag: tag} 261 262 if err := e.client.ImageImportBlocking(e.Context, source, ref, options); err != nil { 263 return nil, fmt.Errorf("Failed to import image: %s", err) 264 } 265 266 image, _, err := e.client.ImageInspectWithRaw(e.Context, ref+":"+tag) 267 if err != nil { 268 e.Debugln("Inspecting imported image", ref, "failed:", err) 269 return nil, err 270 } 271 272 return &image, err 273 } 274 275 func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) { 276 if imageNameFromConfig := e.Config.Docker.HelperImage; imageNameFromConfig != "" { 277 imageNameFromConfig = common.AppVersion.Variables().ExpandValue(imageNameFromConfig) 278 279 e.Debugln("Pull configured helper_image for predefined container instead of import bundled image", imageNameFromConfig, "...") 280 if !e.Build.IsFeatureFlagOn(featureflags.DockerHelperImageV2) { 281 e.Warningln("DEPRECATION: With gitlab-runner 12.0 we will change some tools inside the helper image, please make sure your image is compliant with the new API. https://gitlab.com/gitlab-org/gitlab-runner/issues/4013") 282 } 283 284 return e.getDockerImage(imageNameFromConfig) 285 } 286 287 e.Debugln(fmt.Sprintf("Looking for prebuilt image %s...", e.helperImageInfo)) 288 image, _, err := e.client.ImageInspectWithRaw(e.Context, e.helperImageInfo.String()) 289 if err == nil { 290 return &image, nil 291 } 292 293 // Try to load prebuilt image from local filesystem 294 loadedImage := e.getLocalHelperImage() 295 if loadedImage != nil { 296 return loadedImage, nil 297 } 298 299 // Fallback to getting image from DockerHub 300 e.Debugln(fmt.Sprintf("Loading image form registry: %s", e.helperImageInfo)) 301 return e.getDockerImage(e.helperImageInfo.String()) 302 } 303 304 func (e *executor) getLocalHelperImage() *types.ImageInspect { 305 if !e.helperImageInfo.IsSupportingLocalImport { 306 return nil 307 } 308 309 architecture := e.helperImageInfo.Architecture 310 for _, dockerPrebuiltImagesPath := range DockerPrebuiltImagesPaths { 311 dockerPrebuiltImageFilePath := filepath.Join(dockerPrebuiltImagesPath, "prebuilt-"+architecture+prebuiltImageExtension) 312 image, err := e.loadPrebuiltImage(dockerPrebuiltImageFilePath, prebuiltImageName, e.helperImageInfo.Tag) 313 if err != nil { 314 e.Debugln("Failed to load prebuilt image from:", dockerPrebuiltImageFilePath, "error:", err) 315 continue 316 } 317 318 return image 319 } 320 321 return nil 322 } 323 324 func (e *executor) getBuildImage() (*types.ImageInspect, error) { 325 imageName, err := e.expandImageName(e.Build.Image.Name, []string{}) 326 if err != nil { 327 return nil, err 328 } 329 330 // Fetch image 331 image, err := e.getDockerImage(imageName) 332 if err != nil { 333 return nil, err 334 } 335 336 return image, nil 337 } 338 339 func (e *executor) getLabels(containerType string, otherLabels ...string) map[string]string { 340 labels := make(map[string]string) 341 labels[dockerLabelPrefix+".job.id"] = strconv.Itoa(e.Build.ID) 342 labels[dockerLabelPrefix+".job.sha"] = e.Build.GitInfo.Sha 343 labels[dockerLabelPrefix+".job.before_sha"] = e.Build.GitInfo.BeforeSha 344 labels[dockerLabelPrefix+".job.ref"] = e.Build.GitInfo.Ref 345 labels[dockerLabelPrefix+".project.id"] = strconv.Itoa(e.Build.JobInfo.ProjectID) 346 labels[dockerLabelPrefix+".runner.id"] = e.Build.Runner.ShortDescription() 347 labels[dockerLabelPrefix+".runner.local_id"] = strconv.Itoa(e.Build.RunnerID) 348 labels[dockerLabelPrefix+".type"] = containerType 349 for _, label := range otherLabels { 350 keyValue := strings.SplitN(label, "=", 2) 351 if len(keyValue) == 2 { 352 labels[dockerLabelPrefix+"."+keyValue[0]] = keyValue[1] 353 } 354 } 355 return labels 356 } 357 358 func fakeContainer(id string, names ...string) *types.Container { 359 return &types.Container{ID: id, Names: names} 360 } 361 362 func (e *executor) parseDeviceString(deviceString string) (device container.DeviceMapping, err error) { 363 // Split the device string PathOnHost[:PathInContainer[:CgroupPermissions]] 364 parts := strings.Split(deviceString, ":") 365 366 if len(parts) > 3 { 367 err = fmt.Errorf("Too many colons") 368 return 369 } 370 371 device.PathOnHost = parts[0] 372 373 // Optional container path 374 if len(parts) >= 2 { 375 device.PathInContainer = parts[1] 376 } else { 377 // default: device at same path in container 378 device.PathInContainer = device.PathOnHost 379 } 380 381 // Optional permissions 382 if len(parts) >= 3 { 383 device.CgroupPermissions = parts[2] 384 } else { 385 // default: rwm, just like 'docker run' 386 device.CgroupPermissions = "rwm" 387 } 388 389 return 390 } 391 392 func (e *executor) bindDevices() (err error) { 393 for _, deviceString := range e.Config.Docker.Devices { 394 device, err := e.parseDeviceString(deviceString) 395 if err != nil { 396 err = fmt.Errorf("Failed to parse device string %q: %s", deviceString, err) 397 return err 398 } 399 400 e.devices = append(e.devices, device) 401 } 402 return nil 403 } 404 405 func (e *executor) wasImageUsed(imageName, imageID string) bool { 406 e.usedImagesLock.RLock() 407 defer e.usedImagesLock.RUnlock() 408 409 if e.usedImages[imageName] == imageID { 410 return true 411 } 412 return false 413 } 414 415 func (e *executor) markImageAsUsed(imageName, imageID string) { 416 e.usedImagesLock.Lock() 417 defer e.usedImagesLock.Unlock() 418 419 if e.usedImages == nil { 420 e.usedImages = make(map[string]string) 421 } 422 e.usedImages[imageName] = imageID 423 424 if imageName != imageID { 425 e.Println("Using docker image", imageID, "for", imageName, "...") 426 } 427 } 428 429 func (e *executor) splitServiceAndVersion(serviceDescription string) (service, version, imageName string, linkNames []string) { 430 ReferenceRegexpNoPort := regexp.MustCompile(`^(.*?)(|:[0-9]+)(|/.*)$`) 431 imageName = serviceDescription 432 version = "latest" 433 434 if match := reference.ReferenceRegexp.FindStringSubmatch(serviceDescription); match != nil { 435 matchService := ReferenceRegexpNoPort.FindStringSubmatch(match[1]) 436 service = matchService[1] + matchService[3] 437 438 if len(match[2]) > 0 { 439 version = match[2] 440 } else { 441 imageName = match[1] + ":" + version 442 } 443 } else { 444 return 445 } 446 447 linkName := strings.Replace(service, "/", "__", -1) 448 linkNames = append(linkNames, linkName) 449 450 // Create alternative link name according to RFC 1123 451 // Where you can use only `a-zA-Z0-9-` 452 if alternativeName := strings.Replace(service, "/", "-", -1); linkName != alternativeName { 453 linkNames = append(linkNames, alternativeName) 454 } 455 return 456 } 457 458 func (e *executor) createService(serviceIndex int, service, version, image string, serviceDefinition common.Image) (*types.Container, error) { 459 if len(service) == 0 { 460 return nil, errors.New("invalid service name") 461 } 462 463 if e.volumesManager == nil { 464 return nil, errVolumesManagerUndefined 465 } 466 467 e.Println("Starting service", service+":"+version, "...") 468 serviceImage, err := e.getDockerImage(image) 469 if err != nil { 470 return nil, err 471 } 472 473 serviceSlug := strings.Replace(service, "/", "__", -1) 474 containerName := fmt.Sprintf("%s-%s-%d", e.Build.ProjectUniqueName(), serviceSlug, serviceIndex) 475 476 // this will fail potentially some builds if there's name collision 477 e.removeContainer(e.Context, containerName) 478 479 config := &container.Config{ 480 Image: serviceImage.ID, 481 Labels: e.getLabels("service", "service="+service, "service.version="+version), 482 Env: e.getServiceVariables(), 483 } 484 485 if len(serviceDefinition.Command) > 0 { 486 config.Cmd = serviceDefinition.Command 487 } 488 config.Entrypoint = e.overwriteEntrypoint(&serviceDefinition) 489 490 hostConfig := &container.HostConfig{ 491 DNS: e.Config.Docker.DNS, 492 DNSSearch: e.Config.Docker.DNSSearch, 493 RestartPolicy: neverRestartPolicy, 494 ExtraHosts: e.Config.Docker.ExtraHosts, 495 Privileged: e.Config.Docker.Privileged, 496 NetworkMode: container.NetworkMode(e.Config.Docker.NetworkMode), 497 Binds: e.volumesManager.Binds(), 498 ShmSize: e.Config.Docker.ShmSize, 499 VolumesFrom: e.volumesManager.ContainerIDs(), 500 Tmpfs: e.Config.Docker.ServicesTmpfs, 501 LogConfig: container.LogConfig{ 502 Type: "json-file", 503 }, 504 } 505 506 e.Debugln("Creating service container", containerName, "...") 507 resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName) 508 if err != nil { 509 return nil, err 510 } 511 512 e.Debugln("Starting service container", resp.ID, "...") 513 err = e.client.ContainerStart(e.Context, resp.ID, types.ContainerStartOptions{}) 514 if err != nil { 515 e.temporary = append(e.temporary, resp.ID) 516 return nil, err 517 } 518 519 return fakeContainer(resp.ID, containerName), nil 520 } 521 522 func (e *executor) getServicesDefinitions() (common.Services, error) { 523 serviceDefinitions := common.Services{} 524 for _, service := range e.Config.Docker.Services { 525 serviceDefinitions = append(serviceDefinitions, common.Image{Name: service}) 526 } 527 528 for _, service := range e.Build.Services { 529 serviceName := e.Build.GetAllVariables().ExpandValue(service.Name) 530 err := e.verifyAllowedImage(serviceName, "services", e.Config.Docker.AllowedServices, e.Config.Docker.Services) 531 if err != nil { 532 return nil, err 533 } 534 535 service.Name = serviceName 536 serviceDefinitions = append(serviceDefinitions, service) 537 } 538 539 return serviceDefinitions, nil 540 } 541 542 func (e *executor) waitForServices() { 543 waitForServicesTimeout := e.Config.Docker.WaitForServicesTimeout 544 if waitForServicesTimeout == 0 { 545 waitForServicesTimeout = common.DefaultWaitForServicesTimeout 546 } 547 548 // wait for all services to came up 549 if waitForServicesTimeout > 0 && len(e.services) > 0 { 550 e.Println("Waiting for services to be up and running...") 551 wg := sync.WaitGroup{} 552 for _, service := range e.services { 553 wg.Add(1) 554 go func(service *types.Container) { 555 e.waitForServiceContainer(service, time.Duration(waitForServicesTimeout)*time.Second) 556 wg.Done() 557 }(service) 558 } 559 wg.Wait() 560 } 561 } 562 563 func (e *executor) buildServiceLinks(linksMap map[string]*types.Container) (links []string) { 564 for linkName, linkee := range linksMap { 565 newContainer, err := e.client.ContainerInspect(e.Context, linkee.ID) 566 if err != nil { 567 continue 568 } 569 if newContainer.State.Running { 570 links = append(links, linkee.ID+":"+linkName) 571 } 572 } 573 return 574 } 575 576 func (e *executor) createFromServiceDefinition(serviceIndex int, serviceDefinition common.Image, linksMap map[string]*types.Container) (err error) { 577 var container *types.Container 578 579 service, version, imageName, linkNames := e.splitServiceAndVersion(serviceDefinition.Name) 580 581 if serviceDefinition.Alias != "" { 582 linkNames = append(linkNames, serviceDefinition.Alias) 583 } 584 585 for _, linkName := range linkNames { 586 if linksMap[linkName] != nil { 587 e.Warningln("Service", serviceDefinition.Name, "is already created. Ignoring.") 588 continue 589 } 590 591 // Create service if not yet created 592 if container == nil { 593 container, err = e.createService(serviceIndex, service, version, imageName, serviceDefinition) 594 if err != nil { 595 return 596 } 597 e.Debugln("Created service", serviceDefinition.Name, "as", container.ID) 598 e.services = append(e.services, container) 599 e.temporary = append(e.temporary, container.ID) 600 } 601 linksMap[linkName] = container 602 } 603 return 604 } 605 606 func (e *executor) createServices() (err error) { 607 e.SetCurrentStage(DockerExecutorStageCreatingServices) 608 e.Debugln("Creating services...") 609 610 servicesDefinitions, err := e.getServicesDefinitions() 611 if err != nil { 612 return 613 } 614 615 linksMap := make(map[string]*types.Container) 616 617 for index, serviceDefinition := range servicesDefinitions { 618 err = e.createFromServiceDefinition(index, serviceDefinition, linksMap) 619 if err != nil { 620 return 621 } 622 } 623 624 e.waitForServices() 625 626 e.links = e.buildServiceLinks(linksMap) 627 return 628 } 629 630 func (e *executor) getValidContainers(containers []string) []string { 631 var newContainers []string 632 633 for _, container := range containers { 634 if _, err := e.client.ContainerInspect(e.Context, container); err == nil { 635 newContainers = append(newContainers, container) 636 } 637 } 638 639 return newContainers 640 } 641 642 func (e *executor) createContainer(containerType string, imageDefinition common.Image, cmd []string, allowedInternalImages []string) (*types.ContainerJSON, error) { 643 if e.volumesManager == nil { 644 return nil, errVolumesManagerUndefined 645 } 646 647 image, err := e.expandAndGetDockerImage(imageDefinition.Name, allowedInternalImages) 648 if err != nil { 649 return nil, err 650 } 651 652 hostname := e.Config.Docker.Hostname 653 if hostname == "" { 654 hostname = e.Build.ProjectUniqueName() 655 } 656 657 // Always create unique, but sequential name 658 containerIndex := len(e.builds) 659 containerName := e.Build.ProjectUniqueName() + "-" + 660 containerType + "-" + strconv.Itoa(containerIndex) 661 662 config := &container.Config{ 663 Image: image.ID, 664 Hostname: hostname, 665 Cmd: cmd, 666 Labels: e.getLabels(containerType), 667 Tty: false, 668 AttachStdin: true, 669 AttachStdout: true, 670 AttachStderr: true, 671 OpenStdin: true, 672 StdinOnce: true, 673 Env: append(e.Build.GetAllVariables().StringList(), e.BuildShell.Environment...), 674 } 675 676 config.Entrypoint = e.overwriteEntrypoint(&imageDefinition) 677 678 nanoCPUs, err := e.Config.Docker.GetNanoCPUs() 679 if err != nil { 680 return nil, err 681 } 682 683 // By default we use caches container, 684 // but in later phases we hook to previous build container 685 volumesFrom := e.volumesManager.ContainerIDs() 686 if len(e.builds) > 0 { 687 volumesFrom = []string{ 688 e.builds[len(e.builds)-1], 689 } 690 } 691 692 hostConfig := &container.HostConfig{ 693 Resources: container.Resources{ 694 Memory: e.Config.Docker.GetMemory(), 695 MemorySwap: e.Config.Docker.GetMemorySwap(), 696 MemoryReservation: e.Config.Docker.GetMemoryReservation(), 697 CpusetCpus: e.Config.Docker.CPUSetCPUs, 698 NanoCPUs: nanoCPUs, 699 Devices: e.devices, 700 OomKillDisable: e.Config.Docker.GetOomKillDisable(), 701 }, 702 DNS: e.Config.Docker.DNS, 703 DNSSearch: e.Config.Docker.DNSSearch, 704 Runtime: e.Config.Docker.Runtime, 705 Privileged: e.Config.Docker.Privileged, 706 UsernsMode: container.UsernsMode(e.Config.Docker.UsernsMode), 707 CapAdd: e.Config.Docker.CapAdd, 708 CapDrop: e.Config.Docker.CapDrop, 709 SecurityOpt: e.Config.Docker.SecurityOpt, 710 RestartPolicy: neverRestartPolicy, 711 ExtraHosts: e.Config.Docker.ExtraHosts, 712 NetworkMode: container.NetworkMode(e.Config.Docker.NetworkMode), 713 Links: append(e.Config.Docker.Links, e.links...), 714 Binds: e.volumesManager.Binds(), 715 ShmSize: e.Config.Docker.ShmSize, 716 VolumeDriver: e.Config.Docker.VolumeDriver, 717 VolumesFrom: append(e.Config.Docker.VolumesFrom, volumesFrom...), 718 LogConfig: container.LogConfig{ 719 Type: "json-file", 720 }, 721 Tmpfs: e.Config.Docker.Tmpfs, 722 Sysctls: e.Config.Docker.SysCtls, 723 } 724 725 // this will fail potentially some builds if there's name collision 726 e.removeContainer(e.Context, containerName) 727 728 e.Debugln("Creating container", containerName, "...") 729 resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName) 730 if err != nil { 731 if resp.ID != "" { 732 e.temporary = append(e.temporary, resp.ID) 733 } 734 return nil, err 735 } 736 737 inspect, err := e.client.ContainerInspect(e.Context, resp.ID) 738 if err != nil { 739 e.temporary = append(e.temporary, resp.ID) 740 return nil, err 741 } 742 743 e.builds = append(e.builds, resp.ID) 744 e.temporary = append(e.temporary, resp.ID) 745 return &inspect, nil 746 } 747 748 func (e *executor) killContainer(id string, waitCh chan error) (err error) { 749 for { 750 e.disconnectNetwork(e.Context, id) 751 e.Debugln("Killing container", id, "...") 752 e.client.ContainerKill(e.Context, id, "SIGKILL") 753 754 // Wait for signal that container were killed 755 // or retry after some time 756 select { 757 case err = <-waitCh: 758 return 759 760 case <-time.After(time.Second): 761 } 762 } 763 } 764 765 func (e *executor) waitForContainer(ctx context.Context, id string) error { 766 e.Debugln("Waiting for container", id, "...") 767 768 retries := 0 769 770 // Use active wait 771 for ctx.Err() == nil { 772 container, err := e.client.ContainerInspect(ctx, id) 773 if err != nil { 774 if docker_helpers.IsErrNotFound(err) { 775 return err 776 } 777 778 if retries > 3 { 779 return err 780 } 781 782 retries++ 783 time.Sleep(time.Second) 784 continue 785 } 786 787 // Reset retry timer 788 retries = 0 789 790 if container.State.Running { 791 time.Sleep(time.Second) 792 continue 793 } 794 795 if container.State.ExitCode != 0 { 796 return &common.BuildError{ 797 Inner: fmt.Errorf("exit code %d", container.State.ExitCode), 798 } 799 } 800 801 return nil 802 } 803 804 return ctx.Err() 805 } 806 807 func (e *executor) watchContainer(ctx context.Context, id string, input io.Reader) (err error) { 808 options := types.ContainerAttachOptions{ 809 Stream: true, 810 Stdin: true, 811 Stdout: true, 812 Stderr: true, 813 } 814 815 e.Debugln("Attaching to container", id, "...") 816 hijacked, err := e.client.ContainerAttach(ctx, id, options) 817 if err != nil { 818 return 819 } 820 defer hijacked.Close() 821 822 e.Debugln("Starting container", id, "...") 823 err = e.client.ContainerStart(ctx, id, types.ContainerStartOptions{}) 824 if err != nil { 825 return 826 } 827 828 e.Debugln("Waiting for attach to finish", id, "...") 829 attachCh := make(chan error, 2) 830 831 // Copy any output to the build trace 832 go func() { 833 _, err := stdcopy.StdCopy(e.Trace, e.Trace, hijacked.Reader) 834 if err != nil { 835 attachCh <- err 836 } 837 }() 838 839 // Write the input to the container and close its STDIN to get it to finish 840 go func() { 841 _, err := io.Copy(hijacked.Conn, input) 842 hijacked.CloseWrite() 843 if err != nil { 844 attachCh <- err 845 } 846 }() 847 848 waitCh := make(chan error, 1) 849 go func() { 850 waitCh <- e.waitForContainer(e.Context, id) 851 }() 852 853 select { 854 case <-ctx.Done(): 855 e.killContainer(id, waitCh) 856 err = errors.New("Aborted") 857 858 case err = <-attachCh: 859 e.killContainer(id, waitCh) 860 e.Debugln("Container", id, "finished with", err) 861 862 case err = <-waitCh: 863 e.Debugln("Container", id, "finished with", err) 864 } 865 return 866 } 867 868 func (e *executor) removeContainer(ctx context.Context, id string) error { 869 e.disconnectNetwork(ctx, id) 870 options := types.ContainerRemoveOptions{ 871 RemoveVolumes: true, 872 Force: true, 873 } 874 err := e.client.ContainerRemove(ctx, id, options) 875 e.Debugln("Removed container", id, "with", err) 876 return err 877 } 878 879 func (e *executor) disconnectNetwork(ctx context.Context, id string) error { 880 netList, err := e.client.NetworkList(ctx, types.NetworkListOptions{}) 881 if err != nil { 882 e.Debugln("Can't get network list. ListNetworks exited with", err) 883 return err 884 } 885 886 for _, network := range netList { 887 for _, pluggedContainer := range network.Containers { 888 if id == pluggedContainer.Name { 889 err = e.client.NetworkDisconnect(ctx, network.ID, id, true) 890 if err != nil { 891 e.Warningln("Can't disconnect possibly zombie container", pluggedContainer.Name, "from network", network.Name, "->", err) 892 } else { 893 e.Warningln("Possibly zombie container", pluggedContainer.Name, "is disconnected from network", network.Name) 894 } 895 break 896 } 897 } 898 } 899 return err 900 } 901 902 func (e *executor) verifyAllowedImage(image, optionName string, allowedImages []string, internalImages []string) error { 903 for _, allowedImage := range allowedImages { 904 ok, _ := zglob.Match(allowedImage, image) 905 if ok { 906 return nil 907 } 908 } 909 910 for _, internalImage := range internalImages { 911 if internalImage == image { 912 return nil 913 } 914 } 915 916 if len(allowedImages) != 0 { 917 e.Println() 918 e.Errorln("The", image, "is not present on list of allowed", optionName) 919 for _, allowedImage := range allowedImages { 920 e.Println("-", allowedImage) 921 } 922 e.Println() 923 } else { 924 // by default allow to override the image name 925 return nil 926 } 927 928 e.Println("Please check runner's configuration: http://doc.gitlab.com/ci/docker/using_docker_images.html#overwrite-image-and-services") 929 return errors.New("invalid image") 930 } 931 932 func (e *executor) expandImageName(imageName string, allowedInternalImages []string) (string, error) { 933 if imageName != "" { 934 image := e.Build.GetAllVariables().ExpandValue(imageName) 935 allowedInternalImages = append(allowedInternalImages, e.Config.Docker.Image) 936 err := e.verifyAllowedImage(image, "images", e.Config.Docker.AllowedImages, allowedInternalImages) 937 if err != nil { 938 return "", err 939 } 940 return image, nil 941 } 942 943 if e.Config.Docker.Image == "" { 944 return "", errors.New("No Docker image specified to run the build in") 945 } 946 947 return e.Config.Docker.Image, nil 948 } 949 950 func (e *executor) overwriteEntrypoint(image *common.Image) []string { 951 if len(image.Entrypoint) > 0 { 952 if !e.Config.Docker.DisableEntrypointOverwrite { 953 return image.Entrypoint 954 } 955 956 e.Warningln("Entrypoint override disabled") 957 } 958 959 return nil 960 } 961 962 func (e *executor) connectDocker() error { 963 client, err := docker_helpers.New(e.Config.Docker.DockerCredentials, "") 964 if err != nil { 965 return err 966 } 967 e.client = client 968 969 e.info, err = client.Info(e.Context) 970 if err != nil { 971 return err 972 } 973 974 err = e.validateOSType() 975 if err != nil { 976 return err 977 } 978 979 e.helperImageInfo, err = helperimage.Get(common.REVISION, helperimage.Config{ 980 OSType: e.info.OSType, 981 Architecture: e.info.Architecture, 982 OperatingSystem: e.info.OperatingSystem, 983 }) 984 985 return err 986 } 987 988 // validateOSType checks if the ExecutorOptions metadata matches with the docker 989 // info response. 990 func (e *executor) validateOSType() error { 991 executorOSType := e.ExecutorOptions.Metadata[metadataOSType] 992 if executorOSType == "" { 993 return common.MakeBuildError("%s does not have any OSType specified", e.Config.Executor) 994 } 995 996 if executorOSType != e.info.OSType { 997 return common.MakeBuildError( 998 "executor requires OSType=%s, but Docker Engine supports only OSType=%s", 999 executorOSType, e.info.OSType, 1000 ) 1001 } 1002 1003 return nil 1004 } 1005 1006 func (e *executor) createDependencies() error { 1007 createDependenciesStrategy := []func() error{ 1008 e.bindDevices, 1009 e.createVolumesManager, 1010 e.createVolumes, 1011 e.createBuildVolume, 1012 e.createServices, 1013 } 1014 1015 if e.Build.IsFeatureFlagOn(featureflags.UseLegacyVolumesMountingOrder) { 1016 // TODO: Remove in 12.6 1017 createDependenciesStrategy = []func() error{ 1018 e.bindDevices, 1019 e.createVolumesManager, 1020 e.createBuildVolume, 1021 e.createServices, 1022 e.createVolumes, 1023 } 1024 } 1025 1026 for _, setup := range createDependenciesStrategy { 1027 err := setup() 1028 if err != nil { 1029 return err 1030 } 1031 } 1032 1033 return nil 1034 } 1035 1036 func (e *executor) createVolumes() error { 1037 e.SetCurrentStage(DockerExecutorStageCreatingUserVolumes) 1038 e.Debugln("Creating user-defined volumes...") 1039 1040 if e.volumesManager == nil { 1041 return errVolumesManagerUndefined 1042 } 1043 1044 for _, volume := range e.Config.Docker.Volumes { 1045 err := e.volumesManager.Create(volume) 1046 if err == volumes.ErrCacheVolumesDisabled { 1047 e.Warningln(fmt.Sprintf( 1048 "Container based cache volumes creation is disabled. Will not create volume for %q", 1049 volume, 1050 )) 1051 continue 1052 } 1053 1054 if err != nil { 1055 return err 1056 } 1057 } 1058 1059 return nil 1060 } 1061 1062 func (e *executor) createBuildVolume() error { 1063 e.SetCurrentStage(DockerExecutorStageCreatingBuildVolumes) 1064 e.Debugln("Creating build volume...") 1065 1066 if e.volumesManager == nil { 1067 return errVolumesManagerUndefined 1068 } 1069 1070 jobsDir := e.Build.RootDir 1071 1072 // TODO: Remove in 12.3 1073 if e.Build.IsFeatureFlagOn(featureflags.UseLegacyBuildsDirForDocker) { 1074 // Cache Git sources: 1075 // take path of the projects directory, 1076 // because we use `rm -rf` which could remove the mounted volume 1077 jobsDir = path.Dir(e.Build.FullProjectDir()) 1078 } 1079 1080 var err error 1081 1082 if e.Build.GetGitStrategy() == common.GitFetch { 1083 err = e.volumesManager.Create(jobsDir) 1084 if err == nil { 1085 return nil 1086 } 1087 1088 if err == volumes.ErrCacheVolumesDisabled { 1089 err = e.volumesManager.CreateTemporary(jobsDir) 1090 } 1091 } else { 1092 err = e.volumesManager.CreateTemporary(jobsDir) 1093 } 1094 1095 if err != nil { 1096 if _, ok := err.(*volumes.ErrVolumeAlreadyDefined); !ok { 1097 return err 1098 } 1099 } 1100 1101 return nil 1102 } 1103 1104 func (e *executor) Prepare(options common.ExecutorPrepareOptions) error { 1105 e.SetCurrentStage(DockerExecutorStagePrepare) 1106 1107 if options.Config.Docker == nil { 1108 return errors.New("missing docker configuration") 1109 } 1110 1111 e.AbstractExecutor.PrepareConfiguration(options) 1112 1113 err := e.connectDocker() 1114 if err != nil { 1115 return err 1116 } 1117 1118 err = e.prepareBuildsDir(options) 1119 if err != nil { 1120 return err 1121 } 1122 1123 err = e.AbstractExecutor.PrepareBuildAndShell() 1124 if err != nil { 1125 return err 1126 } 1127 1128 if e.BuildShell.PassFile { 1129 return errors.New("docker doesn't support shells that require script file") 1130 } 1131 1132 imageName, err := e.expandImageName(e.Build.Image.Name, []string{}) 1133 if err != nil { 1134 return err 1135 } 1136 1137 e.Println("Using Docker executor with image", imageName, "...") 1138 1139 err = e.createDependencies() 1140 if err != nil { 1141 return err 1142 } 1143 return nil 1144 } 1145 1146 func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error { 1147 if e.volumeParser == nil { 1148 return common.MakeBuildError("missing volume parser") 1149 } 1150 1151 isHostMounted, err := volumes.IsHostMountedVolume(e.volumeParser, e.RootDir(), options.Config.Docker.Volumes...) 1152 if err != nil { 1153 return &common.BuildError{Inner: err} 1154 } 1155 1156 // We need to set proper value for e.SharedBuildsDir because 1157 // it's required to properly start the job, what is done inside of 1158 // e.AbstractExecutor.Prepare() 1159 // And a started job is required for Volumes Manager to work, so it's 1160 // done before the manager is even created. 1161 if isHostMounted { 1162 e.SharedBuildsDir = true 1163 } 1164 1165 return nil 1166 } 1167 1168 func (e *executor) Cleanup() { 1169 e.SetCurrentStage(DockerExecutorStageCleanup) 1170 1171 var wg sync.WaitGroup 1172 1173 ctx, cancel := context.WithTimeout(context.Background(), dockerCleanupTimeout) 1174 defer cancel() 1175 1176 remove := func(id string) { 1177 wg.Add(1) 1178 go func() { 1179 e.removeContainer(ctx, id) 1180 wg.Done() 1181 }() 1182 } 1183 1184 for _, temporaryID := range e.temporary { 1185 remove(temporaryID) 1186 } 1187 1188 if e.volumesManager != nil { 1189 <-e.volumesManager.Cleanup(ctx) 1190 } 1191 1192 wg.Wait() 1193 1194 if e.client != nil { 1195 e.client.Close() 1196 } 1197 1198 e.AbstractExecutor.Cleanup() 1199 } 1200 1201 type serviceHealthCheckError struct { 1202 Inner error 1203 Logs string 1204 } 1205 1206 func (e *serviceHealthCheckError) Error() string { 1207 if e.Inner == nil { 1208 return "serviceHealthCheckError" 1209 } 1210 1211 return e.Inner.Error() 1212 } 1213 1214 func (e *executor) runServiceHealthCheckContainer(service *types.Container, timeout time.Duration) error { 1215 waitImage, err := e.getPrebuiltImage() 1216 if err != nil { 1217 return fmt.Errorf("getPrebuiltImage: %v", err) 1218 } 1219 1220 containerName := service.Names[0] + "-wait-for-service" 1221 1222 cmd := []string{"gitlab-runner-helper", "health-check"} 1223 // TODO: Remove in 12.0 to start using the command from `gitlab-runner-helper` 1224 if e.checkOutdatedHelperImage() { 1225 e.Debugln(featureflags.DockerHelperImageV2, "is not set, falling back to old command") 1226 cmd = []string{"gitlab-runner-service"} 1227 } 1228 1229 config := &container.Config{ 1230 Cmd: cmd, 1231 Image: waitImage.ID, 1232 Labels: e.getLabels("wait", "wait="+service.ID), 1233 } 1234 hostConfig := &container.HostConfig{ 1235 RestartPolicy: neverRestartPolicy, 1236 Links: []string{service.Names[0] + ":service"}, 1237 NetworkMode: container.NetworkMode(e.Config.Docker.NetworkMode), 1238 LogConfig: container.LogConfig{ 1239 Type: "json-file", 1240 }, 1241 } 1242 e.Debugln("Waiting for service container", containerName, "to be up and running...") 1243 resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName) 1244 if err != nil { 1245 return fmt.Errorf("ContainerCreate: %v", err) 1246 } 1247 defer e.removeContainer(e.Context, resp.ID) 1248 err = e.client.ContainerStart(e.Context, resp.ID, types.ContainerStartOptions{}) 1249 if err != nil { 1250 return fmt.Errorf("ContainerStart: %v", err) 1251 } 1252 1253 waitResult := make(chan error, 1) 1254 go func() { 1255 waitResult <- e.waitForContainer(e.Context, resp.ID) 1256 }() 1257 1258 // these are warnings and they don't make the build fail 1259 select { 1260 case err := <-waitResult: 1261 if err == nil { 1262 return nil 1263 } 1264 1265 return &serviceHealthCheckError{ 1266 Inner: err, 1267 Logs: e.readContainerLogs(resp.ID), 1268 } 1269 case <-time.After(timeout): 1270 return &serviceHealthCheckError{ 1271 Inner: fmt.Errorf("service %q timeout", containerName), 1272 Logs: e.readContainerLogs(resp.ID), 1273 } 1274 } 1275 } 1276 1277 func (e *executor) waitForServiceContainer(service *types.Container, timeout time.Duration) error { 1278 err := e.runServiceHealthCheckContainer(service, timeout) 1279 if err == nil { 1280 return nil 1281 } 1282 1283 var buffer bytes.Buffer 1284 buffer.WriteString("\n") 1285 buffer.WriteString(helpers.ANSI_YELLOW + "*** WARNING:" + helpers.ANSI_RESET + " Service " + service.Names[0] + " probably didn't start properly.\n") 1286 buffer.WriteString("\n") 1287 buffer.WriteString("Health check error:\n") 1288 buffer.WriteString(strings.TrimSpace(err.Error())) 1289 buffer.WriteString("\n") 1290 1291 if healtCheckErr, ok := err.(*serviceHealthCheckError); ok { 1292 buffer.WriteString("\n") 1293 buffer.WriteString("Health check container logs:\n") 1294 buffer.WriteString(healtCheckErr.Logs) 1295 buffer.WriteString("\n") 1296 } 1297 1298 buffer.WriteString("\n") 1299 buffer.WriteString("Service container logs:\n") 1300 buffer.WriteString(e.readContainerLogs(service.ID)) 1301 buffer.WriteString("\n") 1302 1303 buffer.WriteString("\n") 1304 buffer.WriteString(helpers.ANSI_YELLOW + "*********" + helpers.ANSI_RESET + "\n") 1305 buffer.WriteString("\n") 1306 io.Copy(e.Trace, &buffer) 1307 return err 1308 } 1309 1310 func (e *executor) readContainerLogs(containerID string) string { 1311 var containerBuffer bytes.Buffer 1312 1313 options := types.ContainerLogsOptions{ 1314 ShowStdout: true, 1315 ShowStderr: true, 1316 Timestamps: true, 1317 } 1318 1319 hijacked, err := e.client.ContainerLogs(e.Context, containerID, options) 1320 if err != nil { 1321 return strings.TrimSpace(err.Error()) 1322 } 1323 defer hijacked.Close() 1324 1325 stdcopy.StdCopy(&containerBuffer, &containerBuffer, hijacked) 1326 containerLog := containerBuffer.String() 1327 return strings.TrimSpace(containerLog) 1328 }