gitlab.com/jfprevost/gitlab-runner-notlscheck@v11.11.4+incompatible/executors/docker/executor_docker.go (about)

     1  package docker
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	"errors"
     7  	"fmt"
     8  	"io"
     9  	"os"
    10  	"path"
    11  	"path/filepath"
    12  	"regexp"
    13  	"strconv"
    14  	"strings"
    15  	"sync"
    16  	"time"
    17  
    18  	"github.com/docker/distribution/reference"
    19  	"github.com/docker/docker/api/types"
    20  	"github.com/docker/docker/api/types/container"
    21  	"github.com/docker/docker/pkg/stdcopy"
    22  	"github.com/kardianos/osext"
    23  	"github.com/mattn/go-zglob"
    24  	"github.com/sirupsen/logrus"
    25  
    26  	"gitlab.com/gitlab-org/gitlab-runner/common"
    27  	"gitlab.com/gitlab-org/gitlab-runner/executors"
    28  	"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes"
    29  	"gitlab.com/gitlab-org/gitlab-runner/executors/docker/internal/volumes/parser"
    30  	"gitlab.com/gitlab-org/gitlab-runner/helpers"
    31  	docker_helpers "gitlab.com/gitlab-org/gitlab-runner/helpers/docker"
    32  	"gitlab.com/gitlab-org/gitlab-runner/helpers/docker/helperimage"
    33  	"gitlab.com/gitlab-org/gitlab-runner/helpers/featureflags"
    34  )
    35  
    36  const (
    37  	DockerExecutorStagePrepare common.ExecutorStage = "docker_prepare"
    38  	DockerExecutorStageRun     common.ExecutorStage = "docker_run"
    39  	DockerExecutorStageCleanup common.ExecutorStage = "docker_cleanup"
    40  
    41  	DockerExecutorStageCreatingBuildVolumes common.ExecutorStage = "docker_creating_build_volumes"
    42  	DockerExecutorStageCreatingServices     common.ExecutorStage = "docker_creating_services"
    43  	DockerExecutorStageCreatingUserVolumes  common.ExecutorStage = "docker_creating_user_volumes"
    44  	DockerExecutorStagePullingImage         common.ExecutorStage = "docker_pulling_image"
    45  )
    46  
    47  var DockerPrebuiltImagesPaths []string
    48  
    49  var neverRestartPolicy = container.RestartPolicy{Name: "no"}
    50  
    51  var errVolumesManagerUndefined = errors.New("volumesManager is undefined")
    52  
    53  type executor struct {
    54  	executors.AbstractExecutor
    55  	client       docker_helpers.Client
    56  	volumeParser parser.Parser
    57  	info         types.Info
    58  
    59  	temporary []string // IDs of containers that should be removed
    60  
    61  	builds   []string // IDs of successfully created build containers
    62  	services []*types.Container
    63  
    64  	links []string
    65  
    66  	devices []container.DeviceMapping
    67  
    68  	helperImageInfo helperimage.Info
    69  
    70  	usedImages     map[string]string
    71  	usedImagesLock sync.RWMutex
    72  
    73  	volumesManager volumes.Manager
    74  }
    75  
    76  func init() {
    77  	runnerFolder, err := osext.ExecutableFolder()
    78  	if err != nil {
    79  		logrus.Errorln("Docker executor: unable to detect gitlab-runner folder, prebuilt image helpers will be loaded from DockerHub.", err)
    80  	}
    81  
    82  	DockerPrebuiltImagesPaths = []string{
    83  		filepath.Join(runnerFolder, "helper-images"),
    84  		filepath.Join(runnerFolder, "out/helper-images"),
    85  	}
    86  }
    87  
    88  func (e *executor) getServiceVariables() []string {
    89  	return e.Build.GetAllVariables().PublicOrInternal().StringList()
    90  }
    91  
    92  func (e *executor) getUserAuthConfiguration(indexName string) *types.AuthConfig {
    93  	if e.Build == nil {
    94  		return nil
    95  	}
    96  
    97  	buf := bytes.NewBufferString(e.Build.GetDockerAuthConfig())
    98  	authConfigs, _ := docker_helpers.ReadAuthConfigsFromReader(buf)
    99  	if authConfigs != nil {
   100  		return docker_helpers.ResolveDockerAuthConfig(indexName, authConfigs)
   101  	}
   102  	return nil
   103  }
   104  
   105  func (e *executor) getBuildAuthConfiguration(indexName string) *types.AuthConfig {
   106  	if e.Build == nil {
   107  		return nil
   108  	}
   109  
   110  	authConfigs := make(map[string]types.AuthConfig)
   111  
   112  	for _, credentials := range e.Build.Credentials {
   113  		if credentials.Type != "registry" {
   114  			continue
   115  		}
   116  
   117  		authConfigs[credentials.URL] = types.AuthConfig{
   118  			Username:      credentials.Username,
   119  			Password:      credentials.Password,
   120  			ServerAddress: credentials.URL,
   121  		}
   122  	}
   123  
   124  	if authConfigs != nil {
   125  		return docker_helpers.ResolveDockerAuthConfig(indexName, authConfigs)
   126  	}
   127  	return nil
   128  }
   129  
   130  func (e *executor) getHomeDirAuthConfiguration(indexName string) *types.AuthConfig {
   131  	authConfigs, _ := docker_helpers.ReadDockerAuthConfigsFromHomeDir(e.Shell().User)
   132  	if authConfigs != nil {
   133  		return docker_helpers.ResolveDockerAuthConfig(indexName, authConfigs)
   134  	}
   135  	return nil
   136  }
   137  
   138  func (e *executor) getAuthConfig(imageName string) *types.AuthConfig {
   139  	indexName, _ := docker_helpers.SplitDockerImageName(imageName)
   140  
   141  	authConfig := e.getUserAuthConfiguration(indexName)
   142  	if authConfig == nil {
   143  		authConfig = e.getHomeDirAuthConfiguration(indexName)
   144  	}
   145  	if authConfig == nil {
   146  		authConfig = e.getBuildAuthConfiguration(indexName)
   147  	}
   148  
   149  	if authConfig != nil {
   150  		e.Debugln("Using", authConfig.Username, "to connect to", authConfig.ServerAddress,
   151  			"in order to resolve", imageName, "...")
   152  		return authConfig
   153  	}
   154  
   155  	e.Debugln(fmt.Sprintf("No credentials found for %v", indexName))
   156  	return nil
   157  }
   158  
   159  func (e *executor) pullDockerImage(imageName string, ac *types.AuthConfig) (*types.ImageInspect, error) {
   160  	e.SetCurrentStage(DockerExecutorStagePullingImage)
   161  	e.Println("Pulling docker image", imageName, "...")
   162  
   163  	ref := imageName
   164  	// Add :latest to limit the download results
   165  	if !strings.ContainsAny(ref, ":@") {
   166  		ref += ":latest"
   167  	}
   168  
   169  	options := types.ImagePullOptions{}
   170  	if ac != nil {
   171  		options.RegistryAuth, _ = docker_helpers.EncodeAuthConfig(ac)
   172  	}
   173  
   174  	errorRegexp := regexp.MustCompile("(repository does not exist|not found)")
   175  	if err := e.client.ImagePullBlocking(e.Context, ref, options); err != nil {
   176  		if errorRegexp.MatchString(err.Error()) {
   177  			return nil, &common.BuildError{Inner: err}
   178  		}
   179  		return nil, err
   180  	}
   181  
   182  	image, _, err := e.client.ImageInspectWithRaw(e.Context, imageName)
   183  	return &image, err
   184  }
   185  
   186  func (e *executor) getDockerImage(imageName string) (image *types.ImageInspect, err error) {
   187  	pullPolicy, err := e.Config.Docker.PullPolicy.Get()
   188  	if err != nil {
   189  		return nil, err
   190  	}
   191  
   192  	authConfig := e.getAuthConfig(imageName)
   193  
   194  	e.Debugln("Looking for image", imageName, "...")
   195  	existingImage, _, err := e.client.ImageInspectWithRaw(e.Context, imageName)
   196  
   197  	// Return early if we already used that image
   198  	if err == nil && e.wasImageUsed(imageName, existingImage.ID) {
   199  		return &existingImage, nil
   200  	}
   201  
   202  	defer func() {
   203  		if err == nil {
   204  			e.markImageAsUsed(imageName, image.ID)
   205  		}
   206  	}()
   207  
   208  	// If never is specified then we return what inspect did return
   209  	if pullPolicy == common.PullPolicyNever {
   210  		return &existingImage, err
   211  	}
   212  
   213  	if err == nil {
   214  		// Don't pull image that is passed by ID
   215  		if existingImage.ID == imageName {
   216  			return &existingImage, nil
   217  		}
   218  
   219  		// If not-present is specified
   220  		if pullPolicy == common.PullPolicyIfNotPresent {
   221  			e.Println("Using locally found image version due to if-not-present pull policy")
   222  			return &existingImage, err
   223  		}
   224  	}
   225  
   226  	return e.pullDockerImage(imageName, authConfig)
   227  }
   228  
   229  func (e *executor) expandAndGetDockerImage(imageName string, allowedImages []string) (*types.ImageInspect, error) {
   230  	imageName, err := e.expandImageName(imageName, allowedImages)
   231  	if err != nil {
   232  		return nil, err
   233  	}
   234  
   235  	image, err := e.getDockerImage(imageName)
   236  	if err != nil {
   237  		return nil, err
   238  	}
   239  
   240  	return image, nil
   241  }
   242  
   243  func (e *executor) loadPrebuiltImage(path, ref, tag string) (*types.ImageInspect, error) {
   244  	file, err := os.OpenFile(path, os.O_RDONLY, 0600)
   245  	if err != nil {
   246  		if os.IsNotExist(err) {
   247  			return nil, err
   248  		}
   249  
   250  		return nil, fmt.Errorf("Cannot load prebuilt image: %s: %q", path, err.Error())
   251  	}
   252  	defer file.Close()
   253  
   254  	e.Debugln("Loading prebuilt image...")
   255  
   256  	source := types.ImageImportSource{
   257  		Source:     file,
   258  		SourceName: "-",
   259  	}
   260  	options := types.ImageImportOptions{Tag: tag}
   261  
   262  	if err := e.client.ImageImportBlocking(e.Context, source, ref, options); err != nil {
   263  		return nil, fmt.Errorf("Failed to import image: %s", err)
   264  	}
   265  
   266  	image, _, err := e.client.ImageInspectWithRaw(e.Context, ref+":"+tag)
   267  	if err != nil {
   268  		e.Debugln("Inspecting imported image", ref, "failed:", err)
   269  		return nil, err
   270  	}
   271  
   272  	return &image, err
   273  }
   274  
   275  func (e *executor) getPrebuiltImage() (*types.ImageInspect, error) {
   276  	if imageNameFromConfig := e.Config.Docker.HelperImage; imageNameFromConfig != "" {
   277  		imageNameFromConfig = common.AppVersion.Variables().ExpandValue(imageNameFromConfig)
   278  
   279  		e.Debugln("Pull configured helper_image for predefined container instead of import bundled image", imageNameFromConfig, "...")
   280  		if !e.Build.IsFeatureFlagOn(featureflags.DockerHelperImageV2) {
   281  			e.Warningln("DEPRECATION: With gitlab-runner 12.0 we will change some tools inside the helper image, please make sure your image is compliant with the new API. https://gitlab.com/gitlab-org/gitlab-runner/issues/4013")
   282  		}
   283  
   284  		return e.getDockerImage(imageNameFromConfig)
   285  	}
   286  
   287  	e.Debugln(fmt.Sprintf("Looking for prebuilt image %s...", e.helperImageInfo))
   288  	image, _, err := e.client.ImageInspectWithRaw(e.Context, e.helperImageInfo.String())
   289  	if err == nil {
   290  		return &image, nil
   291  	}
   292  
   293  	// Try to load prebuilt image from local filesystem
   294  	loadedImage := e.getLocalHelperImage()
   295  	if loadedImage != nil {
   296  		return loadedImage, nil
   297  	}
   298  
   299  	// Fallback to getting image from DockerHub
   300  	e.Debugln(fmt.Sprintf("Loading image form registry: %s", e.helperImageInfo))
   301  	return e.getDockerImage(e.helperImageInfo.String())
   302  }
   303  
   304  func (e *executor) getLocalHelperImage() *types.ImageInspect {
   305  	if !e.helperImageInfo.IsSupportingLocalImport {
   306  		return nil
   307  	}
   308  
   309  	architecture := e.helperImageInfo.Architecture
   310  	for _, dockerPrebuiltImagesPath := range DockerPrebuiltImagesPaths {
   311  		dockerPrebuiltImageFilePath := filepath.Join(dockerPrebuiltImagesPath, "prebuilt-"+architecture+prebuiltImageExtension)
   312  		image, err := e.loadPrebuiltImage(dockerPrebuiltImageFilePath, prebuiltImageName, e.helperImageInfo.Tag)
   313  		if err != nil {
   314  			e.Debugln("Failed to load prebuilt image from:", dockerPrebuiltImageFilePath, "error:", err)
   315  			continue
   316  		}
   317  
   318  		return image
   319  	}
   320  
   321  	return nil
   322  }
   323  
   324  func (e *executor) getBuildImage() (*types.ImageInspect, error) {
   325  	imageName, err := e.expandImageName(e.Build.Image.Name, []string{})
   326  	if err != nil {
   327  		return nil, err
   328  	}
   329  
   330  	// Fetch image
   331  	image, err := e.getDockerImage(imageName)
   332  	if err != nil {
   333  		return nil, err
   334  	}
   335  
   336  	return image, nil
   337  }
   338  
   339  func (e *executor) getLabels(containerType string, otherLabels ...string) map[string]string {
   340  	labels := make(map[string]string)
   341  	labels[dockerLabelPrefix+".job.id"] = strconv.Itoa(e.Build.ID)
   342  	labels[dockerLabelPrefix+".job.sha"] = e.Build.GitInfo.Sha
   343  	labels[dockerLabelPrefix+".job.before_sha"] = e.Build.GitInfo.BeforeSha
   344  	labels[dockerLabelPrefix+".job.ref"] = e.Build.GitInfo.Ref
   345  	labels[dockerLabelPrefix+".project.id"] = strconv.Itoa(e.Build.JobInfo.ProjectID)
   346  	labels[dockerLabelPrefix+".runner.id"] = e.Build.Runner.ShortDescription()
   347  	labels[dockerLabelPrefix+".runner.local_id"] = strconv.Itoa(e.Build.RunnerID)
   348  	labels[dockerLabelPrefix+".type"] = containerType
   349  	for _, label := range otherLabels {
   350  		keyValue := strings.SplitN(label, "=", 2)
   351  		if len(keyValue) == 2 {
   352  			labels[dockerLabelPrefix+"."+keyValue[0]] = keyValue[1]
   353  		}
   354  	}
   355  	return labels
   356  }
   357  
   358  func fakeContainer(id string, names ...string) *types.Container {
   359  	return &types.Container{ID: id, Names: names}
   360  }
   361  
   362  func (e *executor) parseDeviceString(deviceString string) (device container.DeviceMapping, err error) {
   363  	// Split the device string PathOnHost[:PathInContainer[:CgroupPermissions]]
   364  	parts := strings.Split(deviceString, ":")
   365  
   366  	if len(parts) > 3 {
   367  		err = fmt.Errorf("Too many colons")
   368  		return
   369  	}
   370  
   371  	device.PathOnHost = parts[0]
   372  
   373  	// Optional container path
   374  	if len(parts) >= 2 {
   375  		device.PathInContainer = parts[1]
   376  	} else {
   377  		// default: device at same path in container
   378  		device.PathInContainer = device.PathOnHost
   379  	}
   380  
   381  	// Optional permissions
   382  	if len(parts) >= 3 {
   383  		device.CgroupPermissions = parts[2]
   384  	} else {
   385  		// default: rwm, just like 'docker run'
   386  		device.CgroupPermissions = "rwm"
   387  	}
   388  
   389  	return
   390  }
   391  
   392  func (e *executor) bindDevices() (err error) {
   393  	for _, deviceString := range e.Config.Docker.Devices {
   394  		device, err := e.parseDeviceString(deviceString)
   395  		if err != nil {
   396  			err = fmt.Errorf("Failed to parse device string %q: %s", deviceString, err)
   397  			return err
   398  		}
   399  
   400  		e.devices = append(e.devices, device)
   401  	}
   402  	return nil
   403  }
   404  
   405  func (e *executor) wasImageUsed(imageName, imageID string) bool {
   406  	e.usedImagesLock.RLock()
   407  	defer e.usedImagesLock.RUnlock()
   408  
   409  	if e.usedImages[imageName] == imageID {
   410  		return true
   411  	}
   412  	return false
   413  }
   414  
   415  func (e *executor) markImageAsUsed(imageName, imageID string) {
   416  	e.usedImagesLock.Lock()
   417  	defer e.usedImagesLock.Unlock()
   418  
   419  	if e.usedImages == nil {
   420  		e.usedImages = make(map[string]string)
   421  	}
   422  	e.usedImages[imageName] = imageID
   423  
   424  	if imageName != imageID {
   425  		e.Println("Using docker image", imageID, "for", imageName, "...")
   426  	}
   427  }
   428  
   429  func (e *executor) splitServiceAndVersion(serviceDescription string) (service, version, imageName string, linkNames []string) {
   430  	ReferenceRegexpNoPort := regexp.MustCompile(`^(.*?)(|:[0-9]+)(|/.*)$`)
   431  	imageName = serviceDescription
   432  	version = "latest"
   433  
   434  	if match := reference.ReferenceRegexp.FindStringSubmatch(serviceDescription); match != nil {
   435  		matchService := ReferenceRegexpNoPort.FindStringSubmatch(match[1])
   436  		service = matchService[1] + matchService[3]
   437  
   438  		if len(match[2]) > 0 {
   439  			version = match[2]
   440  		} else {
   441  			imageName = match[1] + ":" + version
   442  		}
   443  	} else {
   444  		return
   445  	}
   446  
   447  	linkName := strings.Replace(service, "/", "__", -1)
   448  	linkNames = append(linkNames, linkName)
   449  
   450  	// Create alternative link name according to RFC 1123
   451  	// Where you can use only `a-zA-Z0-9-`
   452  	if alternativeName := strings.Replace(service, "/", "-", -1); linkName != alternativeName {
   453  		linkNames = append(linkNames, alternativeName)
   454  	}
   455  	return
   456  }
   457  
   458  func (e *executor) createService(serviceIndex int, service, version, image string, serviceDefinition common.Image) (*types.Container, error) {
   459  	if len(service) == 0 {
   460  		return nil, errors.New("invalid service name")
   461  	}
   462  
   463  	if e.volumesManager == nil {
   464  		return nil, errVolumesManagerUndefined
   465  	}
   466  
   467  	e.Println("Starting service", service+":"+version, "...")
   468  	serviceImage, err := e.getDockerImage(image)
   469  	if err != nil {
   470  		return nil, err
   471  	}
   472  
   473  	serviceSlug := strings.Replace(service, "/", "__", -1)
   474  	containerName := fmt.Sprintf("%s-%s-%d", e.Build.ProjectUniqueName(), serviceSlug, serviceIndex)
   475  
   476  	// this will fail potentially some builds if there's name collision
   477  	e.removeContainer(e.Context, containerName)
   478  
   479  	config := &container.Config{
   480  		Image:  serviceImage.ID,
   481  		Labels: e.getLabels("service", "service="+service, "service.version="+version),
   482  		Env:    e.getServiceVariables(),
   483  	}
   484  
   485  	if len(serviceDefinition.Command) > 0 {
   486  		config.Cmd = serviceDefinition.Command
   487  	}
   488  	config.Entrypoint = e.overwriteEntrypoint(&serviceDefinition)
   489  
   490  	hostConfig := &container.HostConfig{
   491  		DNS:           e.Config.Docker.DNS,
   492  		DNSSearch:     e.Config.Docker.DNSSearch,
   493  		RestartPolicy: neverRestartPolicy,
   494  		ExtraHosts:    e.Config.Docker.ExtraHosts,
   495  		Privileged:    e.Config.Docker.Privileged,
   496  		NetworkMode:   container.NetworkMode(e.Config.Docker.NetworkMode),
   497  		Binds:         e.volumesManager.Binds(),
   498  		ShmSize:       e.Config.Docker.ShmSize,
   499  		VolumesFrom:   e.volumesManager.ContainerIDs(),
   500  		Tmpfs:         e.Config.Docker.ServicesTmpfs,
   501  		LogConfig: container.LogConfig{
   502  			Type: "json-file",
   503  		},
   504  	}
   505  
   506  	e.Debugln("Creating service container", containerName, "...")
   507  	resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName)
   508  	if err != nil {
   509  		return nil, err
   510  	}
   511  
   512  	e.Debugln("Starting service container", resp.ID, "...")
   513  	err = e.client.ContainerStart(e.Context, resp.ID, types.ContainerStartOptions{})
   514  	if err != nil {
   515  		e.temporary = append(e.temporary, resp.ID)
   516  		return nil, err
   517  	}
   518  
   519  	return fakeContainer(resp.ID, containerName), nil
   520  }
   521  
   522  func (e *executor) getServicesDefinitions() (common.Services, error) {
   523  	serviceDefinitions := common.Services{}
   524  	for _, service := range e.Config.Docker.Services {
   525  		serviceDefinitions = append(serviceDefinitions, common.Image{Name: service})
   526  	}
   527  
   528  	for _, service := range e.Build.Services {
   529  		serviceName := e.Build.GetAllVariables().ExpandValue(service.Name)
   530  		err := e.verifyAllowedImage(serviceName, "services", e.Config.Docker.AllowedServices, e.Config.Docker.Services)
   531  		if err != nil {
   532  			return nil, err
   533  		}
   534  
   535  		service.Name = serviceName
   536  		serviceDefinitions = append(serviceDefinitions, service)
   537  	}
   538  
   539  	return serviceDefinitions, nil
   540  }
   541  
   542  func (e *executor) waitForServices() {
   543  	waitForServicesTimeout := e.Config.Docker.WaitForServicesTimeout
   544  	if waitForServicesTimeout == 0 {
   545  		waitForServicesTimeout = common.DefaultWaitForServicesTimeout
   546  	}
   547  
   548  	// wait for all services to came up
   549  	if waitForServicesTimeout > 0 && len(e.services) > 0 {
   550  		e.Println("Waiting for services to be up and running...")
   551  		wg := sync.WaitGroup{}
   552  		for _, service := range e.services {
   553  			wg.Add(1)
   554  			go func(service *types.Container) {
   555  				e.waitForServiceContainer(service, time.Duration(waitForServicesTimeout)*time.Second)
   556  				wg.Done()
   557  			}(service)
   558  		}
   559  		wg.Wait()
   560  	}
   561  }
   562  
   563  func (e *executor) buildServiceLinks(linksMap map[string]*types.Container) (links []string) {
   564  	for linkName, linkee := range linksMap {
   565  		newContainer, err := e.client.ContainerInspect(e.Context, linkee.ID)
   566  		if err != nil {
   567  			continue
   568  		}
   569  		if newContainer.State.Running {
   570  			links = append(links, linkee.ID+":"+linkName)
   571  		}
   572  	}
   573  	return
   574  }
   575  
   576  func (e *executor) createFromServiceDefinition(serviceIndex int, serviceDefinition common.Image, linksMap map[string]*types.Container) (err error) {
   577  	var container *types.Container
   578  
   579  	service, version, imageName, linkNames := e.splitServiceAndVersion(serviceDefinition.Name)
   580  
   581  	if serviceDefinition.Alias != "" {
   582  		linkNames = append(linkNames, serviceDefinition.Alias)
   583  	}
   584  
   585  	for _, linkName := range linkNames {
   586  		if linksMap[linkName] != nil {
   587  			e.Warningln("Service", serviceDefinition.Name, "is already created. Ignoring.")
   588  			continue
   589  		}
   590  
   591  		// Create service if not yet created
   592  		if container == nil {
   593  			container, err = e.createService(serviceIndex, service, version, imageName, serviceDefinition)
   594  			if err != nil {
   595  				return
   596  			}
   597  			e.Debugln("Created service", serviceDefinition.Name, "as", container.ID)
   598  			e.services = append(e.services, container)
   599  			e.temporary = append(e.temporary, container.ID)
   600  		}
   601  		linksMap[linkName] = container
   602  	}
   603  	return
   604  }
   605  
   606  func (e *executor) createServices() (err error) {
   607  	e.SetCurrentStage(DockerExecutorStageCreatingServices)
   608  	e.Debugln("Creating services...")
   609  
   610  	servicesDefinitions, err := e.getServicesDefinitions()
   611  	if err != nil {
   612  		return
   613  	}
   614  
   615  	linksMap := make(map[string]*types.Container)
   616  
   617  	for index, serviceDefinition := range servicesDefinitions {
   618  		err = e.createFromServiceDefinition(index, serviceDefinition, linksMap)
   619  		if err != nil {
   620  			return
   621  		}
   622  	}
   623  
   624  	e.waitForServices()
   625  
   626  	e.links = e.buildServiceLinks(linksMap)
   627  	return
   628  }
   629  
   630  func (e *executor) getValidContainers(containers []string) []string {
   631  	var newContainers []string
   632  
   633  	for _, container := range containers {
   634  		if _, err := e.client.ContainerInspect(e.Context, container); err == nil {
   635  			newContainers = append(newContainers, container)
   636  		}
   637  	}
   638  
   639  	return newContainers
   640  }
   641  
   642  func (e *executor) createContainer(containerType string, imageDefinition common.Image, cmd []string, allowedInternalImages []string) (*types.ContainerJSON, error) {
   643  	if e.volumesManager == nil {
   644  		return nil, errVolumesManagerUndefined
   645  	}
   646  
   647  	image, err := e.expandAndGetDockerImage(imageDefinition.Name, allowedInternalImages)
   648  	if err != nil {
   649  		return nil, err
   650  	}
   651  
   652  	hostname := e.Config.Docker.Hostname
   653  	if hostname == "" {
   654  		hostname = e.Build.ProjectUniqueName()
   655  	}
   656  
   657  	// Always create unique, but sequential name
   658  	containerIndex := len(e.builds)
   659  	containerName := e.Build.ProjectUniqueName() + "-" +
   660  		containerType + "-" + strconv.Itoa(containerIndex)
   661  
   662  	config := &container.Config{
   663  		Image:        image.ID,
   664  		Hostname:     hostname,
   665  		Cmd:          cmd,
   666  		Labels:       e.getLabels(containerType),
   667  		Tty:          false,
   668  		AttachStdin:  true,
   669  		AttachStdout: true,
   670  		AttachStderr: true,
   671  		OpenStdin:    true,
   672  		StdinOnce:    true,
   673  		Env:          append(e.Build.GetAllVariables().StringList(), e.BuildShell.Environment...),
   674  	}
   675  
   676  	config.Entrypoint = e.overwriteEntrypoint(&imageDefinition)
   677  
   678  	nanoCPUs, err := e.Config.Docker.GetNanoCPUs()
   679  	if err != nil {
   680  		return nil, err
   681  	}
   682  
   683  	// By default we use caches container,
   684  	// but in later phases we hook to previous build container
   685  	volumesFrom := e.volumesManager.ContainerIDs()
   686  	if len(e.builds) > 0 {
   687  		volumesFrom = []string{
   688  			e.builds[len(e.builds)-1],
   689  		}
   690  	}
   691  
   692  	hostConfig := &container.HostConfig{
   693  		Resources: container.Resources{
   694  			Memory:            e.Config.Docker.GetMemory(),
   695  			MemorySwap:        e.Config.Docker.GetMemorySwap(),
   696  			MemoryReservation: e.Config.Docker.GetMemoryReservation(),
   697  			CpusetCpus:        e.Config.Docker.CPUSetCPUs,
   698  			NanoCPUs:          nanoCPUs,
   699  			Devices:           e.devices,
   700  			OomKillDisable:    e.Config.Docker.GetOomKillDisable(),
   701  		},
   702  		DNS:           e.Config.Docker.DNS,
   703  		DNSSearch:     e.Config.Docker.DNSSearch,
   704  		Runtime:       e.Config.Docker.Runtime,
   705  		Privileged:    e.Config.Docker.Privileged,
   706  		UsernsMode:    container.UsernsMode(e.Config.Docker.UsernsMode),
   707  		CapAdd:        e.Config.Docker.CapAdd,
   708  		CapDrop:       e.Config.Docker.CapDrop,
   709  		SecurityOpt:   e.Config.Docker.SecurityOpt,
   710  		RestartPolicy: neverRestartPolicy,
   711  		ExtraHosts:    e.Config.Docker.ExtraHosts,
   712  		NetworkMode:   container.NetworkMode(e.Config.Docker.NetworkMode),
   713  		Links:         append(e.Config.Docker.Links, e.links...),
   714  		Binds:         e.volumesManager.Binds(),
   715  		ShmSize:       e.Config.Docker.ShmSize,
   716  		VolumeDriver:  e.Config.Docker.VolumeDriver,
   717  		VolumesFrom:   append(e.Config.Docker.VolumesFrom, volumesFrom...),
   718  		LogConfig: container.LogConfig{
   719  			Type: "json-file",
   720  		},
   721  		Tmpfs:   e.Config.Docker.Tmpfs,
   722  		Sysctls: e.Config.Docker.SysCtls,
   723  	}
   724  
   725  	// this will fail potentially some builds if there's name collision
   726  	e.removeContainer(e.Context, containerName)
   727  
   728  	e.Debugln("Creating container", containerName, "...")
   729  	resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName)
   730  	if err != nil {
   731  		if resp.ID != "" {
   732  			e.temporary = append(e.temporary, resp.ID)
   733  		}
   734  		return nil, err
   735  	}
   736  
   737  	inspect, err := e.client.ContainerInspect(e.Context, resp.ID)
   738  	if err != nil {
   739  		e.temporary = append(e.temporary, resp.ID)
   740  		return nil, err
   741  	}
   742  
   743  	e.builds = append(e.builds, resp.ID)
   744  	e.temporary = append(e.temporary, resp.ID)
   745  	return &inspect, nil
   746  }
   747  
   748  func (e *executor) killContainer(id string, waitCh chan error) (err error) {
   749  	for {
   750  		e.disconnectNetwork(e.Context, id)
   751  		e.Debugln("Killing container", id, "...")
   752  		e.client.ContainerKill(e.Context, id, "SIGKILL")
   753  
   754  		// Wait for signal that container were killed
   755  		// or retry after some time
   756  		select {
   757  		case err = <-waitCh:
   758  			return
   759  
   760  		case <-time.After(time.Second):
   761  		}
   762  	}
   763  }
   764  
   765  func (e *executor) waitForContainer(ctx context.Context, id string) error {
   766  	e.Debugln("Waiting for container", id, "...")
   767  
   768  	retries := 0
   769  
   770  	// Use active wait
   771  	for ctx.Err() == nil {
   772  		container, err := e.client.ContainerInspect(ctx, id)
   773  		if err != nil {
   774  			if docker_helpers.IsErrNotFound(err) {
   775  				return err
   776  			}
   777  
   778  			if retries > 3 {
   779  				return err
   780  			}
   781  
   782  			retries++
   783  			time.Sleep(time.Second)
   784  			continue
   785  		}
   786  
   787  		// Reset retry timer
   788  		retries = 0
   789  
   790  		if container.State.Running {
   791  			time.Sleep(time.Second)
   792  			continue
   793  		}
   794  
   795  		if container.State.ExitCode != 0 {
   796  			return &common.BuildError{
   797  				Inner: fmt.Errorf("exit code %d", container.State.ExitCode),
   798  			}
   799  		}
   800  
   801  		return nil
   802  	}
   803  
   804  	return ctx.Err()
   805  }
   806  
   807  func (e *executor) watchContainer(ctx context.Context, id string, input io.Reader) (err error) {
   808  	options := types.ContainerAttachOptions{
   809  		Stream: true,
   810  		Stdin:  true,
   811  		Stdout: true,
   812  		Stderr: true,
   813  	}
   814  
   815  	e.Debugln("Attaching to container", id, "...")
   816  	hijacked, err := e.client.ContainerAttach(ctx, id, options)
   817  	if err != nil {
   818  		return
   819  	}
   820  	defer hijacked.Close()
   821  
   822  	e.Debugln("Starting container", id, "...")
   823  	err = e.client.ContainerStart(ctx, id, types.ContainerStartOptions{})
   824  	if err != nil {
   825  		return
   826  	}
   827  
   828  	e.Debugln("Waiting for attach to finish", id, "...")
   829  	attachCh := make(chan error, 2)
   830  
   831  	// Copy any output to the build trace
   832  	go func() {
   833  		_, err := stdcopy.StdCopy(e.Trace, e.Trace, hijacked.Reader)
   834  		if err != nil {
   835  			attachCh <- err
   836  		}
   837  	}()
   838  
   839  	// Write the input to the container and close its STDIN to get it to finish
   840  	go func() {
   841  		_, err := io.Copy(hijacked.Conn, input)
   842  		hijacked.CloseWrite()
   843  		if err != nil {
   844  			attachCh <- err
   845  		}
   846  	}()
   847  
   848  	waitCh := make(chan error, 1)
   849  	go func() {
   850  		waitCh <- e.waitForContainer(e.Context, id)
   851  	}()
   852  
   853  	select {
   854  	case <-ctx.Done():
   855  		e.killContainer(id, waitCh)
   856  		err = errors.New("Aborted")
   857  
   858  	case err = <-attachCh:
   859  		e.killContainer(id, waitCh)
   860  		e.Debugln("Container", id, "finished with", err)
   861  
   862  	case err = <-waitCh:
   863  		e.Debugln("Container", id, "finished with", err)
   864  	}
   865  	return
   866  }
   867  
   868  func (e *executor) removeContainer(ctx context.Context, id string) error {
   869  	e.disconnectNetwork(ctx, id)
   870  	options := types.ContainerRemoveOptions{
   871  		RemoveVolumes: true,
   872  		Force:         true,
   873  	}
   874  	err := e.client.ContainerRemove(ctx, id, options)
   875  	e.Debugln("Removed container", id, "with", err)
   876  	return err
   877  }
   878  
   879  func (e *executor) disconnectNetwork(ctx context.Context, id string) error {
   880  	netList, err := e.client.NetworkList(ctx, types.NetworkListOptions{})
   881  	if err != nil {
   882  		e.Debugln("Can't get network list. ListNetworks exited with", err)
   883  		return err
   884  	}
   885  
   886  	for _, network := range netList {
   887  		for _, pluggedContainer := range network.Containers {
   888  			if id == pluggedContainer.Name {
   889  				err = e.client.NetworkDisconnect(ctx, network.ID, id, true)
   890  				if err != nil {
   891  					e.Warningln("Can't disconnect possibly zombie container", pluggedContainer.Name, "from network", network.Name, "->", err)
   892  				} else {
   893  					e.Warningln("Possibly zombie container", pluggedContainer.Name, "is disconnected from network", network.Name)
   894  				}
   895  				break
   896  			}
   897  		}
   898  	}
   899  	return err
   900  }
   901  
   902  func (e *executor) verifyAllowedImage(image, optionName string, allowedImages []string, internalImages []string) error {
   903  	for _, allowedImage := range allowedImages {
   904  		ok, _ := zglob.Match(allowedImage, image)
   905  		if ok {
   906  			return nil
   907  		}
   908  	}
   909  
   910  	for _, internalImage := range internalImages {
   911  		if internalImage == image {
   912  			return nil
   913  		}
   914  	}
   915  
   916  	if len(allowedImages) != 0 {
   917  		e.Println()
   918  		e.Errorln("The", image, "is not present on list of allowed", optionName)
   919  		for _, allowedImage := range allowedImages {
   920  			e.Println("-", allowedImage)
   921  		}
   922  		e.Println()
   923  	} else {
   924  		// by default allow to override the image name
   925  		return nil
   926  	}
   927  
   928  	e.Println("Please check runner's configuration: http://doc.gitlab.com/ci/docker/using_docker_images.html#overwrite-image-and-services")
   929  	return errors.New("invalid image")
   930  }
   931  
   932  func (e *executor) expandImageName(imageName string, allowedInternalImages []string) (string, error) {
   933  	if imageName != "" {
   934  		image := e.Build.GetAllVariables().ExpandValue(imageName)
   935  		allowedInternalImages = append(allowedInternalImages, e.Config.Docker.Image)
   936  		err := e.verifyAllowedImage(image, "images", e.Config.Docker.AllowedImages, allowedInternalImages)
   937  		if err != nil {
   938  			return "", err
   939  		}
   940  		return image, nil
   941  	}
   942  
   943  	if e.Config.Docker.Image == "" {
   944  		return "", errors.New("No Docker image specified to run the build in")
   945  	}
   946  
   947  	return e.Config.Docker.Image, nil
   948  }
   949  
   950  func (e *executor) overwriteEntrypoint(image *common.Image) []string {
   951  	if len(image.Entrypoint) > 0 {
   952  		if !e.Config.Docker.DisableEntrypointOverwrite {
   953  			return image.Entrypoint
   954  		}
   955  
   956  		e.Warningln("Entrypoint override disabled")
   957  	}
   958  
   959  	return nil
   960  }
   961  
   962  func (e *executor) connectDocker() error {
   963  	client, err := docker_helpers.New(e.Config.Docker.DockerCredentials, "")
   964  	if err != nil {
   965  		return err
   966  	}
   967  	e.client = client
   968  
   969  	e.info, err = client.Info(e.Context)
   970  	if err != nil {
   971  		return err
   972  	}
   973  
   974  	err = e.validateOSType()
   975  	if err != nil {
   976  		return err
   977  	}
   978  
   979  	e.helperImageInfo, err = helperimage.Get(common.REVISION, helperimage.Config{
   980  		OSType:          e.info.OSType,
   981  		Architecture:    e.info.Architecture,
   982  		OperatingSystem: e.info.OperatingSystem,
   983  	})
   984  
   985  	return err
   986  }
   987  
   988  // validateOSType checks if the ExecutorOptions metadata matches with the docker
   989  // info response.
   990  func (e *executor) validateOSType() error {
   991  	executorOSType := e.ExecutorOptions.Metadata[metadataOSType]
   992  	if executorOSType == "" {
   993  		return common.MakeBuildError("%s does not have any OSType specified", e.Config.Executor)
   994  	}
   995  
   996  	if executorOSType != e.info.OSType {
   997  		return common.MakeBuildError(
   998  			"executor requires OSType=%s, but Docker Engine supports only OSType=%s",
   999  			executorOSType, e.info.OSType,
  1000  		)
  1001  	}
  1002  
  1003  	return nil
  1004  }
  1005  
  1006  func (e *executor) createDependencies() error {
  1007  	createDependenciesStrategy := []func() error{
  1008  		e.bindDevices,
  1009  		e.createVolumesManager,
  1010  		e.createVolumes,
  1011  		e.createBuildVolume,
  1012  		e.createServices,
  1013  	}
  1014  
  1015  	if e.Build.IsFeatureFlagOn(featureflags.UseLegacyVolumesMountingOrder) {
  1016  		// TODO: Remove in 12.6
  1017  		createDependenciesStrategy = []func() error{
  1018  			e.bindDevices,
  1019  			e.createVolumesManager,
  1020  			e.createBuildVolume,
  1021  			e.createServices,
  1022  			e.createVolumes,
  1023  		}
  1024  	}
  1025  
  1026  	for _, setup := range createDependenciesStrategy {
  1027  		err := setup()
  1028  		if err != nil {
  1029  			return err
  1030  		}
  1031  	}
  1032  
  1033  	return nil
  1034  }
  1035  
  1036  func (e *executor) createVolumes() error {
  1037  	e.SetCurrentStage(DockerExecutorStageCreatingUserVolumes)
  1038  	e.Debugln("Creating user-defined volumes...")
  1039  
  1040  	if e.volumesManager == nil {
  1041  		return errVolumesManagerUndefined
  1042  	}
  1043  
  1044  	for _, volume := range e.Config.Docker.Volumes {
  1045  		err := e.volumesManager.Create(volume)
  1046  		if err == volumes.ErrCacheVolumesDisabled {
  1047  			e.Warningln(fmt.Sprintf(
  1048  				"Container based cache volumes creation is disabled. Will not create volume for %q",
  1049  				volume,
  1050  			))
  1051  			continue
  1052  		}
  1053  
  1054  		if err != nil {
  1055  			return err
  1056  		}
  1057  	}
  1058  
  1059  	return nil
  1060  }
  1061  
  1062  func (e *executor) createBuildVolume() error {
  1063  	e.SetCurrentStage(DockerExecutorStageCreatingBuildVolumes)
  1064  	e.Debugln("Creating build volume...")
  1065  
  1066  	if e.volumesManager == nil {
  1067  		return errVolumesManagerUndefined
  1068  	}
  1069  
  1070  	jobsDir := e.Build.RootDir
  1071  
  1072  	// TODO: Remove in 12.3
  1073  	if e.Build.IsFeatureFlagOn(featureflags.UseLegacyBuildsDirForDocker) {
  1074  		// Cache Git sources:
  1075  		// take path of the projects directory,
  1076  		// because we use `rm -rf` which could remove the mounted volume
  1077  		jobsDir = path.Dir(e.Build.FullProjectDir())
  1078  	}
  1079  
  1080  	var err error
  1081  
  1082  	if e.Build.GetGitStrategy() == common.GitFetch {
  1083  		err = e.volumesManager.Create(jobsDir)
  1084  		if err == nil {
  1085  			return nil
  1086  		}
  1087  
  1088  		if err == volumes.ErrCacheVolumesDisabled {
  1089  			err = e.volumesManager.CreateTemporary(jobsDir)
  1090  		}
  1091  	} else {
  1092  		err = e.volumesManager.CreateTemporary(jobsDir)
  1093  	}
  1094  
  1095  	if err != nil {
  1096  		if _, ok := err.(*volumes.ErrVolumeAlreadyDefined); !ok {
  1097  			return err
  1098  		}
  1099  	}
  1100  
  1101  	return nil
  1102  }
  1103  
  1104  func (e *executor) Prepare(options common.ExecutorPrepareOptions) error {
  1105  	e.SetCurrentStage(DockerExecutorStagePrepare)
  1106  
  1107  	if options.Config.Docker == nil {
  1108  		return errors.New("missing docker configuration")
  1109  	}
  1110  
  1111  	e.AbstractExecutor.PrepareConfiguration(options)
  1112  
  1113  	err := e.connectDocker()
  1114  	if err != nil {
  1115  		return err
  1116  	}
  1117  
  1118  	err = e.prepareBuildsDir(options)
  1119  	if err != nil {
  1120  		return err
  1121  	}
  1122  
  1123  	err = e.AbstractExecutor.PrepareBuildAndShell()
  1124  	if err != nil {
  1125  		return err
  1126  	}
  1127  
  1128  	if e.BuildShell.PassFile {
  1129  		return errors.New("docker doesn't support shells that require script file")
  1130  	}
  1131  
  1132  	imageName, err := e.expandImageName(e.Build.Image.Name, []string{})
  1133  	if err != nil {
  1134  		return err
  1135  	}
  1136  
  1137  	e.Println("Using Docker executor with image", imageName, "...")
  1138  
  1139  	err = e.createDependencies()
  1140  	if err != nil {
  1141  		return err
  1142  	}
  1143  	return nil
  1144  }
  1145  
  1146  func (e *executor) prepareBuildsDir(options common.ExecutorPrepareOptions) error {
  1147  	if e.volumeParser == nil {
  1148  		return common.MakeBuildError("missing volume parser")
  1149  	}
  1150  
  1151  	isHostMounted, err := volumes.IsHostMountedVolume(e.volumeParser, e.RootDir(), options.Config.Docker.Volumes...)
  1152  	if err != nil {
  1153  		return &common.BuildError{Inner: err}
  1154  	}
  1155  
  1156  	// We need to set proper value for e.SharedBuildsDir because
  1157  	// it's required to properly start the job, what is done inside of
  1158  	// e.AbstractExecutor.Prepare()
  1159  	// And a started job is required for Volumes Manager to work, so it's
  1160  	// done before the manager is even created.
  1161  	if isHostMounted {
  1162  		e.SharedBuildsDir = true
  1163  	}
  1164  
  1165  	return nil
  1166  }
  1167  
  1168  func (e *executor) Cleanup() {
  1169  	e.SetCurrentStage(DockerExecutorStageCleanup)
  1170  
  1171  	var wg sync.WaitGroup
  1172  
  1173  	ctx, cancel := context.WithTimeout(context.Background(), dockerCleanupTimeout)
  1174  	defer cancel()
  1175  
  1176  	remove := func(id string) {
  1177  		wg.Add(1)
  1178  		go func() {
  1179  			e.removeContainer(ctx, id)
  1180  			wg.Done()
  1181  		}()
  1182  	}
  1183  
  1184  	for _, temporaryID := range e.temporary {
  1185  		remove(temporaryID)
  1186  	}
  1187  
  1188  	if e.volumesManager != nil {
  1189  		<-e.volumesManager.Cleanup(ctx)
  1190  	}
  1191  
  1192  	wg.Wait()
  1193  
  1194  	if e.client != nil {
  1195  		e.client.Close()
  1196  	}
  1197  
  1198  	e.AbstractExecutor.Cleanup()
  1199  }
  1200  
  1201  type serviceHealthCheckError struct {
  1202  	Inner error
  1203  	Logs  string
  1204  }
  1205  
  1206  func (e *serviceHealthCheckError) Error() string {
  1207  	if e.Inner == nil {
  1208  		return "serviceHealthCheckError"
  1209  	}
  1210  
  1211  	return e.Inner.Error()
  1212  }
  1213  
  1214  func (e *executor) runServiceHealthCheckContainer(service *types.Container, timeout time.Duration) error {
  1215  	waitImage, err := e.getPrebuiltImage()
  1216  	if err != nil {
  1217  		return fmt.Errorf("getPrebuiltImage: %v", err)
  1218  	}
  1219  
  1220  	containerName := service.Names[0] + "-wait-for-service"
  1221  
  1222  	cmd := []string{"gitlab-runner-helper", "health-check"}
  1223  	// TODO: Remove in 12.0 to start using the command from `gitlab-runner-helper`
  1224  	if e.checkOutdatedHelperImage() {
  1225  		e.Debugln(featureflags.DockerHelperImageV2, "is not set, falling back to old command")
  1226  		cmd = []string{"gitlab-runner-service"}
  1227  	}
  1228  
  1229  	config := &container.Config{
  1230  		Cmd:    cmd,
  1231  		Image:  waitImage.ID,
  1232  		Labels: e.getLabels("wait", "wait="+service.ID),
  1233  	}
  1234  	hostConfig := &container.HostConfig{
  1235  		RestartPolicy: neverRestartPolicy,
  1236  		Links:         []string{service.Names[0] + ":service"},
  1237  		NetworkMode:   container.NetworkMode(e.Config.Docker.NetworkMode),
  1238  		LogConfig: container.LogConfig{
  1239  			Type: "json-file",
  1240  		},
  1241  	}
  1242  	e.Debugln("Waiting for service container", containerName, "to be up and running...")
  1243  	resp, err := e.client.ContainerCreate(e.Context, config, hostConfig, nil, containerName)
  1244  	if err != nil {
  1245  		return fmt.Errorf("ContainerCreate: %v", err)
  1246  	}
  1247  	defer e.removeContainer(e.Context, resp.ID)
  1248  	err = e.client.ContainerStart(e.Context, resp.ID, types.ContainerStartOptions{})
  1249  	if err != nil {
  1250  		return fmt.Errorf("ContainerStart: %v", err)
  1251  	}
  1252  
  1253  	waitResult := make(chan error, 1)
  1254  	go func() {
  1255  		waitResult <- e.waitForContainer(e.Context, resp.ID)
  1256  	}()
  1257  
  1258  	// these are warnings and they don't make the build fail
  1259  	select {
  1260  	case err := <-waitResult:
  1261  		if err == nil {
  1262  			return nil
  1263  		}
  1264  
  1265  		return &serviceHealthCheckError{
  1266  			Inner: err,
  1267  			Logs:  e.readContainerLogs(resp.ID),
  1268  		}
  1269  	case <-time.After(timeout):
  1270  		return &serviceHealthCheckError{
  1271  			Inner: fmt.Errorf("service %q timeout", containerName),
  1272  			Logs:  e.readContainerLogs(resp.ID),
  1273  		}
  1274  	}
  1275  }
  1276  
  1277  func (e *executor) waitForServiceContainer(service *types.Container, timeout time.Duration) error {
  1278  	err := e.runServiceHealthCheckContainer(service, timeout)
  1279  	if err == nil {
  1280  		return nil
  1281  	}
  1282  
  1283  	var buffer bytes.Buffer
  1284  	buffer.WriteString("\n")
  1285  	buffer.WriteString(helpers.ANSI_YELLOW + "*** WARNING:" + helpers.ANSI_RESET + " Service " + service.Names[0] + " probably didn't start properly.\n")
  1286  	buffer.WriteString("\n")
  1287  	buffer.WriteString("Health check error:\n")
  1288  	buffer.WriteString(strings.TrimSpace(err.Error()))
  1289  	buffer.WriteString("\n")
  1290  
  1291  	if healtCheckErr, ok := err.(*serviceHealthCheckError); ok {
  1292  		buffer.WriteString("\n")
  1293  		buffer.WriteString("Health check container logs:\n")
  1294  		buffer.WriteString(healtCheckErr.Logs)
  1295  		buffer.WriteString("\n")
  1296  	}
  1297  
  1298  	buffer.WriteString("\n")
  1299  	buffer.WriteString("Service container logs:\n")
  1300  	buffer.WriteString(e.readContainerLogs(service.ID))
  1301  	buffer.WriteString("\n")
  1302  
  1303  	buffer.WriteString("\n")
  1304  	buffer.WriteString(helpers.ANSI_YELLOW + "*********" + helpers.ANSI_RESET + "\n")
  1305  	buffer.WriteString("\n")
  1306  	io.Copy(e.Trace, &buffer)
  1307  	return err
  1308  }
  1309  
  1310  func (e *executor) readContainerLogs(containerID string) string {
  1311  	var containerBuffer bytes.Buffer
  1312  
  1313  	options := types.ContainerLogsOptions{
  1314  		ShowStdout: true,
  1315  		ShowStderr: true,
  1316  		Timestamps: true,
  1317  	}
  1318  
  1319  	hijacked, err := e.client.ContainerLogs(e.Context, containerID, options)
  1320  	if err != nil {
  1321  		return strings.TrimSpace(err.Error())
  1322  	}
  1323  	defer hijacked.Close()
  1324  
  1325  	stdcopy.StdCopy(&containerBuffer, &containerBuffer, hijacked)
  1326  	containerLog := containerBuffer.String()
  1327  	return strings.TrimSpace(containerLog)
  1328  }