github.com/kaisenlinux/docker.io@v0.0.0-20230510090727-ea55db55fac7/engine/Jenkinsfile (about)

     1  #!groovy
     2  pipeline {
     3      agent none
     4  
     5      options {
     6          buildDiscarder(logRotator(daysToKeepStr: '30'))
     7          timeout(time: 2, unit: 'HOURS')
     8          timestamps()
     9      }
    10      parameters {
    11          booleanParam(name: 'unit_validate', defaultValue: true, description: 'amd64 (x86_64) unit tests and vendor check')
    12          booleanParam(name: 'validate_force', defaultValue: false, description: 'force validation steps to be run, even if no changes were detected')
    13          booleanParam(name: 'amd64', defaultValue: true, description: 'amd64 (x86_64) Build/Test')
    14          booleanParam(name: 'rootless', defaultValue: true, description: 'amd64 (x86_64) Build/Test (Rootless mode)')
    15          booleanParam(name: 'cgroup2', defaultValue: true, description: 'amd64 (x86_64) Build/Test (cgroup v2)')
    16          booleanParam(name: 'arm64', defaultValue: true, description: 'ARM (arm64) Build/Test')
    17          booleanParam(name: 's390x', defaultValue: false, description: 'IBM Z (s390x) Build/Test')
    18          booleanParam(name: 'ppc64le', defaultValue: false, description: 'PowerPC (ppc64le) Build/Test')
    19          booleanParam(name: 'windowsRS5', defaultValue: true, description: 'Windows 2019 (RS5) Build/Test')
    20          booleanParam(name: 'dco', defaultValue: true, description: 'Run the DCO check')
    21      }
    22      environment {
    23          DOCKER_BUILDKIT     = '1'
    24          DOCKER_EXPERIMENTAL = '1'
    25          DOCKER_GRAPHDRIVER  = 'overlay2'
    26          APT_MIRROR          = 'cdn-fastly.deb.debian.org'
    27          CHECK_CONFIG_COMMIT = '78405559cfe5987174aa2cb6463b9b2c1b917255'
    28          TESTDEBUG           = '0'
    29          TIMEOUT             = '120m'
    30      }
    31      stages {
    32          stage('pr-hack') {
    33              when { changeRequest() }
    34              steps {
    35                  script {
    36                      echo "Workaround for PR auto-cancel feature. Borrowed from https://issues.jenkins-ci.org/browse/JENKINS-43353"
    37                      def buildNumber = env.BUILD_NUMBER as int
    38                      if (buildNumber > 1) milestone(buildNumber - 1)
    39                      milestone(buildNumber)
    40                  }
    41              }
    42          }
    43          stage('DCO-check') {
    44              when {
    45                  beforeAgent true
    46                  expression { params.dco }
    47              }
    48              agent { label 'amd64 && ubuntu-1804 && overlay2' }
    49              steps {
    50                  sh '''
    51                  docker run --rm \
    52                    -v "$WORKSPACE:/workspace" \
    53                    -e VALIDATE_REPO=${GIT_URL} \
    54                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
    55                    alpine sh -c 'apk add --no-cache -q bash git openssh-client && git config --system --add safe.directory /workspace && cd /workspace && hack/validate/dco'
    56                  '''
    57              }
    58          }
    59          stage('Build') {
    60              parallel {
    61                  stage('unit-validate') {
    62                      when {
    63                          beforeAgent true
    64                          expression { params.unit_validate }
    65                      }
    66                      agent { label 'amd64 && ubuntu-1804 && overlay2' }
    67                      environment {
    68                          // On master ("non-pull-request"), force running some validation checks (vendor, swagger),
    69                          // even if no files were changed. This allows catching problems caused by pull-requests
    70                          // that were merged out-of-sequence.
    71                          TEST_FORCE_VALIDATE = sh returnStdout: true, script: 'if [ "${BRANCH_NAME%%-*}" != "PR" ] || [ "${CHANGE_TARGET:-master}" != "master" ] || [ "${validate_force}" = "true" ]; then echo "1"; fi'
    72                      }
    73  
    74                      stages {
    75                          stage("Print info") {
    76                              steps {
    77                                  sh 'docker version'
    78                                  sh 'docker info'
    79                                  sh '''
    80                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
    81                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
    82                                  && bash ${WORKSPACE}/check-config.sh || true
    83                                  '''
    84                              }
    85                          }
    86                          stage("Build dev image") {
    87                              steps {
    88                                  sh 'docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .'
    89                              }
    90                          }
    91                          stage("Validate") {
    92                              steps {
    93                                  sh '''
    94                                  docker run --rm -t --privileged \
    95                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
    96                                    -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \
    97                                    --name docker-pr$BUILD_NUMBER \
    98                                    -e DOCKER_EXPERIMENTAL \
    99                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   100                                    -e DOCKER_GRAPHDRIVER \
   101                                    -e TEST_FORCE_VALIDATE \
   102                                    -e VALIDATE_REPO=${GIT_URL} \
   103                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   104                                    docker:${GIT_COMMIT} \
   105                                    hack/validate/default
   106                                  '''
   107                              }
   108                          }
   109                          stage("Docker-py") {
   110                              steps {
   111                                  sh '''
   112                                  docker run --rm -t --privileged \
   113                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   114                                    --name docker-pr$BUILD_NUMBER \
   115                                    -e DOCKER_EXPERIMENTAL \
   116                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   117                                    -e DOCKER_GRAPHDRIVER \
   118                                    -e VALIDATE_REPO=${GIT_URL} \
   119                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   120                                    docker:${GIT_COMMIT} \
   121                                    hack/make.sh \
   122                                      dynbinary-daemon \
   123                                      test-docker-py
   124                                  '''
   125                              }
   126                              post {
   127                                  always {
   128                                      junit testResults: 'bundles/test-docker-py/junit-report.xml', allowEmptyResults: true
   129  
   130                                      sh '''
   131                                      echo "Ensuring container killed."
   132                                      docker rm -vf docker-pr$BUILD_NUMBER || true
   133                                      '''
   134  
   135                                      sh '''
   136                                      echo 'Chowning /workspace to jenkins user'
   137                                      docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   138                                      '''
   139  
   140                                      catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   141                                          sh '''
   142                                          bundleName=docker-py
   143                                          echo "Creating ${bundleName}-bundles.tar.gz"
   144                                          tar -czf ${bundleName}-bundles.tar.gz bundles/test-docker-py/*.xml bundles/test-docker-py/*.log
   145                                          '''
   146  
   147                                          archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   148                                      }
   149                                  }
   150                              }
   151                          }
   152                          stage("Static") {
   153                              steps {
   154                                  sh '''
   155                                  docker run --rm -t --privileged \
   156                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   157                                    --name docker-pr$BUILD_NUMBER \
   158                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   159                                    -e DOCKER_GRAPHDRIVER \
   160                                    docker:${GIT_COMMIT} \
   161                                    hack/make.sh binary-daemon
   162                                  '''
   163                              }
   164                          }
   165                          stage("Cross") {
   166                              steps {
   167                                  sh '''
   168                                  docker run --rm -t --privileged \
   169                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   170                                    --name docker-pr$BUILD_NUMBER \
   171                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   172                                    -e DOCKER_GRAPHDRIVER \
   173                                    docker:${GIT_COMMIT} \
   174                                    hack/make.sh cross
   175                                  '''
   176                              }
   177                          }
   178                          // needs to be last stage that calls make.sh for the junit report to work
   179                          stage("Unit tests") {
   180                              steps {
   181                                  sh '''
   182                                  docker run --rm -t --privileged \
   183                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   184                                    --name docker-pr$BUILD_NUMBER \
   185                                    -e DOCKER_EXPERIMENTAL \
   186                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   187                                    -e DOCKER_GRAPHDRIVER \
   188                                    -e VALIDATE_REPO=${GIT_URL} \
   189                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   190                                    docker:${GIT_COMMIT} \
   191                                    hack/test/unit
   192                                  '''
   193                              }
   194                              post {
   195                                  always {
   196                                      junit testResults: 'bundles/junit-report.xml', allowEmptyResults: true
   197                                  }
   198                              }
   199                          }
   200                          stage("Validate vendor") {
   201                              steps {
   202                                  sh '''
   203                                  docker run --rm -t --privileged \
   204                                    -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \
   205                                    --name docker-pr$BUILD_NUMBER \
   206                                    -e DOCKER_EXPERIMENTAL \
   207                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   208                                    -e DOCKER_GRAPHDRIVER \
   209                                    -e TEST_FORCE_VALIDATE \
   210                                    -e VALIDATE_REPO=${GIT_URL} \
   211                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   212                                    docker:${GIT_COMMIT} \
   213                                    hack/validate/vendor
   214                                  '''
   215                              }
   216                          }
   217                      }
   218  
   219                      post {
   220                          always {
   221                              sh '''
   222                              echo 'Ensuring container killed.'
   223                              docker rm -vf docker-pr$BUILD_NUMBER || true
   224                              '''
   225  
   226                              sh '''
   227                              echo 'Chowning /workspace to jenkins user'
   228                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   229                              '''
   230  
   231                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   232                                  sh '''
   233                                  bundleName=unit
   234                                  echo "Creating ${bundleName}-bundles.tar.gz"
   235                                  tar -czvf ${bundleName}-bundles.tar.gz bundles/junit-report.xml bundles/go-test-report.json bundles/profile.out
   236                                  '''
   237  
   238                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   239                              }
   240                          }
   241                          cleanup {
   242                              sh 'make clean'
   243                              deleteDir()
   244                          }
   245                      }
   246                  }
   247                  stage('amd64') {
   248                      when {
   249                          beforeAgent true
   250                          expression { params.amd64 }
   251                      }
   252                      agent { label 'amd64 && ubuntu-1804 && overlay2' }
   253  
   254                      stages {
   255                          stage("Print info") {
   256                              steps {
   257                                  sh 'docker version'
   258                                  sh 'docker info'
   259                                  sh '''
   260                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   261                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   262                                  && bash ${WORKSPACE}/check-config.sh || true
   263                                  '''
   264                              }
   265                          }
   266                          stage("Build dev image") {
   267                              steps {
   268                                  sh '''
   269                                  # todo: include ip_vs in base image
   270                                  sudo modprobe ip_vs
   271  
   272                                  docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
   273                                  '''
   274                              }
   275                          }
   276                          stage("Run tests") {
   277                              steps {
   278                                  sh '''#!/bin/bash
   279                                  # bash is needed so 'jobs -p' works properly
   280                                  # it also accepts setting inline envvars for functions without explicitly exporting
   281                                  set -x
   282  
   283                                  run_tests() {
   284                                          [ -n "$TESTDEBUG" ] && rm= || rm=--rm;
   285                                          docker run $rm -t --privileged \
   286                                            -v "$WORKSPACE/bundles/${TEST_INTEGRATION_DEST}:/go/src/github.com/docker/docker/bundles" \
   287                                            -v "$WORKSPACE/bundles/dynbinary-daemon:/go/src/github.com/docker/docker/bundles/dynbinary-daemon" \
   288                                            -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \
   289                                            --name "$CONTAINER_NAME" \
   290                                            -e KEEPBUNDLE=1 \
   291                                            -e TESTDEBUG \
   292                                            -e TESTFLAGS \
   293                                            -e TEST_SKIP_INTEGRATION \
   294                                            -e TEST_SKIP_INTEGRATION_CLI \
   295                                            -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   296                                            -e DOCKER_GRAPHDRIVER \
   297                                            -e TIMEOUT \
   298                                            -e VALIDATE_REPO=${GIT_URL} \
   299                                            -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   300                                            docker:${GIT_COMMIT} \
   301                                            hack/make.sh \
   302                                              "$1" \
   303                                              test-integration
   304                                  }
   305  
   306                                  trap "exit" INT TERM
   307                                  trap 'pids=$(jobs -p); echo "Remaining pids to kill: [$pids]"; [ -z "$pids" ] || kill $pids' EXIT
   308  
   309                                  CONTAINER_NAME=docker-pr$BUILD_NUMBER
   310  
   311                                  docker run --rm -t --privileged \
   312                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   313                                    -v "$WORKSPACE/.git:/go/src/github.com/docker/docker/.git" \
   314                                    --name ${CONTAINER_NAME}-build \
   315                                    -e DOCKER_EXPERIMENTAL \
   316                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   317                                    -e DOCKER_GRAPHDRIVER \
   318                                    docker:${GIT_COMMIT} \
   319                                    hack/make.sh \
   320                                      dynbinary-daemon
   321  
   322                                  # flaky + integration
   323                                  TEST_INTEGRATION_DEST=1 CONTAINER_NAME=${CONTAINER_NAME}-1 TEST_SKIP_INTEGRATION_CLI=1 run_tests test-integration-flaky &
   324  
   325                                  # integration-cli first set
   326                                  TEST_INTEGRATION_DEST=2 CONTAINER_NAME=${CONTAINER_NAME}-2 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSuite|DockerNetworkSuite|DockerHubPullSuite|DockerRegistrySuite|DockerSchema1RegistrySuite|DockerRegistryAuthTokenSuite|DockerRegistryAuthHtpasswdSuite)/" run_tests &
   327  
   328                                  # integration-cli second set
   329                                  TEST_INTEGRATION_DEST=3 CONTAINER_NAME=${CONTAINER_NAME}-3 TEST_SKIP_INTEGRATION=1 TESTFLAGS="-test.run Test(DockerSwarmSuite|DockerDaemonSuite|DockerExternalVolumeSuite)/" run_tests &
   330  
   331                                  c=0
   332                                  for job in $(jobs -p); do
   333                                          wait ${job} || c=$?
   334                                  done
   335                                  exit $c
   336                                  '''
   337                              }
   338                              post {
   339                                  always {
   340                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   341                                  }
   342                              }
   343                          }
   344                      }
   345  
   346                      post {
   347                          always {
   348                              sh '''
   349                              echo "Ensuring container killed."
   350                              cids=$(docker ps -aq -f name=docker-pr${BUILD_NUMBER}-*)
   351                              [ -n "$cids" ] && docker rm -vf $cids || true
   352                              '''
   353  
   354                              sh '''
   355                              echo "Chowning /workspace to jenkins user"
   356                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   357                              '''
   358  
   359                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   360                                  sh '''
   361                                  bundleName=amd64
   362                                  echo "Creating ${bundleName}-bundles.tar.gz"
   363                                  # exclude overlay2 directories
   364                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   365                                  '''
   366  
   367                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   368                              }
   369                          }
   370                          cleanup {
   371                              sh 'make clean'
   372                              deleteDir()
   373                          }
   374                      }
   375                  }
   376                  stage('rootless') {
   377                      when {
   378                          beforeAgent true
   379                          expression { params.rootless }
   380                      }
   381                      agent { label 'amd64 && ubuntu-1804 && overlay2' }
   382                      stages {
   383                          stage("Print info") {
   384                              steps {
   385                                  sh 'docker version'
   386                                  sh 'docker info'
   387                                  sh '''
   388                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   389                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   390                                  && bash ${WORKSPACE}/check-config.sh || true
   391                                  '''
   392                              }
   393                          }
   394                          stage("Build dev image") {
   395                              steps {
   396                                  sh '''
   397                                  docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
   398                                  '''
   399                              }
   400                          }
   401                          stage("Integration tests") {
   402                              environment {
   403                                  DOCKER_ROOTLESS = '1'
   404                                  TEST_SKIP_INTEGRATION_CLI = '1'
   405                              }
   406                              steps {
   407                                  sh '''
   408                                  docker run --rm -t --privileged \
   409                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   410                                    --name docker-pr$BUILD_NUMBER \
   411                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   412                                    -e DOCKER_GRAPHDRIVER \
   413                                    -e DOCKER_EXPERIMENTAL \
   414                                    -e DOCKER_ROOTLESS \
   415                                    -e TEST_SKIP_INTEGRATION_CLI \
   416                                    -e TIMEOUT \
   417                                    -e VALIDATE_REPO=${GIT_URL} \
   418                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   419                                    docker:${GIT_COMMIT} \
   420                                    hack/make.sh \
   421                                      dynbinary \
   422                                      test-integration
   423                                  '''
   424                              }
   425                              post {
   426                                  always {
   427                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   428                                  }
   429                              }
   430                          }
   431                      }
   432  
   433                      post {
   434                          always {
   435                              sh '''
   436                              echo "Ensuring container killed."
   437                              docker rm -vf docker-pr$BUILD_NUMBER || true
   438                              '''
   439  
   440                              sh '''
   441                              echo "Chowning /workspace to jenkins user"
   442                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   443                              '''
   444  
   445                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   446                                  sh '''
   447                                  bundleName=amd64-rootless
   448                                  echo "Creating ${bundleName}-bundles.tar.gz"
   449                                  # exclude overlay2 directories
   450                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   451                                  '''
   452  
   453                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   454                              }
   455                          }
   456                          cleanup {
   457                              sh 'make clean'
   458                              deleteDir()
   459                          }
   460                      }
   461                  }
   462  
   463                  stage('cgroup2') {
   464                      when {
   465                          beforeAgent true
   466                          expression { params.cgroup2 }
   467                      }
   468                      agent { label 'amd64 && ubuntu-2004 && cgroup2' }
   469                      stages {
   470                          stage("Print info") {
   471                              steps {
   472                                  sh 'docker version'
   473                                  sh 'docker info'
   474                              }
   475                          }
   476                          stage("Build dev image") {
   477                              steps {
   478                                  sh '''
   479                                  docker build --force-rm --build-arg APT_MIRROR --build-arg SYSTEMD=true -t docker:${GIT_COMMIT} .
   480                                  '''
   481                              }
   482                          }
   483                          stage("Integration tests") {
   484                              environment {
   485                                  DOCKER_SYSTEMD = '1' // recommended cgroup driver for v2
   486                                  TEST_SKIP_INTEGRATION_CLI = '1' // CLI tests do not support v2
   487                              }
   488                              steps {
   489                                  sh '''
   490                                  docker run --rm -t --privileged \
   491                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   492                                    --name docker-pr$BUILD_NUMBER \
   493                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   494                                    -e DOCKER_GRAPHDRIVER \
   495                                    -e DOCKER_EXPERIMENTAL \
   496                                    -e DOCKER_SYSTEMD \
   497                                    -e TEST_SKIP_INTEGRATION_CLI \
   498                                    -e TIMEOUT \
   499                                    -e VALIDATE_REPO=${GIT_URL} \
   500                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   501                                    docker:${GIT_COMMIT} \
   502                                    hack/make.sh \
   503                                      dynbinary \
   504                                      test-integration
   505                                  '''
   506                              }
   507                              post {
   508                                  always {
   509                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   510                                  }
   511                              }
   512                          }
   513                      }
   514  
   515                      post {
   516                          always {
   517                              sh '''
   518                              echo "Ensuring container killed."
   519                              docker rm -vf docker-pr$BUILD_NUMBER || true
   520                              '''
   521  
   522                              sh '''
   523                              echo "Chowning /workspace to jenkins user"
   524                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   525                              '''
   526  
   527                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   528                                  sh '''
   529                                  bundleName=amd64-cgroup2
   530                                  echo "Creating ${bundleName}-bundles.tar.gz"
   531                                  # exclude overlay2 directories
   532                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   533                                  '''
   534  
   535                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   536                              }
   537                          }
   538                          cleanup {
   539                              sh 'make clean'
   540                              deleteDir()
   541                          }
   542                      }
   543                  }
   544  
   545  
   546                  stage('s390x') {
   547                      when {
   548                          beforeAgent true
   549                          // Skip this stage on PRs unless the checkbox is selected
   550                          anyOf {
   551                              not { changeRequest() }
   552                              expression { params.s390x }
   553                          }
   554                      }
   555                      agent { label 's390x-ubuntu-2004' }
   556  
   557                      stages {
   558                          stage("Print info") {
   559                              steps {
   560                                  sh 'docker version'
   561                                  sh 'docker info'
   562                                  sh '''
   563                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   564                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   565                                  && bash ${WORKSPACE}/check-config.sh || true
   566                                  '''
   567                              }
   568                          }
   569                          stage("Build dev image") {
   570                              steps {
   571                                  sh '''
   572                                  docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
   573                                  '''
   574                              }
   575                          }
   576                          stage("Unit tests") {
   577                              steps {
   578                                  sh '''
   579                                  docker run --rm -t --privileged \
   580                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   581                                    --name docker-pr$BUILD_NUMBER \
   582                                    -e DOCKER_EXPERIMENTAL \
   583                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   584                                    -e DOCKER_GRAPHDRIVER \
   585                                    -e VALIDATE_REPO=${GIT_URL} \
   586                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   587                                    docker:${GIT_COMMIT} \
   588                                    hack/test/unit
   589                                  '''
   590                              }
   591                              post {
   592                                  always {
   593                                      junit testResults: 'bundles/junit-report.xml', allowEmptyResults: true
   594                                  }
   595                              }
   596                          }
   597                          stage("Integration tests") {
   598                              environment { TEST_SKIP_INTEGRATION_CLI = '1' }
   599                              steps {
   600                                  sh '''
   601                                  docker run --rm -t --privileged \
   602                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   603                                    --name docker-pr$BUILD_NUMBER \
   604                                    -e DOCKER_EXPERIMENTAL \
   605                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   606                                    -e DOCKER_GRAPHDRIVER \
   607                                    -e TESTDEBUG \
   608                                    -e TEST_SKIP_INTEGRATION_CLI \
   609                                    -e TIMEOUT \
   610                                    -e VALIDATE_REPO=${GIT_URL} \
   611                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   612                                    docker:${GIT_COMMIT} \
   613                                    hack/make.sh \
   614                                      dynbinary \
   615                                      test-integration
   616                                  '''
   617                              }
   618                              post {
   619                                  always {
   620                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   621                                  }
   622                              }
   623                          }
   624                      }
   625  
   626                      post {
   627                          always {
   628                              sh '''
   629                              echo "Ensuring container killed."
   630                              docker rm -vf docker-pr$BUILD_NUMBER || true
   631                              '''
   632  
   633                              sh '''
   634                              echo "Chowning /workspace to jenkins user"
   635                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   636                              '''
   637  
   638                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   639                                  sh '''
   640                                  bundleName=s390x-integration
   641                                  echo "Creating ${bundleName}-bundles.tar.gz"
   642                                  # exclude overlay2 directories
   643                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   644                                  '''
   645  
   646                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   647                              }
   648                          }
   649                          cleanup {
   650                              sh 'make clean'
   651                              deleteDir()
   652                          }
   653                      }
   654                  }
   655                  stage('s390x integration-cli') {
   656                      when {
   657                          beforeAgent true
   658                          not { changeRequest() }
   659                          expression { params.s390x }
   660                      }
   661                      agent { label 's390x-ubuntu-2004' }
   662  
   663                      stages {
   664                          stage("Print info") {
   665                              steps {
   666                                  sh 'docker version'
   667                                  sh 'docker info'
   668                                  sh '''
   669                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   670                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   671                                  && bash ${WORKSPACE}/check-config.sh || true
   672                                  '''
   673                              }
   674                          }
   675                          stage("Build dev image") {
   676                              steps {
   677                                  sh '''
   678                                  docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
   679                                  '''
   680                              }
   681                          }
   682                          stage("Integration-cli tests") {
   683                              environment { TEST_SKIP_INTEGRATION = '1' }
   684                              steps {
   685                                  sh '''
   686                                  docker run --rm -t --privileged \
   687                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   688                                    --name docker-pr$BUILD_NUMBER \
   689                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   690                                    -e DOCKER_GRAPHDRIVER \
   691                                    -e TEST_SKIP_INTEGRATION \
   692                                    -e TIMEOUT \
   693                                    -e VALIDATE_REPO=${GIT_URL} \
   694                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   695                                    docker:${GIT_COMMIT} \
   696                                    hack/make.sh \
   697                                      dynbinary \
   698                                      test-integration
   699                                  '''
   700                              }
   701                              post {
   702                                  always {
   703                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   704                                  }
   705                              }
   706                          }
   707                      }
   708  
   709                      post {
   710                          always {
   711                              sh '''
   712                              echo "Ensuring container killed."
   713                              docker rm -vf docker-pr$BUILD_NUMBER || true
   714                              '''
   715  
   716                              sh '''
   717                              echo "Chowning /workspace to jenkins user"
   718                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   719                              '''
   720  
   721                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   722                                  sh '''
   723                                  bundleName=s390x-integration-cli
   724                                  echo "Creating ${bundleName}-bundles.tar.gz"
   725                                  # exclude overlay2 directories
   726                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   727                                  '''
   728  
   729                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   730                              }
   731                          }
   732                          cleanup {
   733                              sh 'make clean'
   734                              deleteDir()
   735                          }
   736                      }
   737                  }
   738                  stage('ppc64le') {
   739                      when {
   740                          beforeAgent true
   741                          // Skip this stage on PRs unless the checkbox is selected
   742                          anyOf {
   743                              not { changeRequest() }
   744                              expression { params.ppc64le }
   745                          }
   746                      }
   747                      agent { label 'ppc64le-ubuntu-1604' }
   748                      // ppc64le machines run on Docker 18.06, and buildkit has some
   749                      // bugs on that version. Build and use buildx instead.
   750                      environment {
   751                          USE_BUILDX      = '1'
   752                          DOCKER_BUILDKIT = '0'
   753                      }
   754  
   755                      stages {
   756                          stage("Print info") {
   757                              steps {
   758                                  sh 'docker version'
   759                                  sh 'docker info'
   760                                  sh '''
   761                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   762                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   763                                  && bash ${WORKSPACE}/check-config.sh || true
   764                                  '''
   765                              }
   766                          }
   767                          stage("Build dev image") {
   768                              steps {
   769                                  sh '''
   770                                  make bundles/buildx
   771                                  bundles/buildx build --load --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
   772                                  '''
   773                              }
   774                          }
   775                          stage("Unit tests") {
   776                              steps {
   777                                  sh '''
   778                                  docker run --rm -t --privileged \
   779                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   780                                    --name docker-pr$BUILD_NUMBER \
   781                                    -e DOCKER_EXPERIMENTAL \
   782                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   783                                    -e DOCKER_GRAPHDRIVER \
   784                                    -e VALIDATE_REPO=${GIT_URL} \
   785                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   786                                    docker:${GIT_COMMIT} \
   787                                    hack/test/unit
   788                                  '''
   789                              }
   790                              post {
   791                                  always {
   792                                      junit testResults: 'bundles/junit-report.xml', allowEmptyResults: true
   793                                  }
   794                              }
   795                          }
   796                          stage("Integration tests") {
   797                              environment { TEST_SKIP_INTEGRATION_CLI = '1' }
   798                              steps {
   799                                  sh '''
   800                                  docker run --rm -t --privileged \
   801                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   802                                    --name docker-pr$BUILD_NUMBER \
   803                                    -e DOCKER_EXPERIMENTAL \
   804                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   805                                    -e DOCKER_GRAPHDRIVER \
   806                                    -e TESTDEBUG \
   807                                    -e TEST_SKIP_INTEGRATION_CLI \
   808                                    -e TIMEOUT \
   809                                    -e VALIDATE_REPO=${GIT_URL} \
   810                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   811                                    docker:${GIT_COMMIT} \
   812                                    hack/make.sh \
   813                                      dynbinary \
   814                                      test-integration
   815                                  '''
   816                              }
   817                              post {
   818                                  always {
   819                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   820                                  }
   821                              }
   822                          }
   823                      }
   824  
   825                      post {
   826                          always {
   827                              sh '''
   828                              echo "Ensuring container killed."
   829                              docker rm -vf docker-pr$BUILD_NUMBER || true
   830                              '''
   831  
   832                              sh '''
   833                              echo "Chowning /workspace to jenkins user"
   834                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   835                              '''
   836  
   837                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   838                                  sh '''
   839                                  bundleName=ppc64le-integration
   840                                  echo "Creating ${bundleName}-bundles.tar.gz"
   841                                  # exclude overlay2 directories
   842                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   843                                  '''
   844  
   845                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   846                              }
   847                          }
   848                          cleanup {
   849                              sh 'make clean'
   850                              deleteDir()
   851                          }
   852                      }
   853                  }
   854                  stage('ppc64le integration-cli') {
   855                      when {
   856                          beforeAgent true
   857                          not { changeRequest() }
   858                          expression { params.ppc64le }
   859                      }
   860                      agent { label 'ppc64le-ubuntu-1604' }
   861                      // ppc64le machines run on Docker 18.06, and buildkit has some
   862                      // bugs on that version. Build and use buildx instead.
   863                      environment {
   864                          USE_BUILDX      = '1'
   865                          DOCKER_BUILDKIT = '0'
   866                      }
   867  
   868                      stages {
   869                          stage("Print info") {
   870                              steps {
   871                                  sh 'docker version'
   872                                  sh 'docker info'
   873                                  sh '''
   874                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   875                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   876                                  && bash ${WORKSPACE}/check-config.sh || true
   877                                  '''
   878                              }
   879                          }
   880                          stage("Build dev image") {
   881                              steps {
   882                                  sh '''
   883                                  make bundles/buildx
   884                                  bundles/buildx build --load --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .
   885                                  '''
   886                              }
   887                          }
   888                          stage("Integration-cli tests") {
   889                              environment { TEST_SKIP_INTEGRATION = '1' }
   890                              steps {
   891                                  sh '''
   892                                  docker run --rm -t --privileged \
   893                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   894                                    --name docker-pr$BUILD_NUMBER \
   895                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   896                                    -e DOCKER_GRAPHDRIVER \
   897                                    -e TEST_SKIP_INTEGRATION \
   898                                    -e TIMEOUT \
   899                                    -e VALIDATE_REPO=${GIT_URL} \
   900                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   901                                    docker:${GIT_COMMIT} \
   902                                    hack/make.sh \
   903                                      dynbinary \
   904                                      test-integration
   905                                  '''
   906                              }
   907                              post {
   908                                  always {
   909                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
   910                                  }
   911                              }
   912                          }
   913                      }
   914  
   915                      post {
   916                          always {
   917                              sh '''
   918                              echo "Ensuring container killed."
   919                              docker rm -vf docker-pr$BUILD_NUMBER || true
   920                              '''
   921  
   922                              sh '''
   923                              echo "Chowning /workspace to jenkins user"
   924                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
   925                              '''
   926  
   927                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
   928                                  sh '''
   929                                  bundleName=ppc64le-integration-cli
   930                                  echo "Creating ${bundleName}-bundles.tar.gz"
   931                                  # exclude overlay2 directories
   932                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
   933                                  '''
   934  
   935                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
   936                              }
   937                          }
   938                          cleanup {
   939                              sh 'make clean'
   940                              deleteDir()
   941                          }
   942                      }
   943                  }
   944                  stage('arm64') {
   945                      when {
   946                          beforeAgent true
   947                          expression { params.arm64 }
   948                      }
   949                      agent { label 'arm64 && ubuntu-2004' }
   950                      environment {
   951                          TEST_SKIP_INTEGRATION_CLI = '1'
   952                      }
   953  
   954                      stages {
   955                          stage("Print info") {
   956                              steps {
   957                                  sh 'docker version'
   958                                  sh 'docker info'
   959                                  sh '''
   960                                  echo "check-config.sh version: ${CHECK_CONFIG_COMMIT}"
   961                                  curl -fsSL -o ${WORKSPACE}/check-config.sh "https://raw.githubusercontent.com/moby/moby/${CHECK_CONFIG_COMMIT}/contrib/check-config.sh" \
   962                                  && bash ${WORKSPACE}/check-config.sh || true
   963                                  '''
   964                              }
   965                          }
   966                          stage("Build dev image") {
   967                              steps {
   968                                  sh 'docker build --force-rm --build-arg APT_MIRROR -t docker:${GIT_COMMIT} .'
   969                              }
   970                          }
   971                          stage("Unit tests") {
   972                              steps {
   973                                  sh '''
   974                                  docker run --rm -t --privileged \
   975                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   976                                    --name docker-pr$BUILD_NUMBER \
   977                                    -e DOCKER_EXPERIMENTAL \
   978                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
   979                                    -e DOCKER_GRAPHDRIVER \
   980                                    -e VALIDATE_REPO=${GIT_URL} \
   981                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
   982                                    docker:${GIT_COMMIT} \
   983                                    hack/test/unit
   984                                  '''
   985                              }
   986                              post {
   987                                  always {
   988                                      junit testResults: 'bundles/junit-report.xml', allowEmptyResults: true
   989                                  }
   990                              }
   991                          }
   992                          stage("Integration tests") {
   993                              environment { TEST_SKIP_INTEGRATION_CLI = '1' }
   994                              steps {
   995                                  sh '''
   996                                  docker run --rm -t --privileged \
   997                                    -v "$WORKSPACE/bundles:/go/src/github.com/docker/docker/bundles" \
   998                                    --name docker-pr$BUILD_NUMBER \
   999                                    -e DOCKER_EXPERIMENTAL \
  1000                                    -e DOCKER_GITCOMMIT=${GIT_COMMIT} \
  1001                                    -e DOCKER_GRAPHDRIVER \
  1002                                    -e TESTDEBUG \
  1003                                    -e TEST_SKIP_INTEGRATION_CLI \
  1004                                    -e TIMEOUT \
  1005                                    -e VALIDATE_REPO=${GIT_URL} \
  1006                                    -e VALIDATE_BRANCH=${CHANGE_TARGET} \
  1007                                    docker:${GIT_COMMIT} \
  1008                                    hack/make.sh \
  1009                                      dynbinary \
  1010                                      test-integration
  1011                                  '''
  1012                              }
  1013                              post {
  1014                                  always {
  1015                                      junit testResults: 'bundles/**/*-report.xml', allowEmptyResults: true
  1016                                  }
  1017                              }
  1018                          }
  1019                      }
  1020  
  1021                      post {
  1022                          always {
  1023                              sh '''
  1024                              echo "Ensuring container killed."
  1025                              docker rm -vf docker-pr$BUILD_NUMBER || true
  1026                              '''
  1027  
  1028                              sh '''
  1029                              echo "Chowning /workspace to jenkins user"
  1030                              docker run --rm -v "$WORKSPACE:/workspace" busybox chown -R "$(id -u):$(id -g)" /workspace
  1031                              '''
  1032  
  1033                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
  1034                                  sh '''
  1035                                  bundleName=arm64-integration
  1036                                  echo "Creating ${bundleName}-bundles.tar.gz"
  1037                                  # exclude overlay2 directories
  1038                                  find bundles -path '*/root/*overlay2' -prune -o -type f \\( -name '*-report.json' -o -name '*.log' -o -name '*.prof' -o -name '*-report.xml' \\) -print | xargs tar -czf ${bundleName}-bundles.tar.gz
  1039                                  '''
  1040  
  1041                                  archiveArtifacts artifacts: '*-bundles.tar.gz', allowEmptyArchive: true
  1042                              }
  1043                          }
  1044                          cleanup {
  1045                              sh 'make clean'
  1046                              deleteDir()
  1047                          }
  1048                      }
  1049                  }
  1050                  stage('win-RS5') {
  1051                      when {
  1052                          beforeAgent true
  1053                          expression { params.windowsRS5 }
  1054                      }
  1055                      environment {
  1056                          DOCKER_BUILDKIT        = '0'
  1057                          DOCKER_DUT_DEBUG       = '1'
  1058                          SKIP_VALIDATION_TESTS  = '1'
  1059                          SOURCES_DRIVE          = 'd'
  1060                          SOURCES_SUBDIR         = 'gopath'
  1061                          TESTRUN_DRIVE          = 'd'
  1062                          TESTRUN_SUBDIR         = "CI"
  1063                          WINDOWS_BASE_IMAGE     = 'mcr.microsoft.com/windows/servercore'
  1064                          WINDOWS_BASE_IMAGE_TAG = 'ltsc2019'
  1065                      }
  1066                      agent {
  1067                          node {
  1068                              customWorkspace 'd:\\gopath\\src\\github.com\\docker\\docker'
  1069                              label 'windows-2019'
  1070                          }
  1071                      }
  1072                      stages {
  1073                          stage("Print info") {
  1074                              steps {
  1075                                  sh 'docker version'
  1076                                  sh 'docker info'
  1077                              }
  1078                          }
  1079                          stage("Run tests") {
  1080                              steps {
  1081                                  powershell '''
  1082                                  $ErrorActionPreference = 'Stop'
  1083                                  Invoke-WebRequest https://github.com/moby/docker-ci-zap/blob/master/docker-ci-zap.exe?raw=true -OutFile C:/Windows/System32/docker-ci-zap.exe
  1084                                  ./hack/ci/windows.ps1
  1085                                  exit $LastExitCode
  1086                                  '''
  1087                              }
  1088                          }
  1089                      }
  1090                      post {
  1091                          always {
  1092                              junit testResults: 'bundles/junit-report-*.xml', allowEmptyResults: true
  1093                              catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', message: 'Failed to create bundles.tar.gz') {
  1094                                  powershell '''
  1095                                  cd $env:WORKSPACE
  1096                                  $bundleName="windowsRS5-integration"
  1097                                  Write-Host -ForegroundColor Green "Creating ${bundleName}-bundles.zip"
  1098  
  1099                                  # archiveArtifacts does not support env-vars to , so save the artifacts in a fixed location
  1100                                  Compress-Archive -Path "bundles/CIDUT.out", "bundles/CIDUT.err", "bundles/junit-report-*.xml" -CompressionLevel Optimal -DestinationPath "${bundleName}-bundles.zip"
  1101                                  '''
  1102  
  1103                                  archiveArtifacts artifacts: '*-bundles.zip', allowEmptyArchive: true
  1104                              }
  1105                          }
  1106                          cleanup {
  1107                              sh 'make clean'
  1108                              deleteDir()
  1109                          }
  1110                      }
  1111                  }
  1112              }
  1113          }
  1114      }
  1115  }