k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/config/jobs/kubernetes-csi/gen-jobs.sh (about)

     1  #! /bin/bash -e
     2  # Copyright 2019 The Kubernetes Authors.
     3  #
     4  # Licensed under the Apache License, Version 2.0 (the "License");
     5  # you may not use this file except in compliance with the License.
     6  # You may obtain a copy of the License at
     7  #
     8  #     http://www.apache.org/licenses/LICENSE-2.0
     9  #
    10  # Unless required by applicable law or agreed to in writing, software
    11  # distributed under the License is distributed on an "AS IS" BASIS,
    12  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  # The presubmit jobs for the different Kubernetes-CSI repos are all
    17  # the same except for the repo name. As Prow has no way of specifying
    18  # the same job for multiple repos and manually copy-and-paste would be
    19  # tedious, this script is used instead to generate them.
    20  
    21  base="$(dirname $0)"
    22  
    23  # All the Kubernetes versions we're testing. The patch version is
    24  # irrelevant because the prow.sh script will pick a suitable KinD
    25  # image or build from source.
    26  k8s_versions="
    27  1.27
    28  1.28
    29  1.29
    30  "
    31  
    32  # All the deployment versions we're testing.
    33  deployment_versions="
    34  1.27
    35  1.28
    36  1.29
    37  "
    38  
    39  # The experimental version for which jobs are optional.
    40  experimental_k8s_version="1.29"
    41  
    42  # The latest stable Kubernetes version for testing alpha jobs
    43  latest_stable_k8s_version="1.28"
    44  
    45  # Tag of the hostpath driver we should use for sidecar pull jobs
    46  hostpath_driver_version="v1.12.1"
    47  
    48  # We need this image because it has Docker in Docker and go.
    49  dind_image="gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240515-17c6d50e24-master"
    50  
    51  # All kubernetes-csi repos which are part of the hostpath driver example.
    52  # For these repos we generate the full test matrix. For each entry here
    53  # we need a "sig-storage-<repo>" dashboard in
    54  # config/testgrids/kubernetes/sig-storage/config.yaml.
    55  hostpath_example_repos="
    56  csi-driver-host-path
    57  external-attacher
    58  external-provisioner
    59  external-resizer
    60  external-snapshotter
    61  livenessprobe
    62  node-driver-registrar
    63  "
    64  
    65  # All kubernetes-csi repos for which want to define pull tests for
    66  # the csi-release-tools repo. Ideally, this list should represent
    67  # different ways of using csi-release-tools (for example, single image
    68  # vs. multiple images per repo). csi-sanity tests are used by csi-driver-host-path.
    69  csi_release_tools_repos="
    70  csi-test
    71  external-provisioner
    72  external-snapshotter
    73  csi-driver-host-path
    74  "
    75  
    76  # kubernetes-csi repos which only need to be tested against at most a
    77  # single Kubernetes version. We generate unit, stable and alpha jobs
    78  # for these, without specifying a Kubernetes version. What the repo
    79  # then tests in those jobs is entirely up to the repo.
    80  #
    81  # This list is currently empty, but such a job might be useful again
    82  # in the future, so the code generator code below is kept.
    83  single_kubernetes_repos="
    84  "
    85  
    86  # kubernetes-csi repos which only need unit testing.
    87  unit_testing_repos="
    88  external-health-monitor
    89  csi-test
    90  csi-release-tools
    91  csi-lib-utils
    92  csi-driver-iscsi
    93  csi-driver-nvmf
    94  csi-driver-nfs
    95  csi-proxy
    96  lib-volume-populator
    97  volume-data-source-validator
    98  "
    99  
   100  # No Prow support in them yet.
   101  # csi-driver-image-populator
   102  # csi-lib-fc
   103  # csi-lib-iscsi
   104  
   105  # All branches that do *not* support Prow testing. All new branches
   106  # are expected to have that support, therefore these list should be
   107  # fixed. By excluding old branches we can avoid Prow config
   108  # changes each time a new branch gets created.
   109  skip_branches_cluster_driver_registrar='^(release-1.0)$'
   110  skip_branches_csi_lib_utils='^(release-0.1|release-0.2)$'
   111  skip_branches_csi_test='^(release-0.3|release-1.0|v0.1.0|v0.2.0)$'
   112  skip_branches_external_attacher='^(release-0.2.0|release-0.3.0|release-0.4|release-1.0|v0.1.0)$'
   113  skip_branches_external_provisioner='^(release-0.2.0|release-0.3.0|release-0.4|release-1.0|v0.1.0)$'
   114  skip_branches_external_snapshotter='^(k8s_1.12.0-beta.1|release-0.4|release-1.0)$'
   115  skip_branches_livenessprobe='^(release-0.4|release-1.0)$'
   116  skip_branches_node_driver_registrar='^(release-1.0)$'
   117  
   118  skip_branches () {
   119      eval echo \\\"\$skip_branches_$(echo $1 | tr - _)\\\" | grep -v '""'
   120  }
   121  
   122  find "$base" -name '*.yaml' -exec grep -q 'generated by gen-jobs.sh' '{}' \; -delete
   123  
   124  # Resource usage of a job depends on whether it needs to build Kubernetes or not.
   125  resources_for_kubernetes () {
   126      local kubernetes="$1"
   127      local indentation="$2"
   128      cat <<EOF | sed -e "s/^/${indentation}/"
   129  resources:
   130    requests:
   131      memory: "9Gi"
   132      cpu: 4
   133    limits:
   134      memory: "9Gi"
   135      cpu: 4
   136  EOF
   137  }
   138  
   139  job_cluster() {
   140    local repo=$1
   141  
   142    case "$repo" in
   143     # Add any jobs that should be excluded from the community clusters here
   144     "")
   145       echo "default"
   146       ;;
   147     *)
   148       echo "eks-prow-build-cluster"
   149       ;;
   150    esac
   151  
   152  }
   153  
   154  # Combines deployment and Kubernetes version in a job suffix like "1-14-on-kubernetes-1-13".
   155  kubernetes_job_name () {
   156      local deployment="$1"
   157      local kubernetes="$2"
   158      echo "$(echo "$deployment-on-kubernetes-$kubernetes" | tr . -)"
   159  }
   160  
   161  # Combines type ("ci" or "pull"), repo, test type ("unit", "alpha", "non-alpha") and deployment+kubernetes into a
   162  # Prow job name of the format <type>-kubernetes-csi[-<repo>][-<test type>][-<kubernetes job name].
   163  # The <test type> part is only added for "unit" and "non-alpha" because there is no good name for it ("stable"?!)
   164  # and to keep the job name a bit shorter.
   165  job_name () {
   166      local type="$1"
   167      local repo="$2"
   168      local tests="$3"
   169      local deployment="$4"
   170      local kubernetes="$5"
   171      local name
   172  
   173      name="$type-kubernetes-csi"
   174      if [ "$repo" ]; then
   175          name+="-$repo"
   176      fi
   177      name+=$(test_name "$tests" "$deployment" "$kubernetes")
   178      echo "$name"
   179  }
   180  
   181  # Generates the testgrid annotations. "ci" jobs all land in the same
   182  # "sig-storage-csi-ci" and send alert emails, "pull" jobs land in "sig-storage-csi-<repo>"
   183  # and don't alert. Some repos only have a single pull job. Those
   184  # land in "sig-storage-csi-other".
   185  annotations () {
   186      local indent="$1"
   187      shift
   188      local type="$1"
   189      local repo="$2"
   190      local tests="$3"
   191      local deployment="$4"
   192      local kubernetes="$5"
   193      local description
   194  
   195      echo "annotations:"
   196      case "$type" in
   197          ci)
   198              echo "${indent}testgrid-dashboards: sig-storage-csi-ci"
   199              local alpha_testgrid_prefix="$(if [ "$tests" = "alpha" ]; then echo alpha-; fi)"
   200              echo "${indent}testgrid-tab-name: ${alpha_testgrid_prefix}${deployment}-on-${kubernetes}"
   201              echo "${indent}testgrid-alert-email: kubernetes-sig-storage-test-failures@googlegroups.com"
   202              description="periodic Kubernetes-CSI job"
   203              ;;
   204          pull)
   205              local testgrid
   206              local name=$(test_name "$tests" "$deployment" "$kubernetes" | sed -e 's/^-//')
   207              if [ "$name" ]; then
   208                  testgrid="sig-storage-csi-$repo"
   209              else
   210                  testgrid="sig-storage-csi-other"
   211                  name=$(job_name "$@")
   212              fi
   213              echo "${indent}testgrid-dashboards: $testgrid"
   214              echo "${indent}testgrid-tab-name: $name"
   215              description="Kubernetes-CSI pull job"
   216              ;;
   217      esac
   218  
   219      if [ "$repo" ]; then
   220          description+=" in repo $repo"
   221      fi
   222      if [ "$tests" ]; then
   223          description+=" for $tests tests"
   224      fi
   225      if [ "$deployment" ] || [ "$kubernetes" ]; then
   226          description+=", using deployment $deployment on Kubernetes $kubernetes"
   227      fi
   228      echo "${indent}description: $description"
   229  }
   230  
   231  # Common suffix for job names which contains informatiopn about the test and cluster.
   232  # Empty or starts with a hyphen.
   233  test_name() {
   234      local tests="$1"
   235      local deployment="$2"
   236      local kubernetes="$3"
   237      local name
   238  
   239      if [ "$tests" ] && [ "$tests" != "non-alpha" ]; then
   240          name+="-$tests"
   241      fi
   242      if [ "$deployment" ] || [ "$kubernetes" ]; then
   243          name+="-$(kubernetes_job_name "$deployment" "$kubernetes")"
   244      fi
   245      echo "$name"
   246  }
   247  
   248  # "alpha" and "non-alpha" need to be expanded to different CSI_PROW_TESTS names.
   249  expand_tests () {
   250      case "$1" in
   251          non-alpha)
   252              echo "sanity serial parallel";;
   253          alpha)
   254              echo "serial-alpha parallel-alpha";;
   255          *)
   256              echo "$1";;
   257      esac
   258  }
   259  
   260  # "alpha" features can be breaking across releases and
   261  # therefore cannot be a required job
   262  pull_optional() {
   263      local tests="$1"
   264      local kubernetes="$2"
   265      local deployment_suffix="$3"
   266  
   267      # https://github.com/kubernetes-csi/csi-driver-host-path/pull/282 has not been merged yet,
   268      # therefore pull jobs which depend on the new deployment flavors have to be optional.
   269      # TODO: remove this check once merged.
   270      if [ "$tests" == "alpha" ] || [ "$deployment_suffix" ] ; then
   271          echo "true"
   272      elif [ "$kubernetes" == "$experimental_k8s_version" ]; then
   273          # New k8s versions may require updates to kind or release-tools.
   274          # Make tests optional until everything is updated.
   275          echo "true"
   276      else
   277          echo "false"
   278      fi
   279  }
   280  
   281  pull_alwaysrun() {
   282      if [ "$1" != "alpha" ]; then
   283          echo "true"
   284      else
   285          echo "false"
   286      fi
   287  }
   288  
   289  # version_gt returns true if arg1 is greater than arg2.
   290  #
   291  # This function expects versions to be one of the following formats:
   292  #   X.Y.Z, release-X.Y.Z, vX.Y.Z
   293  #
   294  #   where X,Y, and Z are any number.
   295  #
   296  # Partial versions (1.2, release-1.2) work as well as long as both
   297  # arguments use the same format.
   298  #
   299  # The follow substrings are stripped before version comparison:
   300  #   - "v"
   301  #   - "release-"
   302  #
   303  # Usage:
   304  # version_gt release-1.3 v1.2.0  (returns true)
   305  # version_gt v1.1.1 v1.2.0  (returns false)
   306  # version_gt 1.1.1 v1.2.0  (returns false)
   307  # version_gt 1.3.1 v1.2.0  (returns true)
   308  # version_gt 1.1.1 release-1.2.0  (returns false)
   309  # version_gt 1.2.0 1.2.2  (returns false)
   310  function version_gt() {
   311      versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo ${ver#v}; done)
   312      greaterVersion=${1#"release-"};
   313      greaterVersion=${greaterVersion#"kubernetes-"};
   314      greaterVersion=${greaterVersion#"v"};
   315      test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion"
   316  }
   317  
   318  
   319  
   320  
   321  snapshotter_version() {
   322      local kubernetes="$1"
   323      local canary="$2"
   324  
   325      if [ "$kubernetes" = "latest" ] || [ "$canary" = "canary" ]; then
   326          # Kubernetes master and canary images may need a more recent
   327          # snapshot controller and/or CRD than the ones from the latest
   328          # stable release.
   329          echo '"master"'
   330      else
   331          echo '"v6.1.0"'
   332      fi
   333  }
   334  
   335  additional_deployment_suffices () (
   336      local repo="$1"
   337  
   338      case "$repo" in
   339          csi-driver-host-path) echo "-test";;
   340      esac
   341  )
   342  
   343  for repo in $hostpath_example_repos; do
   344      mkdir -p "$base/$repo"
   345      cat >"$base/$repo/$repo-config.yaml" <<EOF
   346  # generated by gen-jobs.sh, do not edit manually
   347  
   348  presubmits:
   349    kubernetes-csi/$repo:
   350  EOF
   351  
   352      for deployment_suffix in "" $(additional_deployment_suffices "$repo"); do
   353          for tests in non-alpha alpha; do
   354              for deployment in $deployment_versions; do
   355                  for kubernetes in $k8s_versions; do
   356                      # We could generate these pre-submit jobs for all combinations, but to save resources in the Prow
   357                      # cluster we only do it for those cases where the deployment matches the Kubernetes version.
   358                      # Once we have more than two supported Kubernetes releases we should limit this to the most
   359                      # recent two.
   360                      #
   361                      # Periodic jobs need to test the full matrix.
   362                      if [ "$kubernetes" = "$deployment" ]; then
   363                          # Alpha jobs only run on the latest version
   364                          if [ "$tests" != "alpha" ] || [ "$kubernetes" = "$latest_stable_k8s_version" ]; then
   365                              # These required jobs test the binary built from the PR against
   366                              # older, stable hostpath driver deployments and Kubernetes versions
   367                              cat >>"$base/$repo/$repo-config.yaml" <<EOF
   368    - name: $(job_name "pull" "$repo" "$tests" "$deployment$deployment_suffix" "$kubernetes")
   369      cluster: $(job_cluster "$repo")
   370      always_run: $(pull_alwaysrun "$tests")
   371      optional: $(pull_optional "$tests" "$kubernetes" "$deployment_suffix")
   372      decorate: true
   373      skip_report: false
   374      skip_branches: [$(skip_branches $repo)]
   375      labels:
   376        preset-service-account: "true"
   377        preset-dind-enabled: "true"
   378        preset-kind-volume-mounts: "true"
   379      $(annotations "      " "pull" "$repo" "$tests" "$deployment$deployment_suffix" "$kubernetes")
   380      spec:
   381        containers:
   382        # We need this image because it has Docker in Docker and go.
   383        - image: ${dind_image}
   384          command:
   385          - runner.sh
   386          args:
   387          - ./.prow.sh
   388          env:
   389          # We pick some version for which there are pre-built images for kind.
   390          # Update only when the newer version is known to not cause issues,
   391          # otherwise presubmit jobs may start to fail for reasons that are
   392          # unrelated to the PR. Testing against the latest Kubernetes is covered
   393          # by periodic jobs (see https://testgrid.k8s.io/sig-storage-csi-ci#Summary).
   394          - name: CSI_PROW_KUBERNETES_VERSION
   395            value: "$kubernetes.0"
   396          - name: CSI_PROW_KUBERNETES_DEPLOYMENT
   397            value: "$deployment"
   398          - name: CSI_PROW_DEPLOYMENT_SUFFIX
   399            value: "$deployment_suffix"
   400          - name: CSI_PROW_DRIVER_VERSION
   401            value: "$hostpath_driver_version"
   402          - name: CSI_SNAPSHOTTER_VERSION
   403            value: $(snapshotter_version "$kubernetes" "")
   404          - name: CSI_PROW_TESTS
   405            value: "$(expand_tests "$tests")"
   406          # docker-in-docker needs privileged mode
   407          securityContext:
   408            privileged: true
   409  $(resources_for_kubernetes "$kubernetes" "        ")
   410  EOF
   411                          fi
   412                      fi
   413                  done # end kubernetes
   414  
   415  
   416                  # These optional jobs test the binary built from the PR against
   417                  # older, stable hostpath driver deployments and Kubernetes master
   418                  if [ "$tests" != "alpha" ] || [ "$deployment" = "$latest_stable_k8s" ]; then
   419                      cat >>"$base/$repo/$repo-config.yaml" <<EOF
   420    - name: $(job_name "pull" "$repo" "$tests" "$deployment$deployment_suffix" master)
   421      # Explicitly needs to be started with /test.
   422      # This cannot be enabled by default because there's always the risk
   423      # that something changes in master which breaks the pre-merge check.
   424      cluster: $(job_cluster "$repo")
   425      always_run: false
   426      optional: true
   427      decorate: true
   428      skip_report: false
   429      labels:
   430        preset-service-account: "true"
   431        preset-dind-enabled: "true"
   432        preset-kind-volume-mounts: "true"
   433      $(annotations "      " "pull" "$repo" "$tests" "$deployment$deployment_suffix" master)
   434      spec:
   435        containers:
   436        # We need this image because it has Docker in Docker and go.
   437        - image: ${dind_image}
   438          command:
   439          - runner.sh
   440          args:
   441          - ./.prow.sh
   442          env:
   443          - name: CSI_PROW_KUBERNETES_VERSION
   444            value: "latest"
   445          - name: CSI_PROW_DRIVER_VERSION
   446            value: "$hostpath_driver_version"
   447          - name: CSI_PROW_DEPLOYMENT_SUFFIX
   448            value: "$deployment_suffix"
   449          - name: CSI_SNAPSHOTTER_VERSION
   450            value: $(snapshotter_version "latest" "")
   451          - name: CSI_PROW_TESTS
   452            value: "$(expand_tests "$tests")"
   453          # docker-in-docker needs privileged mode
   454          securityContext:
   455            privileged: true
   456  $(resources_for_kubernetes master   "        ")
   457  EOF
   458                  fi
   459              done # end deployment
   460          done # end tests
   461      done # end deployment_suffix
   462  
   463      cat >>"$base/$repo/$repo-config.yaml" <<EOF
   464    - name: $(job_name "pull" "$repo" "unit")
   465      cluster: $(job_cluster "$repo")
   466      always_run: true
   467      decorate: true
   468      skip_report: false
   469      skip_branches: [$(skip_branches $repo)]
   470      labels:
   471        preset-service-account: "true"
   472        preset-dind-enabled: "true"
   473        preset-kind-volume-mounts: "true"
   474      $(annotations "      " "pull" "$repo" "unit")
   475      spec:
   476        containers:
   477        # We need this image because it has Docker in Docker and go.
   478        - image: ${dind_image}
   479          command:
   480          - runner.sh
   481          args:
   482          - ./.prow.sh
   483          env:
   484          - name: CSI_PROW_TESTS
   485            value: "unit"
   486          # docker-in-docker needs privileged mode
   487          securityContext:
   488            privileged: true
   489  $(resources_for_kubernetes master   "        ")
   490  EOF
   491  
   492      if [[ $repo != "csi-driver-host-path" ]]; then
   493        cat >>"$base/$repo/$repo-config.yaml" <<EOF
   494  
   495    - name: $(job_name "pull" "$repo" "canary")
   496      cluster: $(job_cluster "$repo")
   497      optional: true
   498      decorate: true
   499      skip_report: false
   500      skip_branches: [$(skip_branches $repo)]
   501      labels:
   502        preset-service-account: "true"
   503        preset-dind-enabled: "true"
   504        preset-kind-volume-mounts: "true"
   505      $(annotations "      " "pull" "$repo" "canary")
   506      spec:
   507        containers:
   508        # We need this image because it has Docker in Docker and go.
   509        - image: ${dind_image}
   510          command:
   511          - runner.sh
   512          args:
   513          - ./.prow.sh
   514          env:
   515          - name: CSI_PROW_KUBERNETES_VERSION
   516            value: "latest"
   517          - name: CSI_PROW_TESTS
   518            value: "$(expand_tests "$tests")"
   519          - name: CSI_PROW_BUILD_JOB
   520            value: "true"
   521          - name: CSI_PROW_HOSTPATH_CANARY
   522            value: "canary"
   523          - name: CSI_SNAPSHOTTER_VERSION
   524            value: "master"
   525          - name: CSI_PROW_DRIVER_VERSION
   526            value: "master"
   527          # ... but the RBAC rules only when testing on master.
   528          # The other jobs test against the unmodified deployment for
   529          # that Kubernetes version, i.e. with the original RBAC rules.
   530          - name: UPDATE_RBAC_RULES
   531            value: "true"
   532  $(    if [ "$repo" = "external-provisioner" ]; then
   533  cat <<EOF2
   534          - name: VOLUME_MODE_CONVERSION_TESTS
   535            value: "true"
   536  EOF2
   537        fi
   538  )
   539          # docker-in-docker needs privileged mode
   540          securityContext:
   541            privileged: true
   542  $(resources_for_kubernetes master   "        ")
   543  EOF
   544      fi
   545  done
   546  
   547  for repo in $single_kubernetes_repos; do
   548      mkdir -p "$base/$repo"
   549      cat >"$base/$repo/$repo-config.yaml" <<EOF
   550  # generated by gen-jobs.sh, do not edit manually
   551  
   552  presubmits:
   553    kubernetes-csi/$repo:
   554  EOF
   555      for tests in non-alpha unit alpha; do
   556          cat >>"$base/$repo/$repo-config.yaml" <<EOF
   557    - name: $(job_name "pull" "$repo" "$tests")
   558      cluster: $(job_cluster "$repo")
   559      always_run: true
   560      optional: $(pull_optional "$tests")
   561      decorate: true
   562      skip_report: false
   563      skip_branches: [$(skip_branches $repo)]
   564      labels:
   565        preset-service-account: "true"
   566        preset-dind-enabled: "true"
   567        preset-kind-volume-mounts: "true"
   568      $(annotations "      " "pull" "$repo" "$tests")
   569      spec:
   570        containers:
   571        # We need this image because it has Docker in Docker and go.
   572        - image: ${dind_image}
   573          command:
   574          - runner.sh
   575          args:
   576          - ./.prow.sh
   577          env:
   578          - name: CSI_PROW_TESTS
   579            value: "$(expand_tests "$tests")"
   580          # docker-in-docker needs privileged mode
   581          securityContext:
   582            privileged: true
   583  $(resources_for_kubernetes default   "        ")
   584  EOF
   585      done
   586  done
   587  
   588  # Single job for everything.
   589  for repo in $unit_testing_repos; do
   590      mkdir -p "$base/$repo"
   591      cat >"$base/$repo/$repo-config.yaml" <<EOF
   592  # generated by gen-jobs.sh, do not edit manually
   593  
   594  presubmits:
   595    kubernetes-csi/$repo:
   596  EOF
   597  
   598      cat >>"$base/$repo/$repo-config.yaml" <<EOF
   599    - name: pull-kubernetes-csi-$repo
   600      cluster: $(job_cluster "$repo")
   601      always_run: true
   602      decorate: true
   603      skip_report: false
   604      skip_branches: [$(skip_branches $repo)]
   605      labels:
   606        preset-service-account: "true"
   607        preset-dind-enabled: "true"
   608        preset-kind-volume-mounts: "true"
   609      $(annotations "      " "pull" "$repo")
   610      spec:
   611        containers:
   612        # We need this image because it has Docker in Docker and go.
   613        - image: ${dind_image}
   614          command:
   615          - runner.sh
   616          args:
   617          - ./.prow.sh
   618          # docker-in-docker needs privileged mode
   619          securityContext:
   620            privileged: true
   621  $(resources_for_kubernetes default   "        ")
   622  EOF
   623  done
   624  
   625  # The csi-driver-host-path repo contains different deployments. We
   626  # test those against different Kubernetes releases at regular
   627  # intervals. We do this for several reasons:
   628  # - Detect regressions in Kubernetes. This can happen because
   629  #   Kubernetes does not test against all of our deployments when
   630  #   preparing an update.
   631  # - Not all test configurations are covered by pre-submit jobs.
   632  # - The actual deployment content is not used verbatim in pre-submit
   633  #   jobs. The csi-driver-host-path image itself always gets replaced.
   634  #
   635  # This does E2E testing, with alpha tests only enabled in cases where
   636  # it makes sense. Unit tests are not enabled because we aren't building
   637  # the components.
   638  cat >>"$base/csi-driver-host-path/csi-driver-host-path-config.yaml" <<EOF
   639  
   640  periodics:
   641  EOF
   642  
   643  for deployment_suffix in "" "-test"; do
   644      for tests in non-alpha alpha; do
   645          for deployment in $deployment_versions; do
   646              for kubernetes in $deployment_versions master; do # these tests run against top of release-1.X instead of a specific release version
   647                  if [ "$tests" = "alpha" ]; then
   648                      # No version skew testing of alpha features, deployment has to match Kubernetes.
   649                      if ! echo "$kubernetes" | grep -q "^$deployment"; then
   650                          continue
   651                      fi
   652                      # Alpha testing is only done on the latest stable version or
   653                      # master
   654                      if [ "$kubernetes" != "$latest_stable_k8s_minor_version" ] && [ "$kubernetes" != "master" ]; then
   655                          continue
   656                      fi
   657                  fi
   658  
   659                  # Skip generating tests where the k8s version is lower than the deployment version
   660                  # because we do not support running newer deployments and sidecars on older kubernetes releases.
   661                  # The recommended Kubernetes version can be found in each kubernetes-csi sidecar release.
   662                  if [[ $kubernetes < $deployment ]]; then
   663                      continue
   664                  fi
   665                  actual="$(if [ "$kubernetes" = "master" ]; then echo latest; else echo "release-$kubernetes"; fi)"
   666                  cat >>"$base/csi-driver-host-path/csi-driver-host-path-config.yaml" <<EOF
   667  - interval: 6h
   668    name: $(job_name "ci" "" "$tests" "$deployment$deployment_suffix" "$kubernetes")
   669    cluster: k8s-infra-prow-build
   670    decorate: true
   671    extra_refs:
   672    - org: kubernetes-csi
   673      repo: csi-driver-host-path
   674      base_ref: master
   675    labels:
   676      preset-service-account: "true"
   677      preset-dind-enabled: "true"
   678      preset-kind-volume-mounts: "true"
   679    $(annotations "    " "ci" "" "$tests" "$deployment$deployment_suffix" "$kubernetes")
   680    spec:
   681      containers:
   682      # We need this image because it has Docker in Docker and go.
   683      - image: ${dind_image}
   684        command:
   685        - runner.sh
   686        args:
   687        - ./.prow.sh
   688        env:
   689        - name: CSI_PROW_KUBERNETES_VERSION
   690          value: "$actual"
   691        - name: CSI_SNAPSHOTTER_VERSION
   692          value: $(snapshotter_version "$actual" "")
   693        - name: CSI_PROW_BUILD_JOB
   694          value: "false"
   695        - name: CSI_PROW_DEPLOYMENT
   696          value: "kubernetes-$deployment"
   697        - name: CSI_PROW_DEPLOYMENT_SUFFIX
   698          value: "$deployment_suffix"
   699        - name: CSI_PROW_TESTS
   700          value: "$(expand_tests "$tests")"
   701        # docker-in-docker needs privileged mode
   702        securityContext:
   703          privileged: true
   704  $(resources_for_kubernetes "$actual" "      ")
   705  EOF
   706              done
   707          done
   708      done
   709  done
   710  
   711  # The canary builds use the latest sidecars from master and run them on
   712  # specific Kubernetes versions, using the default deployment for that Kubernetes
   713  # release.
   714  for deployment_suffix in "" "-test"; do
   715      for kubernetes in $k8s_versions master; do
   716          # master -> latest
   717          actual="${kubernetes/master/latest}"
   718          # 1.20 -> 1.20.0
   719          actual="$(echo "$actual" | sed -e 's/^\([0-9]*\)\.\([0-9]*\)$/\1.\2.0/')"
   720  
   721          for tests in non-alpha alpha; do
   722              # Alpha with latest sidecars only on master.
   723              if [ "$tests" = "alpha" ] && [ "$kubernetes" != "master" ]; then
   724                  continue
   725              fi
   726              alpha_testgrid_prefix="$(if [ "$tests" = "alpha" ]; then echo alpha-; fi)"
   727              cat >>"$base/csi-driver-host-path/csi-driver-host-path-config.yaml" <<EOF
   728  - interval: 6h
   729    name: $(job_name "ci" "" "$tests" "canary$deployment_suffix" "$kubernetes")
   730    cluster: $(job_cluster "$repo")
   731    decorate: true
   732    extra_refs:
   733    - org: kubernetes-csi
   734      repo: csi-driver-host-path
   735      base_ref: master
   736    labels:
   737      preset-service-account: "true"
   738      preset-dind-enabled: "true"
   739      preset-kind-volume-mounts: "true"
   740    $(annotations "    " "ci" "" "$tests" "canary$deployment_suffix" "$kubernetes")
   741    spec:
   742      containers:
   743      # We need this image because it has Docker in Docker and go.
   744      - image: ${dind_image}
   745        command:
   746        - runner.sh
   747        args:
   748        - ./.prow.sh
   749        env:
   750        - name: CSI_PROW_KUBERNETES_VERSION
   751          value: "$actual"
   752        - name: CSI_PROW_BUILD_JOB
   753          value: "false"
   754        # Replace images....
   755        - name: CSI_PROW_HOSTPATH_CANARY
   756          value: "canary"
   757        - name: CSI_PROW_DEPLOYMENT_SUFFIX
   758          value: "$deployment_suffix"
   759        - name: CSI_SNAPSHOTTER_VERSION
   760          value: $(snapshotter_version "$actual" "canary")
   761        # ... but the RBAC rules only when testing on master.
   762        # The other jobs test against the unmodified deployment for
   763        # that Kubernetes version, i.e. with the original RBAC rules.
   764        - name: UPDATE_RBAC_RULES
   765          value: "$([ "$kubernetes" = "master" ] && echo "true" || echo "false")"
   766        - name: CSI_PROW_TESTS
   767          value: "$(expand_tests "$tests")"
   768        # docker-in-docker needs privileged mode
   769        securityContext:
   770          privileged: true
   771  $(resources_for_kubernetes "$actual" "      ")
   772  EOF
   773          done
   774      done
   775  done
   776  
   777  for repo in $csi_release_tools_repos; do
   778      cat >>"$base/csi-release-tools/csi-release-tools-config.yaml" <<EOF
   779    - name: $(job_name "pull" "release-tools" "$repo" "" "")
   780      cluster: $(job_cluster "$repo")
   781      always_run: true
   782      optional: true # cannot be required because updates in csi-release-tools may include breaking changes
   783      decorate: true
   784      skip_report: false
   785      extra_refs:
   786      - org: kubernetes-csi
   787        repo: $repo
   788        base_ref: master
   789        workdir: false
   790        # Checked out in /home/prow/go/src/github.com/kubernetes-csi/$repo
   791      labels:
   792        preset-service-account: "true"
   793        preset-dind-enabled: "true"
   794        preset-kind-volume-mounts: "true"
   795      annotations:
   796        testgrid-dashboards: sig-storage-csi-other
   797        testgrid-tab-name: pull-csi-release-tools-in-$repo
   798        description: Kubernetes-CSI pull job in repo csi-release-tools for $repo, using deployment $latest_stable_k8s_version on Kubernetes $latest_stable_k8s_version
   799      spec:
   800        containers:
   801        # We need this image because it has Docker in Docker and go.
   802        - image: ${dind_image}
   803          command:
   804          - runner.sh
   805          args:
   806          - ./pull-test.sh # provided by csi-release-tools
   807          # docker-in-docker needs privileged mode
   808          securityContext:
   809            privileged: true
   810  $(resources_for_kubernetes "$latest_stable_k8s_version" "        ")
   811          env:
   812          - name: PULL_TEST_REPO_DIR
   813            value: /home/prow/go/src/github.com/kubernetes-csi/$repo
   814  EOF
   815  
   816      # The environment must mirror the corresponding pull jobs for those repos,
   817      # otherwise our pre-merge testing will not match what those jobs will do
   818      # after updating csi-release-tools.
   819      if [ "${repo}" != "csi-test" ]; then
   820          cat >>"$base/csi-release-tools/csi-release-tools-config.yaml" <<EOF
   821          - name: CSI_PROW_KUBERNETES_VERSION
   822            value: "$latest_stable_k8s_version.0"
   823          - name: CSI_PROW_KUBERNETES_DEPLOYMENT
   824            value: "$latest_stable_k8s_version"
   825          - name: CSI_PROW_DRIVER_VERSION
   826            value: "$hostpath_driver_version"
   827          - name: CSI_PROW_TESTS
   828            value: "unit sanity parallel"
   829          - name: CSI_SNAPSHOTTER_VERSION
   830            value: $(snapshotter_version "$latest_stable_k8s_version" "")
   831  EOF
   832      fi
   833  done