k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/config/jobs/kubernetes/kops/build_jobs.py (about) 1 # Copyright 2020 The Kubernetes Authors. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 # See the License for the specific language governing permissions and 13 # limitations under the License. 14 15 import hashlib 16 import math 17 import json 18 import re 19 import yaml 20 import jinja2 # pylint: disable=import-error 21 22 23 from helpers import ( # pylint: disable=import-error, no-name-in-module 24 build_cron, 25 create_args, 26 distro_images, 27 distros_ssh_user, 28 k8s_version_info, 29 should_skip_newer_k8s, 30 ) 31 32 # These are job tab names of unsupported grid combinations 33 skip_jobs = [ 34 ] 35 36 image = "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20240515-17c6d50e24-master" 37 38 loader = jinja2.FileSystemLoader(searchpath="./templates") 39 40 ############## 41 # Build Test # 42 ############## 43 44 # Returns a string representing the periodic prow job and the number of job invocations per week 45 def build_test(cloud='aws', 46 distro='u2204', 47 networking='cilium', 48 irsa=True, 49 k8s_version='ci', 50 kops_channel='alpha', 51 kops_version=None, 52 publish_version_marker=None, 53 name_override=None, 54 feature_flags=(), 55 extra_flags=None, 56 extra_dashboards=None, 57 terraform_version=None, 58 test_parallelism=25, 59 test_timeout_minutes=60, 60 test_args=None, 61 skip_regex='', 62 focus_regex=None, 63 runs_per_day=0, 64 scenario=None, 65 env=None, 66 kubernetes_feature_gates=None, 67 build_cluster="default", 68 cluster_name=None, 69 template_path=None, 70 storage_e2e_cred=False): 71 # pylint: disable=too-many-statements,too-many-branches,too-many-arguments 72 73 if kops_version is None: 74 # TODO: Move to kops-ci/markers/master/ once validated 75 kops_deploy_url = "https://storage.googleapis.com/kops-ci/bin/latest-ci-updown-green.txt" 76 elif kops_version.startswith("https://"): 77 kops_deploy_url = kops_version 78 kops_version = None 79 else: 80 kops_deploy_url = f"https://storage.googleapis.com/kops-ci/markers/release-{kops_version}/latest-ci-updown-green.txt" # pylint: disable=line-too-long 81 82 if should_skip_newer_k8s(k8s_version, kops_version): 83 return None 84 if networking == 'kopeio' and distro in ('flatcar', 'flatcararm64'): 85 return None 86 87 if extra_flags is None: 88 extra_flags = [] 89 90 if cloud == 'aws': 91 kops_image = distro_images[distro] 92 kops_ssh_user = distros_ssh_user[distro] 93 kops_ssh_key_path = '/etc/aws-ssh/aws-ssh-private' 94 95 elif cloud == 'gce': 96 kops_image = None 97 kops_ssh_user = 'prow' 98 kops_ssh_key_path = '/etc/ssh-key-secret/ssh-private' 99 100 validation_wait = None 101 if distro in ('flatcar', 'flatcararm64') or (distro in ('amzn2', 'rhel8') and kops_version in ('1.26', '1.27')): # pylint: disable=line-too-long 102 validation_wait = '20m' 103 104 suffix = "" 105 if cloud and cloud != "aws": 106 suffix += "-" + cloud 107 if networking: 108 suffix += "-" + networking 109 if distro: 110 suffix += "-" + distro 111 if k8s_version: 112 suffix += "-k" + k8s_version.replace("1.", "") 113 if kops_version: 114 suffix += "-ko" + kops_version.replace("1.", "") 115 116 tab = name_override or (f"kops-grid{suffix}") 117 job_name = f"e2e-{tab}" 118 119 if irsa and cloud == "aws" and scenario is None: 120 if extra_flags is None: 121 extra_flags = [] 122 extra_flags.append("--discovery-store=s3://k8s-kops-prow/discovery") 123 124 marker, k8s_deploy_url, test_package_url, test_package_dir = k8s_version_info(k8s_version) 125 args = create_args(kops_channel, networking, extra_flags, kops_image) 126 127 node_ig_overrides = "" 128 cp_ig_overrides = "" 129 # if distro == "flatcar": 130 # # https://github.com/flatcar-linux/Flatcar/issues/220 131 # node_ig_overrides += "spec.instanceMetadata.httpTokens=optional" 132 # cp_ig_overrides += "spec.instanceMetadata.httpTokens=optional" 133 134 if tab in skip_jobs: 135 return None 136 137 cron, runs_per_week = build_cron(tab, runs_per_day) 138 139 # Scenario-specific parameters 140 if env is None: 141 env = {} 142 143 tmpl_file = "periodic.yaml.jinja" 144 if scenario is not None: 145 tmpl_file = "periodic-scenario.yaml.jinja" 146 name_hash = hashlib.md5(job_name.encode()).hexdigest() 147 if build_cluster == "k8s-infra-kops-prow-build": 148 env['KOPS_STATE_STORE'] = "s3://k8s-kops-ci-prow-state-store" 149 env['KOPS_DNS_DOMAIN'] = "tests-kops-aws.k8s.io" 150 env['DISCOVERY_STORE'] = "s3://k8s-kops-ci-prow" 151 env['CLOUD_PROVIDER'] = cloud 152 if not cluster_name: 153 cluster_name = f"e2e-{name_hash[0:10]}-{name_hash[12:17]}.tests-kops-aws.k8s.io" 154 env['CLUSTER_NAME'] = cluster_name 155 env['KUBE_SSH_USER'] = kops_ssh_user 156 if 'KOPS_STATE_STORE' not in env and cloud == "aws": 157 env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow' 158 if extra_flags: 159 env['KOPS_EXTRA_FLAGS'] = " ".join(extra_flags) 160 if irsa and cloud == "aws": 161 env['KOPS_IRSA'] = "true" 162 163 tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file) 164 job = tmpl.render( 165 job_name=job_name, 166 cloud=cloud, 167 cron=cron, 168 kops_ssh_user=kops_ssh_user, 169 kops_ssh_key_path=kops_ssh_key_path, 170 create_args=args, 171 cp_ig_overrides=cp_ig_overrides, 172 node_ig_overrides=node_ig_overrides, 173 k8s_deploy_url=k8s_deploy_url, 174 kops_deploy_url=kops_deploy_url, 175 test_parallelism=str(test_parallelism), 176 job_timeout=str(test_timeout_minutes + 30) + 'm', 177 test_timeout=str(test_timeout_minutes) + 'm', 178 marker=marker, 179 template_path=template_path, 180 skip_regex=skip_regex, 181 kops_feature_flags=','.join(feature_flags), 182 terraform_version=terraform_version, 183 test_package_url=test_package_url, 184 test_package_dir=test_package_dir, 185 focus_regex=focus_regex, 186 publish_version_marker=publish_version_marker, 187 validation_wait=validation_wait, 188 image=image, 189 scenario=scenario, 190 env=env, 191 build_cluster=build_cluster, 192 kubernetes_feature_gates=kubernetes_feature_gates, 193 test_args=test_args, 194 cluster_name=cluster_name, 195 storage_e2e_cred=storage_e2e_cred, 196 ) 197 198 spec = { 199 'cloud': cloud, 200 'networking': networking, 201 'distro': distro, 202 'k8s_version': k8s_version, 203 'kops_version': f"{kops_version or 'latest'}", 204 'kops_channel': kops_channel, 205 } 206 if feature_flags: 207 spec['feature_flags'] = ','.join(feature_flags) 208 if extra_flags: 209 spec['extra_flags'] = ' '.join(extra_flags) 210 jsonspec = json.dumps(spec, sort_keys=True) 211 212 dashboards = [ 213 'sig-cluster-lifecycle-kops', 214 f"kops-distro-{distro.removesuffix('arm64')}", 215 f"kops-k8s-{k8s_version}", 216 f"kops-{kops_version or 'latest'}", 217 ] 218 if cloud == 'aws': 219 dashboards.extend(['google-aws']) 220 if cloud == 'gce': 221 dashboards.extend(['kops-gce']) 222 223 if extra_dashboards: 224 dashboards.extend(extra_dashboards) 225 226 days_of_results = 90 227 if runs_per_week * days_of_results > 2000: 228 # testgrid has a limit on number of test runs to show for a job 229 days_of_results = math.floor(2000 / runs_per_week) 230 annotations = { 231 'testgrid-dashboards': ', '.join(sorted(dashboards)), 232 'testgrid-days-of-results': str(days_of_results), 233 'testgrid-tab-name': tab, 234 } 235 for (k, v) in spec.items(): 236 annotations[f"test.kops.k8s.io/{k}"] = v or "" 237 238 extra = yaml.dump({'annotations': annotations}, width=9999, default_flow_style=False) 239 240 output = f"\n# {jsonspec}\n{job.strip()}\n" 241 for line in extra.splitlines(): 242 output += f" {line}\n" 243 return output, runs_per_week 244 245 # Returns a string representing a presubmit prow job YAML 246 def presubmit_test(branch='master', 247 cloud='aws', 248 distro='u2204', 249 networking='cilium', 250 irsa=True, 251 k8s_version='stable', 252 kops_channel='alpha', 253 name=None, 254 tab_name=None, 255 feature_flags=(), 256 extra_flags=None, 257 extra_dashboards=None, 258 terraform_version=None, 259 test_parallelism=25, 260 test_timeout_minutes=60, 261 test_args=None, 262 skip_regex='', 263 focus_regex=None, 264 run_if_changed=None, 265 optional=False, 266 skip_report=False, 267 always_run=False, 268 scenario=None, 269 artifacts=None, 270 env=None, 271 template_path=None, 272 use_boskos=False, 273 build_cluster="default", 274 cluster_name=None, 275 use_preset_for_account_creds=None): 276 # pylint: disable=too-many-statements,too-many-branches,too-many-arguments 277 if cloud == 'aws': 278 if distro == "channels": 279 kops_image = None 280 kops_ssh_user = 'ubuntu' 281 else: 282 kops_image = distro_images[distro] 283 kops_ssh_user = distros_ssh_user[distro] 284 kops_ssh_key_path = '/etc/aws-ssh/aws-ssh-private' 285 286 elif cloud == 'gce': 287 kops_image = None 288 kops_ssh_user = 'prow' 289 kops_ssh_key_path = '/etc/ssh-key-secret/ssh-private' 290 291 boskos_resource_type = None 292 if use_boskos: 293 if cloud == 'aws': 294 boskos_resource_type = 'aws-account' 295 else: 296 raise Exception(f"use_boskos not supported on cloud {cloud}") 297 298 if extra_flags is None: 299 extra_flags = [] 300 301 if irsa and cloud == "aws" and scenario is None: 302 extra_flags.append("--discovery-store=s3://k8s-kops-prow/discovery") 303 304 marker, k8s_deploy_url, test_package_url, test_package_dir = k8s_version_info(k8s_version) 305 args = create_args(kops_channel, networking, extra_flags, kops_image) 306 307 # Scenario-specific parameters 308 if env is None: 309 env = {} 310 311 tmpl_file = "presubmit.yaml.jinja" 312 if scenario is not None: 313 tmpl_file = "presubmit-scenario.yaml.jinja" 314 name_hash = hashlib.md5(name.encode()).hexdigest() 315 env['CLOUD_PROVIDER'] = cloud 316 if cloud == "aws": 317 env['CLUSTER_NAME'] = f"e2e-{name_hash[0:10]}-{name_hash[11:16]}.test-cncf-aws.k8s.io" 318 if 'KOPS_STATE_STORE' not in env and cloud == "aws": 319 env['KOPS_STATE_STORE'] = 's3://k8s-kops-prow' 320 if extra_flags: 321 env['KOPS_EXTRA_FLAGS'] = " ".join(extra_flags) 322 if irsa and cloud == "aws": 323 env['KOPS_IRSA'] = "true" 324 325 tmpl = jinja2.Environment(loader=loader).get_template(tmpl_file) 326 job = tmpl.render( 327 job_name=name, 328 branch=branch, 329 cloud=cloud, 330 kops_ssh_key_path=kops_ssh_key_path, 331 kops_ssh_user=kops_ssh_user, 332 create_args=args, 333 k8s_deploy_url=k8s_deploy_url, 334 test_parallelism=str(test_parallelism), 335 job_timeout=str(test_timeout_minutes + 30) + 'm', 336 test_timeout=str(test_timeout_minutes) + 'm', 337 marker=marker, 338 skip_regex=skip_regex, 339 kops_feature_flags=','.join(feature_flags), 340 terraform_version=terraform_version, 341 test_package_url=test_package_url, 342 test_package_dir=test_package_dir, 343 focus_regex=focus_regex, 344 run_if_changed=run_if_changed, 345 optional='true' if optional else 'false', 346 skip_report='true' if skip_report else 'false', 347 always_run='true' if always_run else 'false', 348 image=image, 349 scenario=scenario, 350 artifacts=artifacts, 351 env=env, 352 template_path=template_path, 353 boskos_resource_type=boskos_resource_type, 354 use_preset_for_account_creds=use_preset_for_account_creds, 355 build_cluster=build_cluster, 356 test_args=test_args, 357 cluster_name=cluster_name, 358 ) 359 360 spec = { 361 'cloud': cloud, 362 'networking': networking, 363 'distro': distro, 364 'k8s_version': k8s_version, 365 'kops_channel': kops_channel, 366 } 367 if feature_flags: 368 spec['feature_flags'] = ','.join(feature_flags) 369 if extra_flags: 370 spec['extra_flags'] = ' '.join(extra_flags) 371 jsonspec = json.dumps(spec, sort_keys=True) 372 373 dashboards = [ 374 'presubmits-kops', 375 'kops-presubmits', 376 'sig-cluster-lifecycle-kops', 377 ] 378 if extra_dashboards: 379 dashboards.extend(extra_dashboards) 380 381 annotations = { 382 'testgrid-dashboards': ', '.join(sorted(dashboards)), 383 'testgrid-days-of-results': '90', 384 'testgrid-tab-name': tab_name or name, 385 } 386 for (k, v) in spec.items(): 387 annotations[f"test.kops.k8s.io/{k}"] = v or "" 388 389 extra = yaml.dump({'annotations': annotations}, width=9999, default_flow_style=False) 390 391 output = f"\n# {jsonspec}{job}\n" 392 for line in extra.splitlines(): 393 output += f" {line}\n" 394 return output 395 396 #################### 397 # Grid Definitions # 398 #################### 399 400 networking_options = [ 401 'kubenet', 402 'calico', 403 'cilium', 404 'cilium-etcd', 405 'cilium-eni', 406 'flannel', 407 'kopeio', 408 ] 409 410 distro_options = [ 411 'al2023', 412 'deb12', 413 'flatcar', 414 'rhel8', 415 'u2004', 416 'u2204', 417 ] 418 419 k8s_versions = [ 420 "1.25", 421 "1.26", 422 "1.27", 423 "1.28", 424 "1.29", 425 ] 426 427 kops_versions = [ 428 None, # maps to latest 429 "1.28", 430 "1.29", 431 ] 432 433 434 ############################ 435 # kops-periodics-grid.yaml # 436 ############################ 437 def generate_grid(): 438 results = [] 439 # pylint: disable=too-many-nested-blocks 440 for networking in networking_options: 441 for distro in distro_options: 442 for k8s_version in k8s_versions: 443 for kops_version in kops_versions: 444 extra_flags = [] 445 if networking == 'cilium-eni': 446 # Cilium ENI bug fixed in newer kops versions but not backported to 1.27 447 if kops_version == '1.27': 448 continue 449 extra_flags = ['--node-size=t3.large'] 450 # remove flannel from list of tested CNIs after k8s version < 1.28 is not tested 451 if networking == 'flannel' and k8s_version in ['1.28', '1.29']: 452 continue 453 results.append( 454 build_test(cloud="aws", 455 build_cluster="k8s-infra-kops-prow-build", 456 distro=distro, 457 extra_dashboards=['kops-grid'], 458 k8s_version=k8s_version, 459 kops_version=kops_version, 460 networking=networking, 461 extra_flags=extra_flags, 462 irsa=False) 463 ) 464 465 # Manually expand grid coverage for GCP 466 # TODO(justinsb): merge into above block when we can 467 # pylint: disable=too-many-nested-blocks 468 for networking in ['kubenet', 'calico', 'cilium', 'gce']: # TODO: all networking_options: 469 for distro in ['u2004']: # TODO: all distro_options: 470 for k8s_version in k8s_versions: 471 for kops_version in [None]: # TODO: all kops_versions: 472 results.append( 473 build_test(cloud="gce", 474 runs_per_day=3, 475 distro=distro, 476 extra_dashboards=['kops-grid'], 477 k8s_version=k8s_version, 478 kops_version=kops_version, 479 networking=networking, 480 build_cluster="k8s-infra-prow-build", 481 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 # pylint: disable=line-too-long 482 ) 483 ) 484 485 return filter(None, results) 486 487 ############################# 488 # kops-periodics-misc2.yaml # 489 ############################# 490 def generate_misc(): 491 results = [ 492 # A one-off scenario testing the k8s.gcr.io mirror 493 build_test(name_override="kops-scenario-gcr-mirror", 494 runs_per_day=24, 495 cloud="aws", 496 build_cluster="k8s-infra-kops-prow-build", 497 # Latest runs with a staging AWS CCM, not available in registry.k8s.io 498 k8s_version='1.29', 499 extra_dashboards=['kops-misc']), 500 501 # A one-off scenario testing the artifacts-sandbox.k8s.io mirror 502 build_test(name_override="kops-artifacts-sandbox", 503 runs_per_day=3, 504 cloud="aws", 505 build_cluster="k8s-infra-kops-prow-build", 506 k8s_version='stable', 507 extra_dashboards=['kops-misc'], 508 scenario='smoketest', 509 env={ 510 'KOPS_BASE_URL': "https://artifacts-sandbox.k8s.io/binaries/kops/1.28.4/", # pylint: disable=line-too-long 511 'KOPS_VERSION': "v1.28.4", 512 'K8S_VERSION': "v1.28.6", 513 'KOPS_SKIP_E2E': '1', 514 'KOPS_CONTROL_PLANE_SIZE': '3', 515 }), 516 517 # Test Cilium against ci k8s test suite 518 build_test(name_override="kops-aws-cni-cilium-k8s-ci", 519 cloud="aws", 520 build_cluster="k8s-infra-kops-prow-build", 521 distro="u2204arm64", 522 k8s_version="ci", 523 networking="cilium", 524 runs_per_day=1, 525 extra_flags=["--zones=eu-central-1a", 526 "--node-size=m6g.large", 527 "--master-size=m6g.large"], 528 extra_dashboards=['kops-network-plugins']), 529 build_test(name_override="kops-gce-cni-cilium-k8s-ci", 530 cloud="gce", 531 k8s_version="ci", 532 networking="cilium", 533 build_cluster="k8s-infra-prow-build", 534 runs_per_day=1, 535 extra_flags=["--gce-service-account=default"], 536 extra_dashboards=['kops-network-plugins']), 537 538 # A special test for Calico CNI on Debian 11 539 build_test(name_override="kops-aws-cni-calico-deb11", 540 cloud="aws", 541 build_cluster="k8s-infra-kops-prow-build", 542 distro="deb11", 543 k8s_version="stable", 544 networking="calico", 545 runs_per_day=3, 546 extra_dashboards=['kops-network-plugins']), 547 # A special test for Calico CNI on Flatcar 548 build_test(name_override="kops-aws-cni-calico-flatcar", 549 cloud="aws", 550 build_cluster="k8s-infra-kops-prow-build", 551 distro="flatcararm64", 552 k8s_version="stable", 553 networking="calico", 554 runs_per_day=3, 555 extra_dashboards=['kops-distros', 'kops-network-plugins']), 556 557 # A special test for IPv6 using Calico CNI 558 build_test(name_override="kops-aws-cni-calico-ipv6", 559 cloud="aws", 560 build_cluster="k8s-infra-kops-prow-build", 561 distro="u2204arm64", 562 k8s_version="stable", 563 networking="calico", 564 runs_per_day=3, 565 extra_flags=['--ipv6', 566 '--topology=private', 567 '--dns=public', 568 '--bastion', 569 '--zones=us-west-2a', 570 ], 571 extra_dashboards=['kops-network-plugins', 'kops-ipv6']), 572 # A special test for IPv6 using Cilium CNI 573 build_test(name_override="kops-aws-cni-cilium-ipv6", 574 cloud="aws", 575 build_cluster="k8s-infra-kops-prow-build", 576 distro="u2204arm64", 577 k8s_version="stable", 578 networking="cilium", 579 runs_per_day=3, 580 extra_flags=['--ipv6', 581 '--topology=private', 582 '--dns=public', 583 '--bastion', 584 '--zones=us-west-2a', 585 ], 586 extra_dashboards=['kops-network-plugins', 'kops-ipv6']), 587 # A special test for IPv6 on Flatcar 588 build_test(name_override="kops-aws-ipv6-flatcar", 589 cloud="aws", 590 build_cluster="k8s-infra-kops-prow-build", 591 distro="flatcararm64", 592 k8s_version="stable", 593 runs_per_day=3, 594 extra_flags=['--ipv6', 595 '--topology=private', 596 '--dns=public', 597 '--bastion', 598 ], 599 extra_dashboards=['kops-distros', 'kops-ipv6']), 600 # A special test for IPv6 using Calico on Flatcar 601 build_test(name_override="kops-aws-cni-calico-ipv6-flatcar", 602 cloud="aws", 603 build_cluster="k8s-infra-kops-prow-build", 604 distro="flatcararm64", 605 k8s_version="stable", 606 networking="calico", 607 runs_per_day=3, 608 extra_flags=['--ipv6', 609 '--topology=private', 610 '--dns=public', 611 '--bastion', 612 ], 613 extra_dashboards=['kops-distros', 'kops-network-plugins', 'kops-ipv6']), 614 615 # A special test for disabling IRSA 616 build_test(name_override="kops-scenario-no-irsa", 617 cloud="aws", 618 build_cluster="k8s-infra-kops-prow-build", 619 distro="u2204arm64", 620 k8s_version="stable", 621 runs_per_day=3, 622 irsa=False, 623 extra_flags=['--api-loadbalancer-type=public', 624 ], 625 extra_dashboards=['kops-misc']), 626 627 # A special test for warm pool 628 build_test(name_override="kops-warm-pool", 629 cloud="aws", 630 build_cluster="k8s-infra-kops-prow-build", 631 distro="u2204arm64", 632 k8s_version="stable", 633 runs_per_day=3, 634 networking="cilium", 635 extra_flags=['--api-loadbalancer-type=public', 636 '--set=cluster.spec.cloudProvider.aws.warmPool.minSize=1' 637 ], 638 extra_dashboards=['kops-misc']), 639 640 # A special test for private topology 641 build_test(name_override="kops-aws-private", 642 cloud="aws", 643 build_cluster="k8s-infra-kops-prow-build", 644 distro="u2204arm64", 645 k8s_version="stable", 646 runs_per_day=3, 647 networking="calico", 648 extra_flags=['--topology=private', 649 '--bastion', 650 ], 651 extra_dashboards=['kops-misc']), 652 653 build_test(name_override="kops-scenario-terraform", 654 cloud="aws", 655 build_cluster="k8s-infra-kops-prow-build", 656 distro="u2204arm64", 657 k8s_version="stable", 658 runs_per_day=1, 659 terraform_version="1.5.5", 660 extra_flags=[ 661 "--zones=us-west-1a", 662 "--dns=public", 663 ], 664 extra_dashboards=['kops-misc']), 665 build_test(name_override="kops-scenario-ipv6-terraform", 666 cloud="aws", 667 build_cluster="k8s-infra-kops-prow-build", 668 distro="u2204arm64", 669 k8s_version="stable", 670 runs_per_day=1, 671 terraform_version="1.5.5", 672 extra_flags=[ 673 '--ipv6', 674 '--topology=private', 675 '--bastion', 676 "--zones=us-west-1a", 677 "--dns=public", 678 ], 679 extra_dashboards=['kops-misc', 'kops-ipv6']), 680 681 build_test(name_override="kops-aws-ha-euwest1", 682 cloud="aws", 683 build_cluster="k8s-infra-kops-prow-build", 684 distro="u2204arm64", 685 k8s_version="stable", 686 networking="calico", 687 kops_channel="alpha", 688 runs_per_day=3, 689 extra_flags=[ 690 "--master-count=3", 691 "--zones=eu-west-1a,eu-west-1b,eu-west-1c" 692 ], 693 extra_dashboards=["kops-misc"]), 694 695 build_test(name_override="kops-aws-arm64-release", 696 cloud="aws", 697 build_cluster="k8s-infra-kops-prow-build", 698 k8s_version="latest", 699 distro="u2204arm64", 700 networking="calico", 701 kops_channel="alpha", 702 runs_per_day=3, 703 extra_flags=["--zones=eu-central-1a", 704 "--node-size=m6g.large", 705 "--master-size=m6g.large"], 706 extra_dashboards=["kops-misc"]), 707 708 build_test(name_override="kops-aws-arm64-ci", 709 cloud="aws", 710 build_cluster="k8s-infra-kops-prow-build", 711 k8s_version="ci", 712 distro="u2204arm64", 713 networking="calico", 714 kops_channel="alpha", 715 runs_per_day=3, 716 extra_flags=["--zones=eu-central-1a", 717 "--node-size=m6g.large", 718 "--master-size=m6g.large"], 719 extra_dashboards=["kops-misc"]), 720 721 build_test(name_override="kops-aws-arm64-conformance", 722 cloud="aws", 723 build_cluster="k8s-infra-kops-prow-build", 724 k8s_version="ci", 725 distro="u2204arm64", 726 networking="calico", 727 kops_channel="alpha", 728 runs_per_day=3, 729 extra_flags=["--zones=eu-central-1a", 730 "--node-size=m6g.large", 731 "--master-size=m6g.large"], 732 skip_regex=r'\[Slow\]|\[Serial\]|\[Flaky\]', 733 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 734 extra_dashboards=["kops-misc"]), 735 736 build_test(name_override="kops-aws-amd64-conformance", 737 cloud="aws", 738 build_cluster="k8s-infra-kops-prow-build", 739 k8s_version="ci", 740 distro='u2204', 741 networking="calico", 742 kops_channel="alpha", 743 runs_per_day=3, 744 extra_flags=["--node-size=c5.large", 745 "--master-size=c5.large"], 746 skip_regex=r'\[Slow\]|\[Serial\]|\[Flaky\]', 747 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 748 extra_dashboards=["kops-misc"]), 749 750 build_test(name_override="kops-aws-updown", 751 build_cluster="default", 752 k8s_version="stable", 753 distro="u2204arm64", 754 networking="calico", 755 kops_channel="alpha", 756 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 757 publish_version_marker="gs://kops-ci/bin/latest-ci-updown-green.txt", 758 runs_per_day=24, 759 focus_regex=r'\[k8s.io\]\sNetworking.*\[Conformance\]', 760 extra_dashboards=["kops-misc"]), 761 762 build_test(name_override="kops-aws-aws-load-balancer-controller", 763 cloud="aws", 764 build_cluster="k8s-infra-kops-prow-build", 765 networking="cilium", 766 kops_channel="alpha", 767 k8s_version="stable", 768 runs_per_day=3, 769 scenario="aws-lb-controller", 770 extra_dashboards=['kops-misc']), 771 772 build_test(name_override="kops-aws-keypair-rotation-ha", 773 cloud="aws", 774 build_cluster="k8s-infra-kops-prow-build", 775 kops_channel="alpha", 776 k8s_version="stable", 777 runs_per_day=1, 778 test_timeout_minutes=240, 779 scenario="keypair-rotation", 780 env={'KOPS_CONTROL_PLANE_SIZE': '3'}, 781 extra_dashboards=['kops-misc']), 782 783 build_test(name_override="kops-aws-metrics-server", 784 cloud="aws", 785 build_cluster="k8s-infra-kops-prow-build", 786 networking="cilium", 787 kops_channel="alpha", 788 k8s_version="stable", 789 runs_per_day=3, 790 scenario="metrics-server", 791 extra_dashboards=['kops-misc']), 792 793 build_test(name_override="kops-aws-pod-identity-webhook", 794 cloud="aws", 795 build_cluster="k8s-infra-kops-prow-build", 796 networking="cilium", 797 kops_channel="alpha", 798 k8s_version="stable", 799 runs_per_day=3, 800 scenario="podidentitywebhook", 801 extra_dashboards=['kops-misc']), 802 803 build_test(name_override="kops-aws-addon-resource-tracking", 804 cloud="aws", 805 build_cluster="k8s-infra-kops-prow-build", 806 networking="cilium", 807 kops_channel="alpha", 808 k8s_version="stable", 809 runs_per_day=3, 810 scenario="addon-resource-tracking", 811 extra_dashboards=['kops-misc']), 812 813 build_test(name_override="kops-aws-external-dns", 814 cloud="aws", 815 build_cluster="k8s-infra-kops-prow-build", 816 distro="u2204arm64", 817 k8s_version="stable", 818 networking="cilium", 819 kops_channel="alpha", 820 runs_per_day=3, 821 extra_flags=[ 822 "--set=cluster.spec.externalDNS.provider=external-dns", 823 "--dns=public", 824 ], 825 extra_dashboards=['kops-misc']), 826 827 build_test(name_override="kops-aws-ipv6-external-dns", 828 cloud="aws", 829 build_cluster="k8s-infra-kops-prow-build", 830 distro="u2204arm64", 831 k8s_version="stable", 832 networking="cilium", 833 kops_channel="alpha", 834 runs_per_day=3, 835 extra_flags=[ 836 '--ipv6', 837 '--bastion', 838 "--set=cluster.spec.externalDNS.provider=external-dns", 839 "--dns=public", 840 ], 841 extra_dashboards=['kops-misc', 'kops-ipv6']), 842 843 build_test(name_override="kops-aws-apiserver-nodes", 844 cloud="aws", 845 build_cluster="k8s-infra-kops-prow-build", 846 distro="u2204arm64", 847 k8s_version="stable", 848 runs_per_day=3, 849 template_path="/home/prow/go/src/k8s.io/kops/tests/e2e/templates/apiserver.yaml.tmpl", # pylint: disable=line-too-long 850 extra_dashboards=['kops-misc'], 851 feature_flags=['APIServerNodes']), 852 853 build_test(name_override="kops-aws-karpenter", 854 cloud="aws", 855 build_cluster="k8s-infra-kops-prow-build", 856 distro="u2204arm64", 857 k8s_version="stable", 858 networking="cilium", 859 kops_channel="alpha", 860 runs_per_day=1, 861 extra_flags=[ 862 "--instance-manager=karpenter", 863 "--master-size=c6g.xlarge", 864 ], 865 extra_dashboards=["kops-misc"], 866 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 867 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|HostPort|two.untainted.nodes'), # pylint: disable=line-too-long 868 869 build_test(name_override="kops-aws-ipv6-karpenter", 870 cloud="aws", 871 build_cluster="k8s-infra-kops-prow-build", 872 distro="u2204arm64", 873 k8s_version="stable", 874 networking="cilium", 875 kops_channel="alpha", 876 runs_per_day=1, 877 extra_flags=[ 878 "--instance-manager=karpenter", 879 '--ipv6', 880 '--topology=private', 881 '--bastion', 882 "--master-size=c6g.xlarge", 883 ], 884 extra_dashboards=["kops-misc", "kops-ipv6"], 885 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 886 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|HostPort|two.untainted.nodes'), # pylint: disable=line-too-long 887 888 # A job to isolate a test failure reported in 889 # https://github.com/kubernetes/kubernetes/issues/123255 890 build_test(name_override="kops-aws-k28-hostname-bug123255", 891 cloud="aws", 892 build_cluster="k8s-infra-kops-prow-build", 893 distro="al2023", 894 k8s_version="1.28", 895 networking="cilium", 896 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|nfs|NFS|Gluster|NodeProblemDetector|fallback.to.local.terminating.endpoints.when.there.are.no.ready.endpoints.with.externalTrafficPolicy.Local|Services.*rejected.*endpoints|TCP.CLOSE_WAIT|external.IP.is.not.assigned.to.a.node|same.port.number.but.different.protocols|same.hostPort.but.different.hostIP.and.protocol|serve.endpoints.on.same.port.and.different.protocols|should.check.kube-proxy.urls|should.verify.that.all.nodes.have.volume.limits', # pylint: disable=line-too-long 897 runs_per_day=3, 898 extra_dashboards=['kops-misc']), 899 900 # [sig-storage, @jsafrane] Test SELinux features, because kops 901 # is the only way how to get Kubernetes on a Linux with SELinux in enforcing mode in CI. 902 # Test the latest kops and CI build of Kubernetes (=almost master). 903 build_test(name_override="kops-aws-selinux", 904 # RHEL8 VM image is enforcing SELinux by default. 905 cloud="aws", 906 build_cluster="k8s-infra-kops-prow-build", 907 distro="rhel8", 908 networking="cilium", 909 k8s_version="ci", 910 kops_channel="alpha", 911 feature_flags=['SELinuxMount'], 912 extra_flags=[ 913 "--set=cluster.spec.containerd.selinuxEnabled=true", 914 ], 915 focus_regex=r"\[Feature:SELinux\]", 916 # Skip: 917 # - Feature:Volumes: skips iSCSI and Ceph tests, they don't have client tools 918 # installed on nodes. 919 # - Driver: nfs: NFS does not have client tools installed on nodes. 920 # - Driver: local: this is optimization only, the volume plugin does not 921 # support SELinux and there are several subvariants of local volumes 922 # that multiply nr. of tests. 923 # - FeatureGate:SELinuxMount: the feature gate is alpha / disabled by default 924 # in v1.30. 925 skip_regex=r"\[Feature:Volumes\]|\[Driver:.nfs\]|\[Driver:.local\]|\[FeatureGate:SELinuxMount\]", # pylint: disable=line-too-long 926 # [Serial] and [Disruptive] are intentionally not skipped, therefore run 927 # everything as serial. 928 test_parallelism=1, 929 # Serial and Disruptive tests can be slow. 930 test_timeout_minutes=120, 931 runs_per_day=3), 932 933 # [sig-storage, @jsafrane] A one-off scenario testing SELinuxMount feature (alpha in v1.30). 934 # This will need to merge with kops-aws-selinux when SELinuxMount gets enabled by default. 935 build_test(name_override="kops-aws-selinux-alpha", 936 # RHEL8 VM image is enforcing SELinux by default. 937 cloud="aws", 938 build_cluster="k8s-infra-kops-prow-build", 939 distro="rhel8", 940 networking="cilium", 941 k8s_version="ci", 942 kops_channel="alpha", 943 feature_flags=['SELinuxMount'], 944 kubernetes_feature_gates="SELinuxMount", 945 extra_flags=[ 946 "--set=cluster.spec.containerd.selinuxEnabled=true", 947 ], 948 focus_regex=r"\[Feature:SELinux\]", 949 # Skip: 950 # - Feature:Volumes: skips iSCSI and Ceph tests, they don't have client tools 951 # installed on nodes. 952 # - Driver: nfs: NFS does not have client tools installed on nodes. 953 # - Driver: local: this is optimization only, the volume plugin does not 954 # support SELinux and there are several subvariants of local volumes 955 # that multiply nr. of tests. 956 # - Feature:SELinuxMountReadWriteOncePodOnly: these tests require SELinuxMount 957 # feature gate off. 958 skip_regex=r"\[Feature:Volumes\]|\[Driver:.nfs\]|\[Driver:.local\]|\[Feature:SELinuxMountReadWriteOncePodOnly\]", # pylint: disable=line-too-long 959 # [Serial] and [Disruptive] are intentionally not skipped, therefore run 960 # everything as serial. 961 test_parallelism=1, 962 # Serial and Disruptive tests can be slow. 963 test_timeout_minutes=120, 964 runs_per_day=3), 965 966 967 # test kube-up to kops jobs migration 968 build_test(name_override="ci-kubernetes-e2e-cos-gce-canary", 969 cloud="gce", 970 distro="cos105", 971 networking="kubenet", 972 k8s_version="ci", 973 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 974 kops_channel="alpha", 975 build_cluster="k8s-infra-prow-build", 976 extra_flags=[ 977 "--image=cos-cloud/cos-105-17412-156-49", 978 "--set=spec.nodeProblemDetector.enabled=true", 979 "--gce-service-account=default", 980 ], 981 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[KubeUp\]', # pylint: disable=line-too-long 982 test_timeout_minutes=60, 983 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 984 runs_per_day=8), 985 986 build_test(name_override="ci-kubernetes-e2e-al2023-aws-canary", 987 cloud="aws", 988 distro="al2023", 989 networking="kubenet", 990 k8s_version="ci", 991 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 992 kops_channel="alpha", 993 extra_flags=[ 994 "--set=spec.nodeProblemDetector.enabled=true", 995 "--set=spec.packages=nfs-utils", 996 ], 997 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 998 test_timeout_minutes=60, 999 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1000 runs_per_day=8), 1001 1002 build_test(name_override="ci-kubernetes-e2e-ubuntu-aws-canary", 1003 cloud="aws", 1004 distro="u2204", 1005 networking="kubenet", 1006 k8s_version="ci", 1007 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1008 kops_channel="alpha", 1009 extra_flags=[ 1010 "--set=spec.nodeProblemDetector.enabled=true", 1011 "--set=spec.packages=nfs-common", 1012 ], 1013 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 1014 test_timeout_minutes=60, 1015 test_args="--master-os-distro=ubuntu --node-os-distro=ubuntu", 1016 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1017 runs_per_day=8), 1018 1019 build_test(name_override="ci-kubernetes-e2e-cos-gce-slow-canary", 1020 cloud="gce", 1021 distro="cos105", 1022 networking="kubenet", 1023 k8s_version="ci", 1024 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1025 kops_channel="alpha", 1026 build_cluster="k8s-infra-prow-build", 1027 extra_flags=[ 1028 "--image=cos-cloud/cos-105-17412-156-49", 1029 "--set=spec.networking.networkID=default", 1030 "--gce-service-account=default", 1031 ], 1032 focus_regex=r'\[Slow\]', 1033 skip_regex=r'\[Driver:.gcepd\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]|\[KubeUp\]', # pylint: disable=line-too-long 1034 test_timeout_minutes=150, 1035 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1036 runs_per_day=6), 1037 1038 build_test(name_override="ci-kubernetes-e2e-al2023-aws-slow-canary", 1039 cloud="aws", 1040 distro="al2023", 1041 networking="kubenet", 1042 k8s_version="ci", 1043 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1044 kops_channel="alpha", 1045 build_cluster="k8s-infra-prow-build", 1046 extra_flags=[ 1047 "--set=spec.packages=nfs-utils", 1048 ], 1049 focus_regex=r'\[Slow\]', 1050 skip_regex=r'\[Driver:.gcepd\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 1051 test_timeout_minutes=150, 1052 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1053 runs_per_day=6), 1054 1055 build_test(name_override="ci-kubernetes-e2e-cos-gce-conformance-canary", 1056 cloud="gce", 1057 distro="cos105", 1058 networking="kubenet", 1059 k8s_version="ci", 1060 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1061 kops_channel="alpha", 1062 build_cluster="k8s-infra-prow-build", 1063 extra_flags=[ 1064 "--image=cos-cloud/cos-105-17412-156-49", 1065 "--set=spec.kubeAPIServer.logLevel=4", 1066 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1067 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1068 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1069 "--gce-service-account=default", 1070 ], 1071 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 1072 skip_regex=r'\[FOOBAR\]', # leaving it empty will allow kops to add extra skips 1073 test_timeout_minutes=200, 1074 test_parallelism=1, # serial tests 1075 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1076 runs_per_day=6), 1077 1078 build_test(name_override="ci-kubernetes-e2e-al2023-aws-conformance-canary", 1079 cloud="aws", 1080 distro="al2023", 1081 networking="kubenet", 1082 k8s_version="ci", 1083 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1084 kops_channel="alpha", 1085 build_cluster="eks-prow-build-cluster", 1086 extra_flags=[ 1087 "--set=spec.kubeAPIServer.logLevel=4", 1088 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1089 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1090 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1091 ], 1092 focus_regex=r'\[Conformance\]', 1093 skip_regex=r'\[FOOBAR\]', # leaving it empty will allow kops to add extra skips 1094 test_timeout_minutes=200, 1095 test_parallelism=1, # serial tests 1096 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1097 runs_per_day=6), 1098 1099 build_test(name_override="ci-kubernetes-e2e-al2023-aws-conformance-aws-cni", 1100 cloud="aws", 1101 distro="al2023", 1102 networking="amazonvpc", 1103 k8s_version="stable", 1104 kops_version="https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt", # pylint: disable=line-too-long 1105 cluster_name="kubernetes-e2e-al2023-aws-conformance-aws-cni.k8s.local", 1106 kops_channel="alpha", 1107 build_cluster="eks-prow-build-cluster", 1108 extra_flags=[ 1109 "--node-size=r5d.xlarge", 1110 "--master-size=r5d.xlarge", 1111 "--set=cluster.spec.networking.amazonVPC.env=ENABLE_PREFIX_DELEGATION=true", 1112 "--set=cluster.spec.networking.amazonVPC.env=MINIMUM_IP_TARGET=80", 1113 "--set=cluster.spec.networking.amazonVPC.env=WARM_IP_TARGET=10", 1114 "--set=spec.kubeAPIServer.logLevel=4", 1115 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1116 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1117 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1118 ], 1119 focus_regex=r'\[Conformance\]', 1120 skip_regex=r'\[FOOBAR\]', # leaving it empty will allow kops to add extra skips 1121 test_timeout_minutes=200, 1122 test_parallelism=1, # serial tests 1123 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1124 runs_per_day=6), 1125 1126 build_test(name_override="ci-kubernetes-e2e-al2023-aws-conformance-aws-cni-canary", 1127 cloud="aws", 1128 distro="al2023", 1129 networking="amazonvpc", 1130 k8s_version="ci", 1131 kops_version="https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt", # pylint: disable=line-too-long 1132 cluster_name="kubernetes-e2e-al2023-aws-conformance-aws-cni-canary.k8s.local", 1133 kops_channel="alpha", 1134 build_cluster="k8s-infra-prow-build", 1135 extra_flags=[ 1136 "--node-size=r5d.xlarge", 1137 "--master-size=r5d.xlarge", 1138 "--set=cluster.spec.networking.amazonVPC.env=ENABLE_PREFIX_DELEGATION=true", 1139 "--set=cluster.spec.networking.amazonVPC.env=MINIMUM_IP_TARGET=80", 1140 "--set=cluster.spec.networking.amazonVPC.env=WARM_IP_TARGET=10", 1141 "--set=spec.kubeAPIServer.logLevel=4", 1142 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1143 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1144 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1145 ], 1146 focus_regex=r'\[Conformance\]', 1147 skip_regex=r'\[FOOBAR\]', # leaving it empty will allow kops to add extra skips 1148 test_timeout_minutes=200, 1149 test_parallelism=1, # serial tests 1150 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1151 runs_per_day=6), 1152 1153 build_test(name_override="ci-kubernetes-e2e-al2023-aws-conformance-cilium-canary", 1154 cloud="aws", 1155 distro="al2023", 1156 networking="cilium", 1157 k8s_version="ci", 1158 kops_version="https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/master/latest-ci.txt", # pylint: disable=line-too-long 1159 cluster_name="kubernetes-e2e-al2023-aws-conformance-cilium.k8s.local", 1160 kops_channel="alpha", 1161 build_cluster="k8s-infra-prow-build", 1162 extra_flags=[ 1163 "--set=spec.kubeAPIServer.logLevel=4", 1164 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1165 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1166 "--set=spec.kubeProxy.enabled=false", 1167 "--set=spec.networking.cilium.enableNodePort=true", 1168 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1169 ], 1170 focus_regex=r'\[Conformance\]', 1171 skip_regex=r'should.serve.endpoints.on.same.port.and.different.protocols|same.hostPort.but.different.hostIP.and.protocol', # pylint: disable=line-too-long 1172 # https://github.com/cilium/cilium/pull/29524 1173 test_timeout_minutes=200, 1174 test_parallelism=1, # serial tests 1175 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1176 runs_per_day=6), 1177 1178 build_test(name_override="ci-kubernetes-e2e-cos-gce-disruptive-canary", 1179 cloud="gce", 1180 distro="cos105", 1181 networking="kubenet", 1182 k8s_version="ci", 1183 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1184 kops_channel="alpha", 1185 build_cluster="k8s-infra-prow-build", 1186 extra_flags=[ 1187 "--image=cos-cloud/cos-105-17412-156-49", 1188 "--node-count=3", 1189 "--gce-service-account=default", 1190 ], 1191 focus_regex=r'\[Disruptive\]', 1192 skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]|\[KubeUp\]', # pylint: disable=line-too-long 1193 test_timeout_minutes=600, 1194 test_parallelism=1, # serial tests 1195 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1196 runs_per_day=3), 1197 1198 build_test(name_override="ci-kubernetes-e2e-cos-gce-reboot-canary", 1199 cloud="gce", 1200 distro="cos105", 1201 networking="gce", 1202 k8s_version="ci", 1203 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1204 kops_channel="alpha", 1205 build_cluster="k8s-infra-prow-build", 1206 extra_flags=[ 1207 "--image=cos-cloud/cos-105-17412-156-49", 1208 "--node-count=3", 1209 "--gce-service-account=default", 1210 ], 1211 focus_regex=r'\[Feature:Reboot\]', 1212 skip_regex=r'\[FOOBAR\]', 1213 test_timeout_minutes=300, 1214 test_parallelism=1, # serial tests 1215 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1216 runs_per_day=3), 1217 1218 build_test(name_override="ci-kubernetes-e2e-al2023-aws-disruptive-canary", 1219 cloud="aws", 1220 distro="al2023", 1221 networking="kubenet", 1222 k8s_version="ci", 1223 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1224 kops_channel="alpha", 1225 build_cluster="k8s-infra-prow-build", 1226 focus_regex=r'\[Disruptive\]', 1227 skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 1228 test_timeout_minutes=500, 1229 test_parallelism=1, # serial tests 1230 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1231 runs_per_day=3), 1232 1233 build_test(name_override="ci-kubernetes-e2e-cos-gce-serial-canary", 1234 cloud="gce", 1235 distro="cos105", 1236 networking="kubenet", 1237 k8s_version="ci", 1238 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1239 kops_channel="alpha", 1240 build_cluster="k8s-infra-prow-build", 1241 extra_flags=[ 1242 "--image=cos-cloud/cos-105-17412-156-49", 1243 "--node-volume-size=100", 1244 "--gce-service-account=default", 1245 ], 1246 storage_e2e_cred=True, 1247 focus_regex=r'\[Serial\]', 1248 skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]|\[KubeUp\]', # pylint: disable=line-too-long 1249 test_timeout_minutes=600, 1250 test_parallelism=1, # serial tests 1251 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1252 runs_per_day=4), 1253 1254 build_test(name_override="ci-kubernetes-e2e-al2023-aws-serial-canary", 1255 cloud="aws", 1256 distro="al2023", 1257 networking="kubenet", 1258 k8s_version="ci", 1259 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1260 kops_channel="alpha", 1261 build_cluster="k8s-infra-prow-build", 1262 extra_flags=[ 1263 "--node-volume-size=100", 1264 "--set=spec.packages=nfs-utils", 1265 ], 1266 focus_regex=r'\[Serial\]', 1267 skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 1268 test_timeout_minutes=600, 1269 test_parallelism=1, # serial tests 1270 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1271 runs_per_day=4), 1272 1273 build_test(name_override="ci-kubernetes-e2e-al2023-aws-alpha-features", 1274 cloud="aws", 1275 distro="al2023", 1276 networking="kubenet", 1277 k8s_version="ci", 1278 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1279 kops_channel="alpha", 1280 build_cluster="k8s-infra-prow-build", 1281 extra_flags=[ 1282 "--set=spec.kubeAPIServer.logLevel=4", 1283 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1284 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1285 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1286 "--set=spec.kubeAPIServer.runtimeConfig=api/all=true" 1287 ], 1288 kubernetes_feature_gates="AllAlpha,-InTreePluginGCEUnregister,DisableCloudProviders,DisableKubeletCloudCredentialProviders,-EventedPLEG", # pylint: disable=line-too-long 1289 focus_regex=r'\[Feature:(AdmissionWebhookMatchConditions|InPlacePodVerticalScaling|SidecarContainers|StorageVersionAPI|PodPreset|StatefulSetAutoDeletePVC)\]|Networking', # pylint: disable=line-too-long 1290 skip_regex=r'\[Feature:(SCTPConnectivity|Volumes|Networking-Performance)\]|IPv6|csi-hostpath-v0', # pylint: disable=line-too-long 1291 test_timeout_minutes=240, 1292 test_parallelism=4, 1293 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops", "amazon-ec2-al2023"], 1294 runs_per_day=6), 1295 1296 build_test(name_override="ci-kubernetes-e2e-cos-gce-alpha-features", 1297 cloud="gce", 1298 distro="cos105", 1299 networking="kubenet", 1300 k8s_version="ci", 1301 kops_version="https://storage.googleapis.com/kops-ci/bin/latest-ci.txt", 1302 kops_channel="alpha", 1303 build_cluster="k8s-infra-prow-build", 1304 extra_flags=[ 1305 "--image=cos-cloud/cos-105-17412-156-49", 1306 "--set=spec.kubeAPIServer.logLevel=4", 1307 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1308 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1309 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1310 "--set=spec.kubeAPIServer.runtimeConfig=api/all=true", 1311 "--gce-service-account=default", 1312 ], 1313 kubernetes_feature_gates="AllAlpha,-InTreePluginGCEUnregister,DisableCloudProviders,DisableKubeletCloudCredentialProviders,-EventedPLEG", # pylint: disable=line-too-long 1314 focus_regex=r'\[Feature:(AdmissionWebhookMatchConditions|InPlacePodVerticalScaling|SidecarContainers|StorageVersionAPI|PodPreset|StatefulSetAutoDeletePVC)\]|Networking', # pylint: disable=line-too-long 1315 skip_regex=r'\[Feature:(SCTPConnectivity|Volumes|Networking-Performance)\]|IPv6|csi-hostpath-v0', # pylint: disable=line-too-long 1316 test_timeout_minutes=240, 1317 test_parallelism=4, 1318 extra_dashboards=["sig-cluster-lifecycle-kubeup-to-kops"], 1319 runs_per_day=6), 1320 ] 1321 return results 1322 1323 ################################ 1324 # kops-periodics-versions.yaml # 1325 ################################ 1326 def generate_conformance(): 1327 results = [] 1328 for version in ['1.29', '1.28', '1.27']: 1329 results.append( 1330 build_test( 1331 cloud='aws', 1332 build_cluster='k8s-infra-kops-prow-build', 1333 k8s_version=version, 1334 kops_version=version, 1335 kops_channel='alpha', 1336 name_override=f"kops-aws-conformance-{version.replace('.', '-')}", 1337 networking='calico', 1338 test_parallelism=1, 1339 test_timeout_minutes=150, 1340 extra_dashboards=['kops-conformance'], 1341 runs_per_day=1, 1342 focus_regex=r'\[Conformance\]', 1343 skip_regex=r'\[NoSkip\]', 1344 ) 1345 ) 1346 results.append( 1347 build_test( 1348 cloud='aws', 1349 build_cluster='k8s-infra-kops-prow-build', 1350 k8s_version=version, 1351 kops_version=version, 1352 kops_channel='alpha', 1353 name_override=f"kops-aws-conformance-arm64-{version.replace('.', '-')}", 1354 networking='calico', 1355 distro="u2204arm64", 1356 extra_flags=["--zones=eu-central-1a", 1357 "--node-size=t4g.large", 1358 "--master-size=t4g.large"], 1359 test_parallelism=1, 1360 test_timeout_minutes=150, 1361 extra_dashboards=['kops-conformance'], 1362 runs_per_day=1, 1363 focus_regex=r'\[Conformance\]', 1364 skip_regex=r'\[NoSkip\]', 1365 ) 1366 ) 1367 return results 1368 1369 ############################### 1370 # kops-periodics-distros.yaml # 1371 ############################### 1372 distros = ['debian10', 'debian11', 'debian12', 1373 'ubuntu2004', 'ubuntu2004arm64', 1374 'ubuntu2204', 'ubuntu2204arm64', 1375 'ubuntu2404', 'ubuntu2404arm64', 1376 'amazonlinux2', 'al2023', 1377 'rhel8', 'rhel9', 'rocky8', 1378 'flatcar'] 1379 def generate_distros(): 1380 results = [] 1381 for distro in distros: 1382 distro_short = distro.replace('ubuntu', 'u').replace('debian', 'deb').replace('amazonlinux', 'amzn') # pylint: disable=line-too-long 1383 extra_flags = [] 1384 if 'arm64' in distro: 1385 extra_flags = [ 1386 "--zones=eu-west-1a", 1387 "--node-size=m6g.large", 1388 "--master-size=m6g.large" 1389 ] 1390 results.append( 1391 build_test(distro=distro_short, 1392 networking='cilium', 1393 k8s_version='stable', 1394 kops_channel='alpha', 1395 name_override=f"kops-aws-distro-{distro}", 1396 extra_dashboards=['kops-distros'], 1397 extra_flags=extra_flags, 1398 runs_per_day=3, 1399 build_cluster='k8s-infra-kops-prow-build', 1400 ) 1401 ) 1402 return results 1403 1404 ############################### 1405 # kops-presubmits-distros.yaml # 1406 ############################### 1407 def generate_presubmits_distros(): 1408 results = [] 1409 for distro in distros: 1410 distro_short = distro.replace('ubuntu', 'u').replace('debian', 'deb').replace('amazonlinux', 'amzn') # pylint: disable=line-too-long 1411 extra_flags = [] 1412 if 'arm64' in distro: 1413 extra_flags = [ 1414 "--zones=eu-west-1a", 1415 "--node-size=m6g.large", 1416 "--master-size=m6g.large" 1417 ] 1418 results.append( 1419 presubmit_test( 1420 distro=distro_short, 1421 networking='calico', 1422 k8s_version='stable', 1423 kops_channel='alpha', 1424 name=f"pull-kops-aws-distro-{distro}", 1425 tab_name=f"e2e-{distro}", 1426 extra_flags=extra_flags, 1427 always_run=False, 1428 ) 1429 ) 1430 return results 1431 1432 ####################################### 1433 # kops-periodics-network-plugins.yaml # 1434 ####################################### 1435 def generate_network_plugins(): 1436 1437 plugins = ['amazon-vpc', 'calico', 'canal', 'cilium', 'cilium-etcd', 'cilium-eni', 'flannel', 'kopeio', 'kuberouter'] # pylint: disable=line-too-long 1438 results = [] 1439 for plugin in plugins: 1440 networking_arg = plugin.replace('amazon-vpc', 'amazonvpc').replace('kuberouter', 'kube-router') # pylint: disable=line-too-long 1441 k8s_version = 'stable' 1442 if plugin in ['canal', 'flannel']: 1443 k8s_version = '1.27' 1444 if plugin in ['kuberouter']: 1445 k8s_version = 'ci' 1446 results.append( 1447 build_test( 1448 build_cluster='k8s-infra-kops-prow-build', 1449 distro='u2204', 1450 k8s_version=k8s_version, 1451 kops_channel='alpha', 1452 name_override=f"kops-aws-cni-{plugin}", 1453 networking=networking_arg, 1454 extra_flags=['--node-size=t3.large'], 1455 extra_dashboards=['kops-network-plugins'], 1456 runs_per_day=3, 1457 ) 1458 ) 1459 return results 1460 1461 ################################ 1462 # kops-periodics-upgrades.yaml # 1463 ################################ 1464 def generate_upgrades(): 1465 1466 kops26 = 'v1.26.6' 1467 kops27 = 'v1.27.3' 1468 kops28 = 'v1.28.4' 1469 1470 versions_list = [ 1471 # kops k8s kops k8s 1472 # 1.27 release branch 1473 ((kops26, 'v1.26.11'), ('1.27', 'v1.27.8')), 1474 ((kops27, 'v1.27.7'), ('1.27', 'v1.27.8')), 1475 # 1.28 release branch 1476 ((kops26, 'v1.26.11'), ('1.28', 'v1.27.8')), 1477 ((kops27, 'v1.27.8'), ('1.28', 'v1.28.4')), 1478 ((kops28, 'v1.28.3'), ('1.28', 'v1.28.4')), 1479 # 1.26 upgrade to latest 1480 ((kops26, 'v1.26.0'), ('latest', 'v1.27.0')), 1481 # 1.27 upgrade to latest 1482 ((kops27, 'v1.24.0'), ('latest', 'v1.25.0')), 1483 ((kops27, 'v1.25.0'), ('latest', 'v1.26.0')), 1484 ((kops27, 'v1.26.0'), ('latest', 'v1.27.0')), 1485 ((kops27, 'v1.27.0'), ('latest', 'v1.28.0')), 1486 # 1.28 upgrade to latest 1487 ((kops28, 'v1.24.0'), ('latest', 'v1.25.0')), 1488 ((kops28, 'v1.25.0'), ('latest', 'v1.26.0')), 1489 ((kops28, 'v1.26.0'), ('latest', 'v1.27.0')), 1490 ((kops28, 'v1.27.0'), ('latest', 'v1.28.0')), 1491 # we should have an upgrade test for every supported K8s version 1492 (('latest', 'v1.28.0'), ('latest', 'latest')), 1493 (('latest', 'v1.27.0'), ('latest', 'v1.28.0')), 1494 (('latest', 'v1.26.0'), ('latest', 'v1.27.0')), 1495 (('latest', 'v1.25.0'), ('latest', 'v1.26.0')), 1496 (('latest', 'v1.24.0'), ('latest', 'v1.25.0')), 1497 # kOps latest should always be able to upgrade from stable to latest and stable to ci 1498 (('latest', 'stable'), ('latest', 'latest')), 1499 (('latest', 'stable'), ('latest', 'ci')), 1500 ] 1501 def shorten(version): 1502 version = re.sub(r'^v', '', version) 1503 version = re.sub(r'^(\d+\.\d+)\.\d+$', r'\g<1>', version) 1504 return version.replace('.', '') 1505 results = [] 1506 for versions in versions_list: 1507 kops_a = versions[0][0] 1508 k8s_a = versions[0][1] 1509 kops_b = versions[1][0] 1510 k8s_b = versions[1][1] 1511 job_name = f"kops-aws-upgrade-k{shorten(k8s_a)}-ko{shorten(kops_a)}-to-k{shorten(k8s_b)}-ko{shorten(kops_b)}" # pylint: disable=line-too-long 1512 runs_per_day = 3 if kops_b == 'latest' else 1 1513 env = { 1514 'KOPS_VERSION_A': kops_a, 1515 'K8S_VERSION_A': k8s_a, 1516 'KOPS_VERSION_B': kops_b, 1517 'K8S_VERSION_B': k8s_b, 1518 } 1519 addonsenv = { 1520 'KOPS_VERSION_A': kops_a, 1521 'K8S_VERSION_A': k8s_a, 1522 'KOPS_VERSION_B': kops_b, 1523 'K8S_VERSION_B': k8s_b, 1524 'KOPS_SKIP_E2E': '1', 1525 'KOPS_TEMPLATE': 'tests/e2e/templates/many-addons.yaml.tmpl', 1526 'KOPS_CONTROL_PLANE_SIZE': '3', 1527 } 1528 build_cluster = 'k8s-infra-kops-prow-build' 1529 # Older k/k builds dont have new enough aws-sdk-go versions to support 1530 # running their e2e.test binary from community-owned EKS clusters. 1531 if re.match(r'v1\.2[4-6]', k8s_a): 1532 build_cluster = 'default' 1533 1534 results.append( 1535 build_test(name_override=job_name, 1536 distro='u2204', 1537 networking='calico', 1538 k8s_version='stable', 1539 kops_channel='alpha', 1540 extra_dashboards=['kops-upgrades'], 1541 runs_per_day=runs_per_day, 1542 test_timeout_minutes=120, 1543 scenario='upgrade-ab', 1544 env=env, 1545 build_cluster=build_cluster, 1546 ) 1547 ) 1548 results.append( 1549 build_test(name_override=job_name + "-many-addons", 1550 distro='u2204', 1551 networking='calico', 1552 k8s_version='stable', 1553 kops_channel='alpha', 1554 extra_dashboards=['kops-upgrades-many-addons'], 1555 test_timeout_minutes=120, 1556 runs_per_day=runs_per_day, 1557 scenario='upgrade-ab', 1558 env=addonsenv, 1559 build_cluster=build_cluster, 1560 ) 1561 ) 1562 1563 # One-off upgrade test: from kops-128 -> latest with k8s.local, with calico 1564 # Exit criteria: Remove when https://github.com/kubernetes/kops/issues/16276 is fixed 1565 results.append( 1566 build_test(name_override='kops-aws-upgrade-bug16276-calico', 1567 build_cluster='k8s-infra-kops-prow-build', 1568 cluster_name='bug16276-calico.k8s.local', 1569 distro='u2204', 1570 networking='calico', 1571 k8s_version='stable', 1572 kops_channel='alpha', 1573 extra_dashboards=['kops-upgrades'], 1574 runs_per_day=3, 1575 test_timeout_minutes=120, 1576 scenario='upgrade-ab', 1577 env={ 1578 'KOPS_VERSION_A': kops28, 1579 'K8S_VERSION_A': 'v1.28.0', 1580 'KOPS_VERSION_B': 'latest', 1581 'K8S_VERSION_B': 'v1.28.0', 1582 }, 1583 ) 1584 ) 1585 1586 # One-off upgrade test: from kops-128 -> latest with k8s.local, with cilium 1587 # Exit criteria: Remove when https://github.com/kubernetes/kops/issues/16276 is fixed 1588 results.append( 1589 build_test(name_override='kops-aws-upgrade-bug16276-cilium', 1590 build_cluster='k8s-infra-kops-prow-build', 1591 cluster_name='bug16276-cilium.k8s.local', 1592 distro='u2204', 1593 networking='cilium', 1594 k8s_version='stable', 1595 kops_channel='alpha', 1596 extra_dashboards=['kops-upgrades'], 1597 runs_per_day=3, 1598 test_timeout_minutes=120, 1599 scenario='upgrade-ab', 1600 env={ 1601 'KOPS_VERSION_A': kops28, 1602 'K8S_VERSION_A': 'v1.28.0', 1603 'KOPS_VERSION_B': 'latest', 1604 'K8S_VERSION_B': 'v1.28.0', 1605 }, 1606 ) 1607 ) 1608 return results 1609 1610 ############################### 1611 # kops-presubmits-scale.yaml # 1612 ############################### 1613 def generate_presubmits_scale(): 1614 results = [ 1615 presubmit_test( 1616 name='presubmit-kops-aws-scale-amazonvpc', 1617 scenario='scalability', 1618 build_cluster="eks-prow-build-cluster", 1619 # only helps with setting the right anotation test.kops.k8s.io/networking 1620 networking='amazonvpc', 1621 always_run=False, 1622 env={ 1623 'CNI_PLUGIN': "amazonvpc", 1624 } 1625 ), 1626 presubmit_test( 1627 name='presubmit-kops-aws-scale-amazonvpc-using-cl2', 1628 scenario='scalability', 1629 build_cluster="eks-prow-build-cluster", 1630 # only helps with setting the right anotation test.kops.k8s.io/networking 1631 networking='amazonvpc', 1632 always_run=False, 1633 artifacts='$(ARTIFACTS)', 1634 test_timeout_minutes=450, 1635 use_preset_for_account_creds='preset-aws-credential-boskos-scale-001-kops', 1636 env={ 1637 'CNI_PLUGIN': "amazonvpc", 1638 'KUBE_NODE_COUNT': "5000", 1639 'CL2_LOAD_TEST_THROUGHPUT': "50", 1640 'CL2_DELETE_TEST_THROUGHPUT': "50", 1641 'CL2_RATE_LIMIT_POD_CREATION': "false", 1642 'NODE_MODE': "master", 1643 'CONTROL_PLANE_COUNT': "3", 1644 'CONTROL_PLANE_SIZE': "c5.18xlarge", 1645 'KOPS_STATE_STORE' : "s3://k8s-infra-kops-scale-tests", 1646 'PROMETHEUS_SCRAPE_KUBE_PROXY': "true", 1647 'CL2_ENABLE_DNS_PROGRAMMING': "true", 1648 'CL2_ENABLE_API_AVAILABILITY_MEASUREMENT': "true", 1649 'CL2_API_AVAILABILITY_PERCENTAGE_THRESHOLD': "99.5", 1650 'CL2_ALLOWED_SLOW_API_CALLS': "1", 1651 'ENABLE_PROMETHEUS_SERVER': "true", 1652 'PROMETHEUS_PVC_STORAGE_CLASS': "gp2", 1653 'CL2_NETWORK_LATENCY_THRESHOLD': "0.5s", 1654 'CL2_ENABLE_VIOLATIONS_FOR_NETWORK_PROGRAMMING_LATENCIES': "true", 1655 'CL2_NETWORK_PROGRAMMING_LATENCY_THRESHOLD': "20s", 1656 'CL2_ENABLE_DNSTESTS': "false", 1657 'CL2_USE_ADVANCED_DNSTEST': "false", 1658 'SMALL_STATEFUL_SETS_PER_NAMESPACE': 0, 1659 'MEDIUM_STATEFUL_SETS_PER_NAMESPACE': 0 1660 } 1661 ), 1662 presubmit_test( 1663 name='presubmit-kops-aws-small-scale-amazonvpc-using-cl2', 1664 scenario='scalability', 1665 build_cluster="eks-prow-build-cluster", 1666 # only helps with setting the right anotation test.kops.k8s.io/networking 1667 networking='amazonvpc', 1668 always_run=False, 1669 artifacts='$(ARTIFACTS)', 1670 test_timeout_minutes=450, 1671 use_preset_for_account_creds='preset-aws-credential-boskos-scale-001-kops', 1672 env={ 1673 'CNI_PLUGIN': "amazonvpc", 1674 'KUBE_NODE_COUNT': "500", 1675 'CL2_SCHEDULER_THROUGHPUT_THRESHOLD': "20", 1676 'CONTROL_PLANE_COUNT': "3", 1677 'CONTROL_PLANE_SIZE': "c5.4xlarge", 1678 'CL2_LOAD_TEST_THROUGHPUT': "50", 1679 'CL2_DELETE_TEST_THROUGHPUT': "50", 1680 'CL2_RATE_LIMIT_POD_CREATION': "false", 1681 'NODE_MODE': "master", 1682 'KOPS_STATE_STORE' : "s3://k8s-infra-kops-scale-tests", 1683 'PROMETHEUS_SCRAPE_KUBE_PROXY': "true", 1684 'CL2_ENABLE_DNS_PROGRAMMING': "true", 1685 'CL2_ENABLE_API_AVAILABILITY_MEASUREMENT': "true", 1686 'CL2_API_AVAILABILITY_PERCENTAGE_THRESHOLD': "99.5", 1687 'CL2_ALLOWED_SLOW_API_CALLS': "1", 1688 'ENABLE_PROMETHEUS_SERVER': "true", 1689 'PROMETHEUS_PVC_STORAGE_CLASS': "gp2", 1690 'CL2_NETWORK_LATENCY_THRESHOLD': "0.5s", 1691 'CL2_ENABLE_VIOLATIONS_FOR_NETWORK_PROGRAMMING_LATENCIES': "true", 1692 'CL2_NETWORK_PROGRAMMING_LATENCY_THRESHOLD': "20s" 1693 } 1694 ), 1695 presubmit_test( 1696 name='presubmit-kops-gce-scale-ipalias-using-cl2', 1697 scenario='scalability', 1698 build_cluster="k8s-infra-prow-build", 1699 # only helps with setting the right anotation test.kops.k8s.io/networking 1700 networking='gce', 1701 cloud="gce", 1702 always_run=False, 1703 artifacts='$(ARTIFACTS)', 1704 test_timeout_minutes=450, 1705 use_preset_for_account_creds='preset-aws-credential-boskos-scale-001-kops', 1706 env={ 1707 'CNI_PLUGIN': "gce", 1708 'KUBE_NODE_COUNT': "5000", 1709 'CL2_LOAD_TEST_THROUGHPUT': "50", 1710 'CL2_DELETE_TEST_THROUGHPUT': "50", 1711 'CL2_RATE_LIMIT_POD_CREATION': "false", 1712 'NODE_MODE': "master", 1713 'CONTROL_PLANE_COUNT': "1", 1714 'CONTROL_PLANE_SIZE': "c3-standard-88", 1715 'PROMETHEUS_SCRAPE_KUBE_PROXY': "true", 1716 'CL2_ENABLE_DNS_PROGRAMMING': "true", 1717 'CL2_ENABLE_API_AVAILABILITY_MEASUREMENT': "true", 1718 'CL2_API_AVAILABILITY_PERCENTAGE_THRESHOLD': "99.5", 1719 'CL2_ALLOWED_SLOW_API_CALLS': "1", 1720 'ENABLE_PROMETHEUS_SERVER': "true", 1721 'PROMETHEUS_PVC_STORAGE_CLASS': "ssd-csi", 1722 'CL2_NETWORK_LATENCY_THRESHOLD': "0.5s", 1723 'CL2_ENABLE_VIOLATIONS_FOR_NETWORK_PROGRAMMING_LATENCIES': "true", 1724 'CL2_NETWORK_PROGRAMMING_LATENCY_THRESHOLD': "20s", 1725 'CL2_ENABLE_DNSTESTS': "false", 1726 'CL2_USE_ADVANCED_DNSTEST': "false", 1727 'SMALL_STATEFUL_SETS_PER_NAMESPACE': 0, 1728 'MEDIUM_STATEFUL_SETS_PER_NAMESPACE': 0 1729 } 1730 ), 1731 presubmit_test( 1732 name='presubmit-kops-gce-small-scale-ipalias-using-cl2', 1733 scenario='scalability', 1734 build_cluster="k8s-infra-prow-build", 1735 # only helps with setting the right anotation test.kops.k8s.io/networking 1736 networking='gce', 1737 cloud="gce", 1738 always_run=False, 1739 artifacts='$(ARTIFACTS)', 1740 test_timeout_minutes=450, 1741 env={ 1742 'CNI_PLUGIN': "gce", 1743 'KUBE_NODE_COUNT': "500", 1744 'CL2_SCHEDULER_THROUGHPUT_THRESHOLD': "20", 1745 'CONTROL_PLANE_COUNT': "1", 1746 'CONTROL_PLANE_SIZE': "c3-standard-88", 1747 'CL2_LOAD_TEST_THROUGHPUT': "50", 1748 'CL2_DELETE_TEST_THROUGHPUT': "50", 1749 'CL2_RATE_LIMIT_POD_CREATION': "false", 1750 'NODE_MODE': "master", 1751 'PROMETHEUS_SCRAPE_KUBE_PROXY': "true", 1752 'CL2_ENABLE_DNS_PROGRAMMING': "true", 1753 'CL2_ENABLE_API_AVAILABILITY_MEASUREMENT': "true", 1754 'CL2_API_AVAILABILITY_PERCENTAGE_THRESHOLD': "99.5", 1755 'CL2_ALLOWED_SLOW_API_CALLS': "1", 1756 'ENABLE_PROMETHEUS_SERVER': "true", 1757 'PROMETHEUS_PVC_STORAGE_CLASS': "ssd-csi", 1758 'CL2_NETWORK_LATENCY_THRESHOLD': "0.5s", 1759 'CL2_ENABLE_VIOLATIONS_FOR_NETWORK_PROGRAMMING_LATENCIES': "true", 1760 'CL2_NETWORK_PROGRAMMING_LATENCY_THRESHOLD': "20s" 1761 } 1762 ) 1763 ] 1764 return results 1765 1766 ################################ 1767 # kops-periodics-versions.yaml # 1768 ################################ 1769 def generate_versions(): 1770 results = [ 1771 build_test( 1772 k8s_version='ci', 1773 kops_channel='alpha', 1774 name_override='kops-aws-k8s-latest', 1775 networking='calico', 1776 extra_dashboards=['kops-versions'], 1777 runs_per_day=8, 1778 # This version marker is only used by the k/k presubmit job 1779 publish_version_marker='gs://kops-ci/bin/latest-ci-green.txt', 1780 ) 1781 ] 1782 for version in ['1.29', '1.28', '1.27', '1.26', '1.25']: 1783 results.append( 1784 build_test( 1785 cloud='aws', 1786 build_cluster='k8s-infra-kops-prow-build', 1787 k8s_version=version, 1788 kops_channel='alpha', 1789 name_override=f"kops-aws-k8s-{version.replace('.', '-')}", 1790 networking='calico', 1791 extra_dashboards=['kops-versions'], 1792 runs_per_day=8, 1793 ) 1794 ) 1795 return results 1796 1797 ###################### 1798 # kops-pipeline.yaml # 1799 ###################### 1800 def generate_pipeline(): 1801 results = [] 1802 for version in ['master', '1.29', '1.28', '1.27']: 1803 branch = version if version == 'master' else f"release-{version}" 1804 publish_version_marker = f"gs://kops-ci/markers/{branch}/latest-ci-updown-green.txt" 1805 kops_version = f"https://storage.googleapis.com/k8s-staging-kops/kops/releases/markers/{branch}/latest-ci.txt" # pylint: disable=line-too-long 1806 results.append( 1807 build_test( 1808 cloud="aws", 1809 build_cluster="default", 1810 k8s_version=version.replace('master', 'latest'), 1811 kops_version=kops_version, 1812 kops_channel='alpha', 1813 name_override=f"kops-pipeline-updown-kops{version.replace('.', '')}", 1814 networking='calico', 1815 extra_dashboards=['kops-versions'], 1816 runs_per_day=24, 1817 skip_regex=r'\[Slow\]|\[Serial\]', 1818 focus_regex=r'\[k8s.io\]\sNetworking.*\[Conformance\]', 1819 publish_version_marker=publish_version_marker, 1820 ) 1821 ) 1822 return results 1823 1824 ######################################## 1825 # kops-presubmits-network-plugins.yaml # 1826 ######################################## 1827 def generate_presubmits_network_plugins(): 1828 plugins = { 1829 'amazonvpc': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.amazon-vpc-routed-eni\/|pkg\/model\/(firewall|components\/containerd|components\/kubeproxy|iam\/iam_builder)\.go|nodeup\/pkg\/model\/kubelet\.go)', # pylint: disable=line-too-long 1830 'calico': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.projectcalico\.org\/|pkg\/model\/(components\/containerd|firewall|pki|iam\/iam_builder)\.go|nodeup\/pkg\/model\/networking\/calico\.go)', # pylint: disable=line-too-long 1831 'canal': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.projectcalico\.org\.canal\/)', # pylint: disable=line-too-long 1832 'cilium': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.cilium\.io\/|pkg\/model\/(components\/containerd|firewall|components\/cilium|iam\/iam_builder)\.go|nodeup\/pkg\/model\/(context|networking\/cilium)\.go)', # pylint: disable=line-too-long 1833 'cilium-etcd': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.cilium\.io\/|pkg\/model\/(components\/containerd|firewall|components\/cilium|iam\/iam_builder)\.go|nodeup\/pkg\/model\/(context|networking\/cilium)\.go)', # pylint: disable=line-too-long 1834 'cilium-eni': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.cilium\.io\/|pkg\/model\/(components\/containerd|firewall|components\/cilium|iam\/iam_builder)\.go|nodeup\/pkg\/model\/(context|networking\/cilium)\.go)', # pylint: disable=line-too-long 1835 'flannel': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.flannel\/|pkg\/model\/components\/containerd\.go)', # pylint: disable=line-too-long 1836 'kuberouter': r'^(upup\/models\/cloudup\/resources\/addons\/networking\.kuberouter\/|pkg\/model\/components\/containerd\.go)', # pylint: disable=line-too-long 1837 } 1838 supports_ipv6 = {'amazonvpc', 'calico', 'cilium'} 1839 results = [] 1840 for plugin, run_if_changed in plugins.items(): 1841 k8s_version = 'stable' 1842 networking_arg = plugin 1843 optional = False 1844 if plugin in ['canal', 'flannel']: 1845 k8s_version = '1.27' 1846 if plugin == 'kuberouter': 1847 networking_arg = 'kube-router' 1848 k8s_version = 'ci' 1849 optional = True 1850 results.append( 1851 presubmit_test( 1852 distro='u2204arm64', 1853 k8s_version=k8s_version, 1854 kops_channel='alpha', 1855 name=f"pull-kops-e2e-cni-{plugin}", 1856 tab_name=f"e2e-{plugin}", 1857 networking=networking_arg, 1858 extra_flags=["--node-size=t4g.large"], 1859 run_if_changed=run_if_changed, 1860 optional=optional, 1861 ) 1862 ) 1863 if plugin in supports_ipv6: 1864 optional = True 1865 if plugin == 'amazonvpc': 1866 run_if_changed = None 1867 results.append( 1868 presubmit_test( 1869 name=f"pull-kops-e2e-cni-{plugin}-ipv6", 1870 distro='u2204arm64', 1871 tab_name=f"e2e-{plugin}-ipv6", 1872 networking=networking_arg, 1873 extra_flags=['--ipv6', 1874 '--topology=private', 1875 '--bastion', 1876 '--zones=us-west-2a', 1877 '--dns=public', 1878 ], 1879 run_if_changed=run_if_changed, 1880 optional=optional, 1881 ) 1882 ) 1883 1884 return results 1885 1886 ############################ 1887 # kops-presubmits-e2e.yaml # 1888 ############################ 1889 def generate_presubmits_e2e(): 1890 jobs = [ 1891 presubmit_test( 1892 distro='u2204arm64', 1893 k8s_version='ci', 1894 kops_channel='alpha', 1895 name='pull-kops-e2e-k8s-ci', 1896 networking='calico', 1897 tab_name='e2e-containerd-ci', 1898 always_run=False, 1899 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 1900 ), 1901 presubmit_test( 1902 distro='u2204arm64', 1903 k8s_version='ci', 1904 kops_channel='alpha', 1905 name='pull-kops-e2e-k8s-ci-ha', 1906 networking='calico', 1907 extra_flags=[ 1908 "--master-count=3", 1909 "--node-count=6", 1910 "--zones=eu-central-1a,eu-central-1b,eu-central-1c"], 1911 tab_name='e2e-containerd-ci-ha', 1912 always_run=False, 1913 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 1914 ), 1915 presubmit_test( 1916 distro='channels', 1917 k8s_version='stable', 1918 kops_channel='alpha', 1919 name='pull-kops-e2e-k8s-aws-calico', 1920 networking='calico', 1921 tab_name='e2e-aws-calico', 1922 always_run=True, 1923 ), 1924 presubmit_test( 1925 distro='al2023', 1926 k8s_version='stable', 1927 kops_channel='alpha', 1928 name='pull-kops-e2e-k8s-aws-amazonvpc', 1929 extra_flags=[ 1930 "--node-size=r5d.xlarge", 1931 "--master-size=r5d.xlarge", 1932 "--set=cluster.spec.networking.amazonVPC.env=ENABLE_PREFIX_DELEGATION=true", 1933 "--set=cluster.spec.networking.amazonVPC.env=MINIMUM_IP_TARGET=80", 1934 "--set=cluster.spec.networking.amazonVPC.env=WARM_IP_TARGET=10", 1935 "--set=spec.kubeAPIServer.logLevel=4", 1936 "--set=spec.kubeAPIServer.auditLogMaxSize=2000000000", 1937 "--set=spec.kubeAPIServer.enableAggregatorRouting=true", 1938 "--set=spec.kubeAPIServer.auditLogPath=/var/log/kube-apiserver-audit.log", 1939 ], 1940 networking='amazonvpc', 1941 tab_name='e2e-aws-amazonvpc', 1942 always_run=False, 1943 optional=True, 1944 ), 1945 presubmit_test( 1946 cloud='gce', 1947 k8s_version='stable', 1948 kops_channel='alpha', 1949 name='pull-kops-e2e-k8s-gce-cilium', 1950 networking='cilium', 1951 tab_name='e2e-gce-cilium', 1952 build_cluster="k8s-infra-prow-build", 1953 always_run=True, 1954 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 1955 ), 1956 presubmit_test( 1957 cloud='gce', 1958 k8s_version='stable', 1959 kops_channel='alpha', 1960 name='pull-kops-e2e-k8s-gce-cilium-etcd', 1961 networking='cilium-etcd', 1962 tab_name='e2e-gce-cilium-etcd', 1963 build_cluster="k8s-infra-prow-build", 1964 always_run=False, 1965 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 1966 ), 1967 presubmit_test( 1968 cloud='gce', 1969 k8s_version='stable', 1970 kops_channel='alpha', 1971 name='pull-kops-e2e-k8s-gce-ipalias', 1972 networking='gce', 1973 tab_name='e2e-gce', 1974 build_cluster="k8s-infra-prow-build", 1975 always_run=True, 1976 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 1977 ), 1978 presubmit_test( 1979 cloud='gce', 1980 k8s_version='stable', 1981 kops_channel='alpha', 1982 name='pull-kops-e2e-k8s-gce-long-cluster-name', 1983 networking='cilium', 1984 tab_name='e2e-gce-long-name', 1985 build_cluster="k8s-infra-prow-build", 1986 always_run=False, 1987 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 1988 ), 1989 presubmit_test( 1990 cloud='gce', 1991 k8s_version='ci', 1992 kops_channel='alpha', 1993 name='pull-kops-e2e-k8s-gce-ci', 1994 networking='cilium', 1995 tab_name='e2e-gce-ci', 1996 build_cluster="k8s-infra-prow-build", 1997 always_run=False, 1998 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 1999 ), 2000 presubmit_test( 2001 cloud='gce', 2002 k8s_version='stable', 2003 kops_channel='alpha', 2004 name='pull-kops-e2e-k8s-gce-calico-u2004-k22-containerd', 2005 networking='calico', 2006 tab_name='pull-kops-e2e-k8s-gce-calico-u2004-k22-containerd', 2007 build_cluster="k8s-infra-prow-build", 2008 always_run=False, 2009 feature_flags=['GoogleCloudBucketACL'], 2010 extra_flags=["--gce-service-account=default"], # Workaround for test-infra#24747 2011 ), 2012 # A special test for AWS Cloud-Controller-Manager 2013 presubmit_test( 2014 name="pull-kops-e2e-aws-cloud-controller-manager", 2015 cloud="aws", 2016 distro="u2204", 2017 k8s_version="ci", 2018 extra_flags=['--set=cluster.spec.cloudControllerManager.cloudProvider=aws'], 2019 tab_name='e2e-ccm', 2020 ), 2021 2022 presubmit_test( 2023 name="pull-kops-e2e-aws-load-balancer-controller", 2024 cloud="aws", 2025 distro="u2004", 2026 networking="calico", 2027 scenario="aws-lb-controller", 2028 tab_name="pull-kops-e2e-aws-load-balancer-controller", 2029 ), 2030 2031 presubmit_test( 2032 name="pull-kops-e2e-addon-resource-tracking", 2033 cloud="aws", 2034 distro="u2204", 2035 networking="calico", 2036 scenario="addon-resource-tracking", 2037 tab_name="pull-kops-e2e-aws-addon-resource-tracking", 2038 ), 2039 2040 presubmit_test( 2041 name="pull-kops-e2e-metrics-server", 2042 cloud="aws", 2043 distro="u2204", 2044 networking="calico", 2045 scenario="metrics-server", 2046 tab_name="pull-kops-e2e-aws-metrics-server", 2047 ), 2048 2049 presubmit_test( 2050 name="pull-kops-e2e-pod-identity-webhook", 2051 cloud="aws", 2052 distro="u2204", 2053 networking="calico", 2054 scenario="podidentitywebhook", 2055 tab_name="pull-kops-e2e-aws-pod-identity-webhook", 2056 ), 2057 2058 presubmit_test( 2059 name="pull-kops-e2e-aws-external-dns", 2060 cloud="aws", 2061 networking="calico", 2062 extra_flags=[ 2063 '--set=cluster.spec.externalDNS.provider=external-dns', 2064 '--dns=public', 2065 ], 2066 ), 2067 presubmit_test( 2068 name="pull-kops-e2e-aws-ipv6-external-dns", 2069 cloud="aws", 2070 networking="calico", 2071 extra_flags=[ 2072 '--ipv6', 2073 '--bastion', 2074 '--set=cluster.spec.externalDNS.provider=external-dns', 2075 '--dns=public', 2076 ], 2077 ), 2078 presubmit_test( 2079 name="pull-kops-e2e-aws-node-local-dns", 2080 cloud="aws", 2081 distro='u2204arm64', 2082 extra_flags=[ 2083 '--set=cluster.spec.kubeDNS.nodeLocalDNS.enabled=true' 2084 ], 2085 ), 2086 2087 presubmit_test( 2088 name="pull-kops-e2e-aws-apiserver-nodes", 2089 cloud="aws", 2090 template_path="/home/prow/go/src/k8s.io/kops/tests/e2e/templates/apiserver.yaml.tmpl", 2091 feature_flags=['APIServerNodes'] 2092 ), 2093 2094 presubmit_test( 2095 name="pull-kops-e2e-arm64", 2096 cloud="aws", 2097 distro="u2204arm64", 2098 networking="calico", 2099 extra_flags=["--zones=eu-central-1a", 2100 "--node-size=m6g.large", 2101 "--master-size=m6g.large"], 2102 ), 2103 2104 presubmit_test( 2105 name="pull-kops-e2e-aws-dns-none", 2106 cloud="aws", 2107 distro="u2204arm64", 2108 networking="calico", 2109 extra_flags=["--dns=none"], 2110 ), 2111 presubmit_test( 2112 name="pull-kops-e2e-gce-dns-none", 2113 cloud="gce", 2114 networking="calico", 2115 build_cluster="k8s-infra-prow-build", 2116 extra_flags=["--dns=none", "--gce-service-account=default"], 2117 optional=True, 2118 ), 2119 2120 presubmit_test( 2121 name="pull-kops-e2e-aws-nlb", 2122 cloud="aws", 2123 distro="u2204arm64", 2124 networking="calico", 2125 extra_flags=[ 2126 "--api-loadbalancer-type=public", 2127 "--api-loadbalancer-class=network" 2128 ], 2129 ), 2130 2131 presubmit_test( 2132 name="pull-kops-e2e-aws-terraform", 2133 cloud="aws", 2134 distro="u2204arm64", 2135 terraform_version="1.5.5", 2136 extra_flags=[ 2137 "--dns=public", 2138 ], 2139 ), 2140 presubmit_test( 2141 name="pull-kops-e2e-aws-ipv6-terraform", 2142 cloud="aws", 2143 distro="u2204arm64", 2144 terraform_version="1.5.5", 2145 extra_flags=[ 2146 '--ipv6', 2147 '--bastion', 2148 '--dns=public', 2149 ], 2150 ), 2151 2152 presubmit_test( 2153 distro='channels', 2154 branch='release-1.29', 2155 k8s_version='1.29', 2156 kops_channel='alpha', 2157 name='pull-kops-e2e-k8s-aws-calico-1-29', 2158 networking='calico', 2159 tab_name='e2e-1-29', 2160 always_run=True, 2161 ), 2162 presubmit_test( 2163 distro='channels', 2164 branch='release-1.28', 2165 k8s_version='1.28', 2166 kops_channel='alpha', 2167 name='pull-kops-e2e-k8s-aws-calico-1-28', 2168 networking='calico', 2169 tab_name='e2e-1-28', 2170 always_run=True, 2171 ), 2172 presubmit_test( 2173 distro='channels', 2174 branch='release-1.27', 2175 k8s_version='1.27', 2176 kops_channel='alpha', 2177 name='pull-kops-e2e-k8s-aws-calico-1-27', 2178 networking='calico', 2179 tab_name='e2e-1-27', 2180 always_run=True, 2181 ), 2182 presubmit_test( 2183 distro='channels', 2184 branch='release-1.26', 2185 k8s_version='1.26', 2186 kops_channel='alpha', 2187 name='pull-kops-e2e-k8s-aws-calico-1-26', 2188 networking='calico', 2189 tab_name='e2e-1-26', 2190 always_run=True, 2191 ), 2192 2193 presubmit_test( 2194 distro='u2204arm64', 2195 name="pull-kops-e2e-aws-karpenter", 2196 run_if_changed=r'^upup\/models\/cloudup\/resources\/addons\/karpenter\.sh\/', 2197 networking="cilium", 2198 kops_channel="alpha", 2199 extra_flags=[ 2200 "--instance-manager=karpenter", 2201 "--master-size=c6g.xlarge", 2202 ], 2203 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 2204 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|HostPort|two.untainted.nodes', 2205 ), 2206 presubmit_test( 2207 distro='u2204arm64', 2208 name="pull-kops-e2e-aws-ipv6-karpenter", 2209 #run_if_changed=r'^upup\/models\/cloudup\/resources\/addons\/karpenter\.sh\/', 2210 networking="cilium", 2211 kops_channel="alpha", 2212 extra_flags=[ 2213 "--instance-manager=karpenter", 2214 '--ipv6', 2215 '--topology=private', 2216 '--bastion', 2217 "--master-size=c6g.xlarge", 2218 ], 2219 focus_regex=r'\[Conformance\]|\[NodeConformance\]', 2220 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|HostPort|two.untainted.nodes', 2221 ), 2222 presubmit_test( 2223 name="pull-kops-e2e-aws-upgrade-k124-ko124-to-k125-kolatest", 2224 optional=True, 2225 distro='u2204', 2226 networking='cilium', 2227 k8s_version='stable', 2228 kops_channel='alpha', 2229 scenario='upgrade-ab', 2230 env={ 2231 'KOPS_VERSION_A': "1.24", 2232 'K8S_VERSION_A': "v1.24.0", 2233 'KOPS_VERSION_B': "latest", 2234 'K8S_VERSION_B': "1.25.0", 2235 } 2236 ), 2237 presubmit_test( 2238 name="pull-kops-e2e-aws-upgrade-k125-kolatest-to-k126-kolatest", 2239 optional=True, 2240 distro='u2204', 2241 networking='cilium', 2242 k8s_version='stable', 2243 kops_channel='alpha', 2244 scenario='upgrade-ab', 2245 env={ 2246 'KOPS_VERSION_A': "latest", 2247 'K8S_VERSION_A': "v1.25.0", 2248 'KOPS_VERSION_B': "latest", 2249 'K8S_VERSION_B': "v1.26.0", 2250 'KOPS_SKIP_E2E': '1', 2251 'KOPS_TEMPLATE': 'tests/e2e/templates/many-addons.yaml.tmpl', 2252 'KOPS_CONTROL_PLANE_SIZE': '3', 2253 } 2254 ), 2255 presubmit_test( 2256 name="pull-kops-e2e-aws-upgrade-k127-ko127-to-klatest-kolatest-many-addons", 2257 optional=True, 2258 distro='u2204', 2259 networking='cilium', 2260 k8s_version='stable', 2261 kops_channel='alpha', 2262 test_timeout_minutes=150, 2263 run_if_changed=r'^upup\/(models\/cloudup\/resources\/addons\/|pkg\/fi\/cloudup\/bootstrapchannelbuilder\/)', # pylint: disable=line-too-long 2264 scenario='upgrade-ab', 2265 env={ 2266 'KOPS_VERSION_A': "1.28", 2267 'K8S_VERSION_A': "v1.28.0", 2268 'KOPS_VERSION_B': "latest", 2269 'K8S_VERSION_B': "latest", 2270 'KOPS_SKIP_E2E': '1', 2271 'KOPS_TEMPLATE': 'tests/e2e/templates/many-addons.yaml.tmpl', 2272 'KOPS_CONTROL_PLANE_SIZE': '3', 2273 } 2274 ), 2275 presubmit_test( 2276 name="pull-kops-e2e-aws-upgrade-k127-ko127-to-k128-kolatest-karpenter", 2277 optional=True, 2278 distro='u2204arm64', 2279 networking='cilium', 2280 k8s_version='stable', 2281 kops_channel='alpha', 2282 test_timeout_minutes=150, 2283 run_if_changed=r'^upup\/models\/cloudup\/resources\/addons\/karpenter\.sh\/', 2284 scenario='upgrade-ab', 2285 extra_flags=[ 2286 "--instance-manager=karpenter", 2287 "--master-size=c6g.xlarge", 2288 ], 2289 env={ 2290 'KOPS_VERSION_A': "1.28", 2291 'K8S_VERSION_A': "v1.28.0", 2292 'KOPS_VERSION_B': "latest", 2293 'K8S_VERSION_B': "v1.29.0", 2294 'KOPS_SKIP_E2E': '1', 2295 'KOPS_CONTROL_PLANE_SIZE': '3', 2296 } 2297 ), 2298 presubmit_test( 2299 name='presubmit-kops-aws-boskos', 2300 scenario='aws-boskos', 2301 always_run=False, 2302 use_boskos=True, 2303 ), 2304 presubmit_test( 2305 name='presubmit-kops-aws-boskos-kubetest2', 2306 always_run=False, 2307 use_boskos=True, 2308 ), 2309 2310 presubmit_test( 2311 name="pull-kops-kubernetes-e2e-cos-gce-serial", 2312 cloud="gce", 2313 distro="cos105", 2314 networking="kubenet", 2315 k8s_version="ci", 2316 kops_channel="alpha", 2317 extra_flags=[ 2318 "--image=cos-cloud/cos-105-17412-156-49", 2319 "--node-volume-size=100", 2320 "--gce-service-account=default", 2321 ], 2322 build_cluster="k8s-infra-prow-build", 2323 focus_regex=r'\[Serial\]', 2324 skip_regex=r'\[Driver:.gcepd\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 2325 test_timeout_minutes=500, 2326 optional=True, 2327 test_parallelism=1, # serial tests 2328 ), 2329 2330 presubmit_test( 2331 name="pull-kops-kubernetes-e2e-cos-gce", 2332 cloud="gce", 2333 distro="cos105", 2334 networking="kubenet", 2335 k8s_version="ci", 2336 kops_channel="alpha", 2337 build_cluster="k8s-infra-prow-build", 2338 extra_flags=[ 2339 "--image=cos-cloud/cos-105-17412-156-49", 2340 "--node-volume-size=100", 2341 "--gce-service-account=default", 2342 ], 2343 skip_regex=r'\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 2344 test_timeout_minutes=40, 2345 optional=True, 2346 ), 2347 2348 presubmit_test( 2349 name="pull-kops-kubernetes-e2e-cos-gce-slow", 2350 cloud="gce", 2351 distro="cos105", 2352 networking="kubenet", 2353 k8s_version="ci", 2354 kops_channel="alpha", 2355 build_cluster="k8s-infra-prow-build", 2356 extra_flags=[ 2357 "--image=cos-cloud/cos-105-17412-156-49", 2358 "--set=spec.networking.networkID=default", 2359 "--gce-service-account=default", 2360 ], 2361 focus_regex=r'\[Slow\]', 2362 skip_regex=r'\[Driver:.gcepd\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]', # pylint: disable=line-too-long 2363 test_timeout_minutes=70, 2364 optional=True, 2365 ), 2366 ] 2367 2368 return jobs 2369 2370 ######################## 2371 # YAML File Generation # 2372 ######################## 2373 periodics_files = { 2374 'kops-periodics-conformance.yaml': generate_conformance, 2375 'kops-periodics-distros.yaml': generate_distros, 2376 'kops-periodics-grid.yaml': generate_grid, 2377 'kops-periodics-misc2.yaml': generate_misc, 2378 'kops-periodics-network-plugins.yaml': generate_network_plugins, 2379 'kops-periodics-upgrades.yaml': generate_upgrades, 2380 'kops-periodics-versions.yaml': generate_versions, 2381 'kops-periodics-pipeline.yaml': generate_pipeline, 2382 } 2383 2384 presubmits_files = { 2385 'kops-presubmits-distros.yaml':generate_presubmits_distros, 2386 'kops-presubmits-network-plugins.yaml': generate_presubmits_network_plugins, 2387 'kops-presubmits-e2e.yaml': generate_presubmits_e2e, 2388 'kops-presubmits-scale.yaml': generate_presubmits_scale, 2389 } 2390 2391 def main(): 2392 for filename, generate_func in periodics_files.items(): 2393 print(f"Generating {filename}") 2394 output = [] 2395 runs_per_week = 0 2396 job_count = 0 2397 for res in generate_func(): 2398 output.append(res[0]) 2399 runs_per_week += res[1] 2400 job_count += 1 2401 output.insert(0, "# Test jobs generated by build_jobs.py (do not manually edit)\n") 2402 output.insert(1, f"# {job_count} jobs, total of {runs_per_week} runs per week\n") 2403 output.insert(2, "periodics:\n") 2404 with open(filename, 'w') as fd: 2405 fd.write(''.join(output)) 2406 for filename, generate_func in presubmits_files.items(): 2407 print(f"Generating {filename}") 2408 output = [] 2409 job_count = 0 2410 for res in generate_func(): 2411 output.append(res) 2412 job_count += 1 2413 output.insert(0, "# Test jobs generated by build_jobs.py (do not manually edit)\n") 2414 output.insert(1, f"# {job_count} jobs\n") 2415 output.insert(2, "presubmits:\n") 2416 output.insert(3, " kubernetes/kops:\n") 2417 with open(filename, 'w') as fd: 2418 fd.write(''.join(output)) 2419 2420 if __name__ == "__main__": 2421 main()