k8s.io/perf-tests/clusterloader2@v0.0.0-20240304094227-64bdb12da87e/testing/experimental/storage/pod-startup/config.yaml (about) 1 # ASSUMPTIONS: 2 # - Number of nodes should be divisible by NODES_PER_NAMESPACE (default 100). 3 # - If using Persistent Volumes, the default storage class must have volumeBindingMode: Immediate 4 5 # Cluster Variables 6 # Cater for the case where the number of nodes is less than nodes per namespace. See https://github.com/kubernetes/perf-tests/issues/887 7 {{$NODES_PER_NAMESPACE := MinInt .Nodes (DefaultParam .NODES_PER_NAMESPACE 100)}} 8 9 # Test Variales 10 {{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 1}} 11 {{$DEPLOYMENT_TEMPLATE_PATH := .DEPLOYMENT_TEMPLATE_PATH }} 12 {{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 1}} 13 {{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}} 14 # Set this to false if metrics data is not needed. 15 {{$GATHER_METRICS := DefaultParam .GATHER_METRICS true}} 16 {{$START_PODS := DefaultParam .START_PODS true}} 17 {{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}} 18 {{$STORAGE_CLASS := DefaultParam .STORAGE_CLASS "csi-gce-pd"}} 19 # Shortcut for provisioning volumes with some external provisioner. Without it, 20 # the PV controller must update each PVC before the external provisioner 21 # starts to work on it, which limits the volume creation rate. 22 # With it, volume creation starts immediately. This is best 23 # used together with WAIT_FOR_PVS_CREATED=true and 24 # WAIT_FOR_PVS_BOUND=false because then test does not to wait for the 25 # PV controller during volume creation at all. 26 {{$PROVISIONER := DefaultParam .PROVISIONER ""}} 27 # When checking PVs, the test by default considers *all* PVs in the cluster, 28 # whether they were created by the test or not. In other words, the test 29 # only works in clusters with no PVs. To make it a bit more flexible, 30 # the value of the expected provisioner can be set here and then only 31 # PVs with that annotation will be counted. 32 {{$EXPECTED_PROVISIONER := DefaultParam .EXPECTED_PROVISIONER ""}} 33 {{$VOL_SIZE := DefaultParam .VOL_SIZE "8Gi"}} 34 {{$WAIT_FOR_PVS_CREATED := DefaultParam .WAIT_FOR_PVS_CREATED false}} 35 {{$WAIT_FOR_PVS_BOUND := DefaultParam .WAIT_FOR_PVS_BOUND false}} 36 {{$WAIT_FOR_PVS_DELETED := DefaultParam .WAIT_FOR_PVS_DELETED false}} 37 {{$POD_THROUGHPUT := DefaultParam .POD_THROUGHPUT 10}} 38 # TODO(hantaowang): remove knob after deciding on right values 39 {{$POD_STARTUP_TIMEOUT := DefaultParam .POD_STARTUP_TIMEOUT "15m"}} 40 {{$POD_STARTUP_SLO := DefaultParam .POD_STARTUP_SLO 300}} 41 42 # Computed Variables 43 {{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE | MaxInt 1}} 44 {{$podsPerNamespace := MultiplyInt $NODES_PER_NAMESPACE $PODS_PER_NODE}} 45 {{$volumesPerNamespace := MultiplyInt $podsPerNamespace $VOLUMES_PER_POD}} 46 {{$totalVols := MultiplyInt $volumesPerNamespace $namespaces}} 47 {{$guessedStepTime := MultiplyInt $totalVols 2 | MaxInt 60}} 48 49 # This is a very conservative estimate of 2 seconds per volume. 50 {{$StepTimeSeconds := DefaultParam .STEP_TIME_SECONDS $guessedStepTime}} 51 52 53 name: storage 54 namespace: 55 number: {{$namespaces}} 56 tuningSets: 57 - name: UniformQPS 58 qpsLoad: 59 qps: {{$POD_THROUGHPUT}} 60 steps: 61 {{ if $GATHER_METRICS }} 62 # Start measurements 63 - name: Starting measurement for the entire test 64 measurements: 65 - Identifier: APIResponsivenessPrometheus 66 Method: APIResponsivenessPrometheus 67 Params: 68 action: start 69 - Identifier: APIResponsivenessPrometheusSimple 70 Method: APIResponsivenessPrometheus 71 Params: 72 action: start 73 - Identifier: PodWithVolumesStartupLatency 74 Method: PodStartupLatency 75 Params: 76 action: start 77 labelSelector: group = volume-test 78 threshold: {{$POD_STARTUP_SLO}}s 79 {{ end }} 80 {{ if $PROVISION_VOLUME }} 81 # Provision volumes 82 - name: Provisioning volumes 83 phases: 84 - namespaceRange: 85 min: 1 86 max: {{$namespaces}} 87 replicasPerNamespace: {{$volumesPerNamespace}} 88 tuningSet: UniformQPS 89 objectBundle: 90 - basename: vol 91 objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}} 92 templateFillMap: 93 Group: volume-test 94 VolSize: {{$VOL_SIZE}} 95 StorageClass: {{$STORAGE_CLASS}} 96 Provisioner: {{$PROVISIONER}} 97 {{ end }} 98 {{if $WAIT_FOR_PVS_CREATED }} 99 - name: Waiting for volume creation 100 measurements: 101 - Identifier: WaitForPVsToBeCreated 102 Method: WaitForAvailablePVs 103 Params: 104 desiredPVCount: {{$totalVols}} 105 apiVersion: v1 106 provisioner: {{$EXPECTED_PROVISIONER}} 107 timeout: {{$StepTimeSeconds}}s 108 {{ end }} 109 {{ if $WAIT_FOR_PVS_BOUND }} 110 - name: Waiting for PVs to be bound 111 measurements: 112 - Identifier: WaitForPVCsToBeBound 113 Method: WaitForBoundPVCs 114 Params: 115 desiredPVCCount: {{$totalVols}} 116 apiVersion: v1 117 labelSelector: group = volume-test 118 timeout: {{$StepTimeSeconds}}s 119 {{ end }} 120 {{ if $START_PODS }} 121 - name: Starting measurement for waiting for deployments 122 measurements: 123 - Identifier: WaitForRunningDeployments 124 Method: WaitForControlledPodsRunning 125 Params: 126 action: start 127 apiVersion: apps/v1 128 kind: Deployment 129 labelSelector: group = volume-test 130 operationTimeout: {{$POD_STARTUP_TIMEOUT}} 131 # Create deployments 132 - name: Creating deployments 133 phases: 134 - namespaceRange: 135 min: 1 136 max: {{$namespaces}} 137 replicasPerNamespace: {{$podsPerNamespace}} 138 tuningSet: UniformQPS 139 objectBundle: 140 - basename: deployment 141 objectTemplatePath: {{$DEPLOYMENT_TEMPLATE_PATH}} 142 templateFillMap: 143 Group: volume-test 144 VolumesPerPod: {{$VOLUMES_PER_POD}} 145 VolSize: {{$VOL_SIZE}} 146 StorageClass: {{$STORAGE_CLASS}} 147 Provisioner: {{$PROVISIONER}} 148 - name: Waiting for deployments to be running 149 measurements: 150 - Identifier: WaitForRunningDeployments 151 Method: WaitForControlledPodsRunning 152 Params: 153 action: gather 154 # Delete deployments 155 - name: Deleting deployments 156 phases: 157 - namespaceRange: 158 min: 1 159 max: {{$namespaces}} 160 replicasPerNamespace: 0 161 tuningSet: UniformQPS 162 objectBundle: 163 - basename: deployment 164 objectTemplatePath: {{$DEPLOYMENT_TEMPLATE_PATH}} 165 {{ end }} 166 {{ if $PROVISION_VOLUME }} 167 # Delete volumes 168 - name: Deleting volumes 169 phases: 170 - namespaceRange: 171 min: 1 172 max: {{$namespaces}} 173 replicasPerNamespace: 0 174 tuningSet: UniformQPS 175 objectBundle: 176 - basename: vol 177 objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}} 178 {{ end }} 179 {{if $WAIT_FOR_PVS_DELETED }} 180 - name: Waiting for volume deletion 181 measurements: 182 - Identifier: WaitForPVsToBeDeleted 183 Method: WaitForAvailablePVs 184 Params: 185 desiredPVCount: 0 186 apiVersion: v1 187 provisioner: {{$EXPECTED_PROVISIONER}} 188 timeout: {{$StepTimeSeconds}}s 189 {{ end }} 190 {{ if $GATHER_METRICS }} 191 # Collect measurements 192 - name: Gather test measurements 193 measurements: 194 - Identifier: APIResponsivenessPrometheusSimple 195 Method: APIResponsivenessPrometheus 196 Params: 197 action: gather 198 enableViolations: true 199 useSimpleLatencyQuery: true 200 summaryName: APIResponsivenessPrometheus_simple 201 - Identifier: APIResponsivenessPrometheus 202 Method: APIResponsivenessPrometheus 203 Params: 204 action: gather 205 - Identifier: PodWithVolumesStartupLatency 206 Method: PodStartupLatency 207 Params: 208 action: gather 209 {{ end }}