github.com/nginxinc/kubernetes-ingress@v1.12.5/tests/suite/test_annotations.py (about) 1 import pytest 2 import yaml 3 from kubernetes.client import ExtensionsV1beta1Api 4 5 from suite.custom_assertions import assert_event_count_increased 6 from suite.fixtures import PublicEndpoint 7 from suite.resources_utils import ensure_connection_to_public_endpoint, \ 8 get_ingress_nginx_template_conf, \ 9 get_first_pod_name, create_example_app, wait_until_all_pods_are_ready, \ 10 delete_common_app, create_items_from_yaml, delete_items_from_yaml, \ 11 wait_before_test, replace_configmap_from_yaml, get_events, \ 12 generate_ingresses_with_annotation, replace_ingress 13 from suite.yaml_utils import get_first_ingress_host_from_yaml, get_name_from_yaml 14 from settings import TEST_DATA, DEPLOYMENTS 15 16 17 def get_event_count(event_text, events_list) -> int: 18 for i in range(len(events_list) - 1, -1, -1): 19 if event_text in events_list[i].message: 20 return events_list[i].count 21 return 0 22 23 24 def replace_ingresses_from_yaml(extensions_v1_beta1: ExtensionsV1beta1Api, namespace, yaml_manifest) -> None: 25 """ 26 Parse file and replace all Ingresses based on its contents. 27 28 :param extensions_v1_beta1: ExtensionsV1beta1Api 29 :param namespace: namespace 30 :param yaml_manifest: an absolute path to a file 31 :return: 32 """ 33 print(f"Replace an Ingresses from yaml") 34 with open(yaml_manifest) as f: 35 docs = yaml.safe_load_all(f) 36 for doc in docs: 37 if doc['kind'] == 'Ingress': 38 replace_ingress(extensions_v1_beta1, doc['metadata']['name'], namespace, doc) 39 40 41 def get_minions_info_from_yaml(file) -> []: 42 """ 43 Parse yaml file and return minions details. 44 45 :param file: an absolute path to file 46 :return: [{name, svc_name}] 47 """ 48 res = [] 49 with open(file) as f: 50 docs = yaml.safe_load_all(f) 51 for dep in docs: 52 if 'minion' in dep['metadata']['name']: 53 res.append({"name": dep['metadata']['name'], 54 "svc_name": dep['spec']['rules'][0]['http']['paths'][0]['backend']['serviceName']}) 55 return res 56 57 58 class AnnotationsSetup: 59 """Encapsulate Annotations example details. 60 61 Attributes: 62 public_endpoint: PublicEndpoint 63 ingress_name: 64 ingress_pod_name: 65 ingress_host: 66 namespace: example namespace 67 """ 68 def __init__(self, public_endpoint: PublicEndpoint, ingress_src_file, ingress_name, ingress_host, ingress_pod_name, 69 namespace, ingress_event_text, ingress_error_event_text, upstream_names=None): 70 self.public_endpoint = public_endpoint 71 self.ingress_name = ingress_name 72 self.ingress_pod_name = ingress_pod_name 73 self.namespace = namespace 74 self.ingress_host = ingress_host 75 self.ingress_src_file = ingress_src_file 76 self.ingress_event_text = ingress_event_text 77 self.ingress_error_event_text = ingress_error_event_text 78 self.upstream_names = upstream_names 79 80 81 @pytest.fixture(scope="class") 82 def annotations_setup(request, 83 kube_apis, 84 ingress_controller_prerequisites, 85 ingress_controller_endpoint, ingress_controller, test_namespace) -> AnnotationsSetup: 86 print("------------------------- Deploy Annotations-Example -----------------------------------") 87 create_items_from_yaml(kube_apis, 88 f"{TEST_DATA}/annotations/{request.param}/annotations-ingress.yaml", 89 test_namespace) 90 ingress_name = get_name_from_yaml(f"{TEST_DATA}/annotations/{request.param}/annotations-ingress.yaml") 91 ingress_host = get_first_ingress_host_from_yaml(f"{TEST_DATA}/annotations/{request.param}/annotations-ingress.yaml") 92 if request.param == 'mergeable': 93 minions_info = get_minions_info_from_yaml(f"{TEST_DATA}/annotations/{request.param}/annotations-ingress.yaml") 94 else: 95 minions_info = None 96 create_example_app(kube_apis, "simple", test_namespace) 97 wait_until_all_pods_are_ready(kube_apis.v1, test_namespace) 98 ensure_connection_to_public_endpoint(ingress_controller_endpoint.public_ip, 99 ingress_controller_endpoint.port, 100 ingress_controller_endpoint.port_ssl) 101 ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace) 102 upstream_names = [] 103 if request.param == 'mergeable': 104 event_text = f"Configuration for {test_namespace}/{ingress_name} was added or updated" 105 error_text = f"{test_namespace}/{ingress_name} was rejected: with error" 106 for minion in minions_info: 107 upstream_names.append(f"{test_namespace}-{minion['name']}-{ingress_host}-{minion['svc_name']}-80") 108 else: 109 event_text = f"Configuration for {test_namespace}/{ingress_name} was added or updated" 110 error_text = f"{test_namespace}/{ingress_name} was rejected: with error" 111 upstream_names.append(f"{test_namespace}-{ingress_name}-{ingress_host}-backend1-svc-80") 112 upstream_names.append(f"{test_namespace}-{ingress_name}-{ingress_host}-backend2-svc-80") 113 114 def fin(): 115 print("Clean up Annotations Example:") 116 replace_configmap_from_yaml(kube_apis.v1, 117 ingress_controller_prerequisites.config_map['metadata']['name'], 118 ingress_controller_prerequisites.namespace, 119 f"{DEPLOYMENTS}/common/nginx-config.yaml") 120 delete_common_app(kube_apis, "simple", test_namespace) 121 delete_items_from_yaml(kube_apis, 122 f"{TEST_DATA}/annotations/{request.param}/annotations-ingress.yaml", 123 test_namespace) 124 125 request.addfinalizer(fin) 126 127 return AnnotationsSetup(ingress_controller_endpoint, 128 f"{TEST_DATA}/annotations/{request.param}/annotations-ingress.yaml", 129 ingress_name, ingress_host, ic_pod_name, test_namespace, event_text, error_text, 130 upstream_names) 131 132 133 @pytest.fixture(scope="class") 134 def annotations_grpc_setup(request, 135 kube_apis, 136 ingress_controller_prerequisites, 137 ingress_controller_endpoint, ingress_controller, test_namespace) -> AnnotationsSetup: 138 print("------------------------- Deploy gRPC Annotations-Example -----------------------------------") 139 create_items_from_yaml(kube_apis, 140 f"{TEST_DATA}/annotations/grpc/annotations-ingress.yaml", 141 test_namespace) 142 ingress_name = get_name_from_yaml(f"{TEST_DATA}/annotations/grpc/annotations-ingress.yaml") 143 ingress_host = get_first_ingress_host_from_yaml(f"{TEST_DATA}/annotations/grpc/annotations-ingress.yaml") 144 replace_configmap_from_yaml(kube_apis.v1, 145 ingress_controller_prerequisites.config_map['metadata']['name'], 146 ingress_controller_prerequisites.namespace, 147 f"{TEST_DATA}/common/configmap-with-grpc.yaml") 148 ic_pod_name = get_first_pod_name(kube_apis.v1, ingress_controller_prerequisites.namespace) 149 event_text = f"Configuration for {test_namespace}/{ingress_name} was added or updated" 150 error_text = f"{event_text} ; but was not applied: Error reloading NGINX" 151 152 def fin(): 153 print("Clean up gRPC Annotations Example:") 154 delete_items_from_yaml(kube_apis, 155 f"{TEST_DATA}/annotations/grpc/annotations-ingress.yaml", 156 test_namespace) 157 158 request.addfinalizer(fin) 159 160 return AnnotationsSetup(ingress_controller_endpoint, 161 f"{TEST_DATA}/annotations/grpc/annotations-ingress.yaml", 162 ingress_name, ingress_host, ic_pod_name, test_namespace, event_text, error_text) 163 164 165 @pytest.mark.ingresses 166 @pytest.mark.parametrize('annotations_setup', ["standard", "mergeable"], indirect=True) 167 class TestAnnotations: 168 def test_nginx_config_defaults(self, kube_apis, annotations_setup, ingress_controller_prerequisites): 169 print("Case 1: no ConfigMap keys, no annotations in Ingress") 170 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 171 annotations_setup.namespace, 172 annotations_setup.ingress_name, 173 annotations_setup.ingress_pod_name, 174 ingress_controller_prerequisites.namespace) 175 176 assert "proxy_send_timeout 60s;" in result_conf 177 assert "max_conns=0;" in result_conf 178 179 assert "Strict-Transport-Security" not in result_conf 180 181 for upstream in annotations_setup.upstream_names: 182 assert f"zone {upstream} 256k;" in result_conf 183 184 @pytest.mark.parametrize('annotations, expected_strings, unexpected_strings', [ 185 ({"nginx.org/proxy-send-timeout": "10s", "nginx.org/max-conns": "1024", 186 "nginx.org/hsts": "True", "nginx.org/hsts-behind-proxy": "True", 187 "nginx.org/upstream-zone-size": "124k"}, 188 ["proxy_send_timeout 10s;", "max_conns=1024", 189 'set $hsts_header_val "";', "proxy_hide_header Strict-Transport-Security;", 190 'add_header Strict-Transport-Security "$hsts_header_val" always;', 191 "if ($http_x_forwarded_proto = 'https')", 'set $hsts_header_val "max-age=2592000; preload";', 192 " 124k;"], 193 ["proxy_send_timeout 60s;", "if ($https = on)", 194 " 256k;"]) 195 ]) 196 def test_when_annotation_in_ing_only(self, kube_apis, annotations_setup, ingress_controller_prerequisites, 197 annotations, expected_strings, unexpected_strings): 198 initial_events = get_events(kube_apis.v1, annotations_setup.namespace) 199 initial_count = get_event_count(annotations_setup.ingress_event_text, initial_events) 200 print("Case 2: no ConfigMap keys, annotations in Ingress only") 201 new_ing = generate_ingresses_with_annotation(annotations_setup.ingress_src_file, 202 annotations) 203 for ing in new_ing: 204 # in mergeable case this will update master ingress only 205 if ing['metadata']['name'] == annotations_setup.ingress_name: 206 replace_ingress(kube_apis.extensions_v1_beta1, 207 annotations_setup.ingress_name, annotations_setup.namespace, ing) 208 wait_before_test(1) 209 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 210 annotations_setup.namespace, 211 annotations_setup.ingress_name, 212 annotations_setup.ingress_pod_name, 213 ingress_controller_prerequisites.namespace) 214 new_events = get_events(kube_apis.v1, annotations_setup.namespace) 215 216 assert_event_count_increased(annotations_setup.ingress_event_text, initial_count, new_events) 217 for _ in expected_strings: 218 assert _ in result_conf 219 for _ in unexpected_strings: 220 assert _ not in result_conf 221 222 @pytest.mark.parametrize('configmap_file, expected_strings, unexpected_strings', [ 223 (f"{TEST_DATA}/annotations/configmap-with-keys.yaml", 224 ["proxy_send_timeout 33s;", 225 'set $hsts_header_val "";', "proxy_hide_header Strict-Transport-Security;", 226 'add_header Strict-Transport-Security "$hsts_header_val" always;', 227 "if ($http_x_forwarded_proto = 'https')", 'set $hsts_header_val "max-age=2592000; preload";', 228 " 100k;"], 229 ["proxy_send_timeout 60s;", "if ($https = on)", 230 " 256k;"]), 231 ]) 232 def test_when_annotation_in_configmap_only(self, kube_apis, annotations_setup, ingress_controller_prerequisites, 233 configmap_file, expected_strings, unexpected_strings): 234 initial_events = get_events(kube_apis.v1, annotations_setup.namespace) 235 initial_count = get_event_count(annotations_setup.ingress_event_text, initial_events) 236 print("Case 3: keys in ConfigMap, no annotations in Ingress") 237 replace_ingresses_from_yaml(kube_apis.extensions_v1_beta1, annotations_setup.namespace, 238 annotations_setup.ingress_src_file) 239 replace_configmap_from_yaml(kube_apis.v1, 240 ingress_controller_prerequisites.config_map['metadata']['name'], 241 ingress_controller_prerequisites.namespace, 242 configmap_file) 243 wait_before_test(1) 244 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 245 annotations_setup.namespace, 246 annotations_setup.ingress_name, 247 annotations_setup.ingress_pod_name, 248 ingress_controller_prerequisites.namespace) 249 new_events = get_events(kube_apis.v1, annotations_setup.namespace) 250 251 assert_event_count_increased(annotations_setup.ingress_event_text, initial_count, new_events) 252 for _ in expected_strings: 253 assert _ in result_conf 254 for _ in unexpected_strings: 255 assert _ not in result_conf 256 257 @pytest.mark.parametrize('annotations, configmap_file, expected_strings, unexpected_strings', [ 258 ({"nginx.org/proxy-send-timeout": "10s", 259 "nginx.org/hsts": "False", "nginx.org/hsts-behind-proxy": "False", 260 "nginx.org/upstream-zone-size": "124k"}, 261 f"{TEST_DATA}/annotations/configmap-with-keys.yaml", 262 ["proxy_send_timeout 10s;", " 124k;"], 263 ["proxy_send_timeout 33s;", "Strict-Transport-Security", " 100k;", " 256k;"]), 264 ]) 265 def test_ing_overrides_configmap(self, kube_apis, annotations_setup, ingress_controller_prerequisites, 266 annotations, configmap_file, expected_strings, unexpected_strings): 267 initial_events = get_events(kube_apis.v1, annotations_setup.namespace) 268 initial_count = get_event_count(annotations_setup.ingress_event_text, initial_events) 269 print("Case 4: keys in ConfigMap, annotations in Ingress") 270 new_ing = generate_ingresses_with_annotation(annotations_setup.ingress_src_file, 271 annotations) 272 for ing in new_ing: 273 # in mergeable case this will update master ingress only 274 if ing['metadata']['name'] == annotations_setup.ingress_name: 275 replace_ingress(kube_apis.extensions_v1_beta1, 276 annotations_setup.ingress_name, annotations_setup.namespace, ing) 277 replace_configmap_from_yaml(kube_apis.v1, 278 ingress_controller_prerequisites.config_map['metadata']['name'], 279 ingress_controller_prerequisites.namespace, 280 configmap_file) 281 wait_before_test(1) 282 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 283 annotations_setup.namespace, 284 annotations_setup.ingress_name, 285 annotations_setup.ingress_pod_name, 286 ingress_controller_prerequisites.namespace) 287 new_events = get_events(kube_apis.v1, annotations_setup.namespace) 288 289 assert_event_count_increased(annotations_setup.ingress_event_text, initial_count, new_events) 290 for _ in expected_strings: 291 assert _ in result_conf 292 for _ in unexpected_strings: 293 assert _ not in result_conf 294 295 @pytest.mark.parametrize('annotations', [ 296 ({"nginx.org/upstream-zone-size": "0"}), 297 ]) 298 def test_upstream_zone_size_0(self, cli_arguments, kube_apis, 299 annotations_setup, ingress_controller_prerequisites, annotations): 300 initial_events = get_events(kube_apis.v1, annotations_setup.namespace) 301 initial_count = get_event_count(annotations_setup.ingress_event_text, initial_events) 302 print("Edge Case: upstream-zone-size is 0") 303 new_ing = generate_ingresses_with_annotation(annotations_setup.ingress_src_file, annotations) 304 for ing in new_ing: 305 # in mergeable case this will update master ingress only 306 if ing['metadata']['name'] == annotations_setup.ingress_name: 307 replace_ingress(kube_apis.extensions_v1_beta1, 308 annotations_setup.ingress_name, annotations_setup.namespace, ing) 309 wait_before_test(1) 310 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 311 annotations_setup.namespace, 312 annotations_setup.ingress_name, 313 annotations_setup.ingress_pod_name, 314 ingress_controller_prerequisites.namespace) 315 new_events = get_events(kube_apis.v1, annotations_setup.namespace) 316 317 assert_event_count_increased(annotations_setup.ingress_event_text, initial_count, new_events) 318 if cli_arguments["ic-type"] == "nginx-plus-ingress": 319 print("Run assertions for Nginx Plus case") 320 assert "zone " in result_conf 321 assert " 256k;" in result_conf 322 elif cli_arguments["ic-type"] == "nginx-ingress": 323 print("Run assertions for Nginx OSS case") 324 assert "zone " not in result_conf 325 assert " 256k;" not in result_conf 326 327 @pytest.mark.parametrize('annotations', [ 328 ({"nginx.org/proxy-send-timeout": "invalid", "nginx.org/max-conns": "-10", 329 "nginx.org/upstream-zone-size": "-10I'm S±!@£$%^&*()invalid"}) 330 ]) 331 def test_validation(self, kube_apis, annotations_setup, ingress_controller_prerequisites, 332 annotations): 333 initial_events = get_events(kube_apis.v1, annotations_setup.namespace) 334 print("Case 6: IC doesn't validate, only nginx validates") 335 initial_count = get_event_count(annotations_setup.ingress_error_event_text, initial_events) 336 new_ing = generate_ingresses_with_annotation(annotations_setup.ingress_src_file, 337 annotations) 338 for ing in new_ing: 339 # in mergeable case this will update master ingress only 340 if ing['metadata']['name'] == annotations_setup.ingress_name: 341 replace_ingress(kube_apis.extensions_v1_beta1, 342 annotations_setup.ingress_name, annotations_setup.namespace, ing) 343 wait_before_test() 344 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 345 annotations_setup.namespace, 346 annotations_setup.ingress_name, 347 annotations_setup.ingress_pod_name, 348 ingress_controller_prerequisites.namespace) 349 new_events = get_events(kube_apis.v1, annotations_setup.namespace) 350 assert "server {" not in result_conf 351 assert "No such file or directory" in result_conf 352 assert_event_count_increased(annotations_setup.ingress_error_event_text, initial_count, new_events) 353 354 355 @pytest.mark.ingresses 356 @pytest.mark.parametrize('annotations_setup', ["mergeable"], indirect=True) 357 class TestMergeableFlows: 358 @pytest.mark.parametrize('yaml_file, expected_strings, unexpected_strings', [ 359 (f"{TEST_DATA}/annotations/mergeable/minion-annotations-differ.yaml", 360 ["proxy_send_timeout 25s;", "proxy_send_timeout 33s;", "max_conns=1048;", "max_conns=1024;"], 361 ["proxy_send_timeout 10s;", "max_conns=108;"]), 362 ]) 363 def test_minion_overrides_master(self, kube_apis, annotations_setup, ingress_controller_prerequisites, 364 yaml_file, expected_strings, unexpected_strings): 365 initial_events = get_events(kube_apis.v1, annotations_setup.namespace) 366 initial_count = get_event_count(annotations_setup.ingress_event_text, initial_events) 367 print("Case 7: minion annotation overrides master") 368 replace_ingresses_from_yaml(kube_apis.extensions_v1_beta1, annotations_setup.namespace, yaml_file) 369 wait_before_test(1) 370 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 371 annotations_setup.namespace, 372 annotations_setup.ingress_name, 373 annotations_setup.ingress_pod_name, 374 ingress_controller_prerequisites.namespace) 375 new_events = get_events(kube_apis.v1, annotations_setup.namespace) 376 377 assert_event_count_increased(annotations_setup.ingress_event_text, initial_count, new_events) 378 for _ in expected_strings: 379 assert _ in result_conf 380 for _ in unexpected_strings: 381 assert _ not in result_conf 382 383 384 @pytest.mark.ingresses 385 class TestGrpcFlows: 386 @pytest.mark.parametrize('annotations, expected_strings, unexpected_strings', [ 387 ({"nginx.org/proxy-send-timeout": "10s"}, ["grpc_send_timeout 10s;"], ["proxy_send_timeout 60s;"]), 388 ]) 389 def test_grpc_flow(self, kube_apis, annotations_grpc_setup, ingress_controller_prerequisites, 390 annotations, expected_strings, unexpected_strings): 391 initial_events = get_events(kube_apis.v1, annotations_grpc_setup.namespace) 392 initial_count = get_event_count(annotations_grpc_setup.ingress_event_text, initial_events) 393 print("Case 5: grpc annotations override http ones") 394 new_ing = generate_ingresses_with_annotation(annotations_grpc_setup.ingress_src_file, 395 annotations) 396 for ing in new_ing: 397 if ing['metadata']['name'] == annotations_grpc_setup.ingress_name: 398 replace_ingress(kube_apis.extensions_v1_beta1, 399 annotations_grpc_setup.ingress_name, annotations_grpc_setup.namespace, ing) 400 wait_before_test(1) 401 result_conf = get_ingress_nginx_template_conf(kube_apis.v1, 402 annotations_grpc_setup.namespace, 403 annotations_grpc_setup.ingress_name, 404 annotations_grpc_setup.ingress_pod_name, 405 ingress_controller_prerequisites.namespace) 406 new_events = get_events(kube_apis.v1, annotations_grpc_setup.namespace) 407 408 assert_event_count_increased(annotations_grpc_setup.ingress_event_text, initial_count, new_events) 409 for _ in expected_strings: 410 assert _ in result_conf 411 for _ in unexpected_strings: 412 assert _ not in result_conf