github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/options/pipeline_options_validator_test.py (about) 1 # 2 # Licensed to the Apache Software Foundation (ASF) under one or more 3 # contributor license agreements. See the NOTICE file distributed with 4 # this work for additional information regarding copyright ownership. 5 # The ASF licenses this file to You under the Apache License, Version 2.0 6 # (the "License"); you may not use this file except in compliance with 7 # the License. You may obtain a copy of the License at 8 # 9 # http://www.apache.org/licenses/LICENSE-2.0 10 # 11 # Unless required by applicable law or agreed to in writing, software 12 # distributed under the License is distributed on an "AS IS" BASIS, 13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 # See the License for the specific language governing permissions and 15 # limitations under the License. 16 # 17 18 """Unit tests for the pipeline options validator module.""" 19 20 # pytype: skip-file 21 22 import logging 23 import unittest 24 25 from hamcrest import assert_that 26 from hamcrest import contains_string 27 from hamcrest import only_contains 28 from hamcrest.core.base_matcher import BaseMatcher 29 30 from apache_beam.internal import pickler 31 from apache_beam.options.pipeline_options import DebugOptions 32 from apache_beam.options.pipeline_options import GoogleCloudOptions 33 from apache_beam.options.pipeline_options import PipelineOptions 34 from apache_beam.options.pipeline_options import WorkerOptions 35 from apache_beam.options.pipeline_options_validator import PipelineOptionsValidator 36 37 38 # Mock runners to use for validations. 39 class MockRunners(object): 40 class DataflowRunner(object): 41 def get_default_gcp_region(self): 42 # Return a default so we don't have to specify --region in every test 43 # (unless specifically testing it). 44 return 'us-central1' 45 46 class TestDataflowRunner(DataflowRunner): 47 pass 48 49 class OtherRunner(object): 50 pass 51 52 53 # Matcher that always passes for testing on_success_matcher option 54 class AlwaysPassMatcher(BaseMatcher): 55 def _matches(self, item): 56 return True 57 58 59 class SetupTest(unittest.TestCase): 60 def check_errors_for_arguments(self, errors, args): 61 """Checks that there is exactly one error for each given argument.""" 62 missing = [] 63 remaining = list(errors) 64 65 for arg in args: 66 found = False 67 for error in remaining: 68 if arg in error: 69 remaining.remove(error) 70 found = True 71 break 72 if not found: 73 missing.append('Missing error for: %s.' % arg) 74 75 # Return missing and remaining (not matched) errors. 76 return missing + remaining 77 78 def test_local_runner(self): 79 runner = MockRunners.OtherRunner() 80 options = PipelineOptions([]) 81 validator = PipelineOptionsValidator(options, runner) 82 errors = validator.validate() 83 self.assertEqual(len(errors), 0) 84 85 def test_missing_required_options(self): 86 options = PipelineOptions(['']) 87 runner = MockRunners.DataflowRunner() 88 # Remove default region for this test. 89 runner.get_default_gcp_region = lambda: None 90 validator = PipelineOptionsValidator(options, runner) 91 errors = validator.validate() 92 93 self.assertEqual( 94 self.check_errors_for_arguments( 95 errors, ['project', 'staging_location', 'temp_location', 'region']), 96 []) 97 98 def test_gcs_path(self): 99 def get_validator(temp_location, staging_location): 100 options = ['--project=example:example', '--job_name=job'] 101 102 if temp_location is not None: 103 options.append('--temp_location=' + temp_location) 104 105 if staging_location is not None: 106 options.append('--staging_location=' + staging_location) 107 108 pipeline_options = PipelineOptions(options) 109 runner = MockRunners.DataflowRunner() 110 validator = PipelineOptionsValidator(pipeline_options, runner) 111 return validator 112 113 test_cases = [ 114 { 115 'temp_location': None, 116 'staging_location': 'gs://foo/bar', 117 'errors': ['temp_location'] 118 }, 119 { 120 'temp_location': None, 121 'staging_location': None, 122 'errors': ['staging_location', 'temp_location'] 123 }, 124 { 125 'temp_location': 'gs://foo/bar', 126 'staging_location': None, 127 'errors': [] 128 }, 129 { 130 'temp_location': 'gs://foo/bar', 131 'staging_location': 'gs://ABC/bar', 132 'errors': ['staging_location'] 133 }, 134 { 135 'temp_location': 'gcs:/foo/bar', 136 'staging_location': 'gs://foo/bar', 137 'errors': ['temp_location'] 138 }, 139 { 140 'temp_location': 'gs:/foo/bar', 141 'staging_location': 'gs://foo/bar', 142 'errors': ['temp_location'] 143 }, 144 { 145 'temp_location': 'gs://ABC/bar', 146 'staging_location': 'gs://foo/bar', 147 'errors': ['temp_location'] 148 }, 149 { 150 'temp_location': 'gs://ABC/bar', 151 'staging_location': 'gs://foo/bar', 152 'errors': ['temp_location'] 153 }, 154 { 155 'temp_location': 'gs://foo', 156 'staging_location': 'gs://foo/bar', 157 'errors': ['temp_location'] 158 }, 159 { 160 'temp_location': 'gs://foo/', 161 'staging_location': 'gs://foo/bar', 162 'errors': [] 163 }, 164 { 165 'temp_location': 'gs://foo/bar', 166 'staging_location': 'gs://foo/bar', 167 'errors': [] 168 }, 169 ] 170 171 for case in test_cases: 172 errors = get_validator(case['temp_location'], 173 case['staging_location']).validate() 174 self.assertEqual( 175 self.check_errors_for_arguments(errors, case['errors']), []) 176 177 def test_project(self): 178 def get_validator(project): 179 options = [ 180 '--job_name=job', 181 '--staging_location=gs://foo/bar', 182 '--temp_location=gs://foo/bar' 183 ] 184 185 if project is not None: 186 options.append('--project=' + project) 187 188 pipeline_options = PipelineOptions(options) 189 runner = MockRunners.DataflowRunner() 190 validator = PipelineOptionsValidator(pipeline_options, runner) 191 return validator 192 193 test_cases = [ 194 { 195 'project': None, 'errors': ['project'] 196 }, 197 { 198 'project': '12345', 'errors': ['project'] 199 }, 200 { 201 'project': 'FOO', 'errors': ['project'] 202 }, 203 { 204 'project': 'foo:BAR', 'errors': ['project'] 205 }, 206 { 207 'project': 'fo', 'errors': ['project'] 208 }, 209 { 210 'project': 'foo', 'errors': [] 211 }, 212 { 213 'project': 'foo:bar', 'errors': [] 214 }, 215 ] 216 217 for case in test_cases: 218 errors = get_validator(case['project']).validate() 219 self.assertEqual( 220 self.check_errors_for_arguments(errors, case['errors']), []) 221 222 def test_job_name(self): 223 def get_validator(job_name): 224 options = [ 225 '--project=example:example', 226 '--staging_location=gs://foo/bar', 227 '--temp_location=gs://foo/bar' 228 ] 229 230 if job_name is not None: 231 options.append('--job_name=' + job_name) 232 233 pipeline_options = PipelineOptions(options) 234 runner = MockRunners.DataflowRunner() 235 validator = PipelineOptionsValidator(pipeline_options, runner) 236 return validator 237 238 test_cases = [ 239 { 240 'job_name': None, 'errors': [] 241 }, 242 { 243 'job_name': '12345', 'errors': ['job_name'] 244 }, 245 { 246 'job_name': 'FOO', 'errors': ['job_name'] 247 }, 248 { 249 'job_name': 'foo:bar', 'errors': ['job_name'] 250 }, 251 { 252 'job_name': 'fo', 'errors': [] 253 }, 254 { 255 'job_name': 'foo', 'errors': [] 256 }, 257 ] 258 259 for case in test_cases: 260 errors = get_validator(case['job_name']).validate() 261 self.assertEqual( 262 self.check_errors_for_arguments(errors, case['errors']), []) 263 264 def test_num_workers(self): 265 def get_validator(num_workers): 266 options = [ 267 '--project=example:example', 268 '--job_name=job', 269 '--staging_location=gs://foo/bar', 270 '--temp_location=gs://foo/bar' 271 ] 272 273 if num_workers is not None: 274 options.append('--num_workers=' + num_workers) 275 276 pipeline_options = PipelineOptions(options) 277 runner = MockRunners.DataflowRunner() 278 validator = PipelineOptionsValidator(pipeline_options, runner) 279 return validator 280 281 test_cases = [ 282 { 283 'num_workers': None, 'errors': [] 284 }, 285 { 286 'num_workers': '1', 'errors': [] 287 }, 288 { 289 'num_workers': '0', 'errors': ['num_workers'] 290 }, 291 { 292 'num_workers': '-1', 'errors': ['num_workers'] 293 }, 294 ] 295 296 for case in test_cases: 297 errors = get_validator(case['num_workers']).validate() 298 self.assertEqual( 299 self.check_errors_for_arguments(errors, case['errors']), []) 300 301 def test_is_service_runner(self): 302 test_cases = [ 303 { 304 'runner': MockRunners.OtherRunner(), 305 'options': [], 306 'expected': False, 307 }, 308 { 309 'runner': MockRunners.OtherRunner(), 310 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com'], 311 'expected': False, 312 }, 313 { 314 'runner': MockRunners.OtherRunner(), 315 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com/'], 316 'expected': False, 317 }, 318 { 319 'runner': MockRunners.DataflowRunner(), 320 'options': ['--dataflow_endpoint=https://another.service.com'], 321 'expected': False, 322 }, 323 { 324 'runner': MockRunners.DataflowRunner(), 325 'options': ['--dataflow_endpoint=https://another.service.com/'], 326 'expected': False, 327 }, 328 { 329 'runner': MockRunners.DataflowRunner(), 330 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com'], 331 'expected': True, 332 }, 333 { 334 'runner': MockRunners.DataflowRunner(), 335 'options': ['--dataflow_endpoint=https://dataflow.googleapis.com/'], 336 'expected': True, 337 }, 338 { 339 'runner': MockRunners.DataflowRunner(), 340 'options': [], 341 'expected': True, 342 }, 343 ] 344 345 for case in test_cases: 346 validator = PipelineOptionsValidator( 347 PipelineOptions(case['options']), case['runner']) 348 self.assertEqual(validator.is_service_runner(), case['expected']) 349 350 def test_dataflow_job_file_and_template_location_mutually_exclusive(self): 351 runner = MockRunners.OtherRunner() 352 options = PipelineOptions( 353 ['--template_location', 'abc', '--dataflow_job_file', 'def']) 354 validator = PipelineOptionsValidator(options, runner) 355 errors = validator.validate() 356 self.assertTrue(errors) 357 358 def test_validate_template_location(self): 359 runner = MockRunners.OtherRunner() 360 options = PipelineOptions([ 361 '--template_location', 362 'abc', 363 ]) 364 validator = PipelineOptionsValidator(options, runner) 365 errors = validator.validate() 366 self.assertFalse(errors) 367 368 def test_validate_dataflow_job_file(self): 369 runner = MockRunners.OtherRunner() 370 options = PipelineOptions(['--dataflow_job_file', 'abc']) 371 validator = PipelineOptionsValidator(options, runner) 372 errors = validator.validate() 373 self.assertFalse(errors) 374 375 def test_num_workers_is_positive(self): 376 runner = MockRunners.DataflowRunner() 377 options = PipelineOptions([ 378 '--num_workers=-1', 379 '--worker_region=us-east1', 380 '--project=example:example', 381 '--temp_location=gs://foo/bar', 382 ]) 383 validator = PipelineOptionsValidator(options, runner) 384 errors = validator.validate() 385 self.assertEqual(len(errors), 1) 386 self.assertIn('num_workers', errors[0]) 387 self.assertIn('-1', errors[0]) 388 389 def test_max_num_workers_is_positive(self): 390 runner = MockRunners.DataflowRunner() 391 options = PipelineOptions([ 392 '--max_num_workers=-1', 393 '--worker_region=us-east1', 394 '--project=example:example', 395 '--temp_location=gs://foo/bar', 396 ]) 397 validator = PipelineOptionsValidator(options, runner) 398 errors = validator.validate() 399 self.assertEqual(len(errors), 1) 400 self.assertIn('max_num_workers', errors[0]) 401 self.assertIn('-1', errors[0]) 402 403 def test_num_workers_cannot_exceed_max_num_workers(self): 404 runner = MockRunners.DataflowRunner() 405 options = PipelineOptions([ 406 '--num_workers=43', 407 '--max_num_workers=42', 408 '--worker_region=us-east1', 409 '--project=example:example', 410 '--temp_location=gs://foo/bar', 411 ]) 412 validator = PipelineOptionsValidator(options, runner) 413 errors = validator.validate() 414 self.assertEqual(len(errors), 1) 415 self.assertIn('num_workers', errors[0]) 416 self.assertIn('43', errors[0]) 417 self.assertIn('max_num_workers', errors[0]) 418 self.assertIn('42', errors[0]) 419 420 def test_num_workers_can_equal_max_num_workers(self): 421 runner = MockRunners.DataflowRunner() 422 options = PipelineOptions([ 423 '--num_workers=42', 424 '--max_num_workers=42', 425 '--worker_region=us-east1', 426 '--project=example:example', 427 '--temp_location=gs://foo/bar', 428 ]) 429 validator = PipelineOptionsValidator(options, runner) 430 errors = validator.validate() 431 self.assertEqual(len(errors), 0) 432 433 def test_zone_and_worker_region_mutually_exclusive(self): 434 runner = MockRunners.DataflowRunner() 435 options = PipelineOptions([ 436 '--zone', 437 'us-east1-b', 438 '--worker_region', 439 'us-east1', 440 '--project=example:example', 441 '--temp_location=gs://foo/bar', 442 ]) 443 validator = PipelineOptionsValidator(options, runner) 444 errors = validator.validate() 445 self.assertEqual(len(errors), 1) 446 self.assertIn('zone', errors[0]) 447 self.assertIn('worker_region', errors[0]) 448 449 def test_zone_and_worker_zone_mutually_exclusive(self): 450 runner = MockRunners.DataflowRunner() 451 options = PipelineOptions([ 452 '--zone', 453 'us-east1-b', 454 '--worker_zone', 455 'us-east1-c', 456 '--project=example:example', 457 '--temp_location=gs://foo/bar', 458 ]) 459 validator = PipelineOptionsValidator(options, runner) 460 errors = validator.validate() 461 self.assertEqual(len(errors), 1) 462 self.assertIn('zone', errors[0]) 463 self.assertIn('worker_zone', errors[0]) 464 465 def test_experiment_region_and_worker_region_mutually_exclusive(self): 466 runner = MockRunners.DataflowRunner() 467 options = PipelineOptions([ 468 '--experiments', 469 'worker_region=us-west1', 470 '--worker_region', 471 'us-east1', 472 '--project=example:example', 473 '--temp_location=gs://foo/bar', 474 ]) 475 validator = PipelineOptionsValidator(options, runner) 476 errors = validator.validate() 477 self.assertEqual(len(errors), 1) 478 self.assertIn('experiment', errors[0]) 479 self.assertIn('worker_region', errors[0]) 480 481 def test_experiment_region_and_worker_zone_mutually_exclusive(self): 482 runner = MockRunners.DataflowRunner() 483 options = PipelineOptions([ 484 '--experiments', 485 'worker_region=us-west1', 486 '--worker_zone', 487 'us-east1-b', 488 '--project=example:example', 489 '--temp_location=gs://foo/bar', 490 ]) 491 validator = PipelineOptionsValidator(options, runner) 492 errors = validator.validate() 493 self.assertEqual(len(errors), 1) 494 self.assertIn('experiment', errors[0]) 495 self.assertIn('worker_region', errors[0]) 496 self.assertIn('worker_zone', errors[0]) 497 498 def test_programmatically_set_experiment_passed_as_string(self): 499 runner = MockRunners.DataflowRunner() 500 options = PipelineOptions( 501 project='example.com:example', 502 temp_location='gs://foo/bar/', 503 experiments='enable_prime', 504 dataflow_service_options='use_runner_v2', 505 ) 506 validator = PipelineOptionsValidator(options, runner) 507 errors = validator.validate() 508 self.assertEqual(len(errors), 2) 509 self.assertIn('experiments', errors[0]) 510 self.assertIn('dataflow_service_options', errors[1]) 511 512 def test_programmatically_set_experiment_passed_as_list(self): 513 runner = MockRunners.DataflowRunner() 514 options = PipelineOptions( 515 project='example.com:example', 516 temp_location='gs://foo/bar/', 517 experiments=['enable_prime'], 518 dataflow_service_options=['use_runner_v2'], 519 ) 520 validator = PipelineOptionsValidator(options, runner) 521 errors = validator.validate() 522 self.assertEqual(len(errors), 0) 523 self.assertEqual( 524 options.view_as(DebugOptions).experiments, ['enable_prime']) 525 self.assertEqual( 526 options.view_as(GoogleCloudOptions).dataflow_service_options, 527 ['use_runner_v2']) 528 529 def test_worker_region_and_worker_zone_mutually_exclusive(self): 530 runner = MockRunners.DataflowRunner() 531 options = PipelineOptions([ 532 '--worker_region', 533 'us-east1', 534 '--worker_zone', 535 'us-east1-b', 536 '--project=example:example', 537 '--temp_location=gs://foo/bar', 538 ]) 539 validator = PipelineOptionsValidator(options, runner) 540 errors = validator.validate() 541 self.assertEqual(len(errors), 1) 542 self.assertIn('worker_region', errors[0]) 543 self.assertIn('worker_zone', errors[0]) 544 545 def test_zone_alias_worker_zone(self): 546 runner = MockRunners.DataflowRunner() 547 options = PipelineOptions([ 548 '--zone=us-east1-b', 549 '--project=example:example', 550 '--temp_location=gs://foo/bar', 551 ]) 552 validator = PipelineOptionsValidator(options, runner) 553 errors = validator.validate() 554 self.assertEqual(len(errors), 0) 555 self.assertIsNone(options.view_as(WorkerOptions).zone) 556 self.assertEqual(options.view_as(WorkerOptions).worker_zone, 'us-east1-b') 557 558 def test_region_optional_for_non_service_runner(self): 559 runner = MockRunners.DataflowRunner() 560 # Remove default region for this test. 561 runner.get_default_gcp_region = lambda: None 562 options = PipelineOptions([ 563 '--project=example:example', 564 '--temp_location=gs://foo/bar', 565 '--dataflow_endpoint=http://localhost:20281', 566 ]) 567 validator = PipelineOptionsValidator(options, runner) 568 errors = validator.validate() 569 self.assertEqual(len(errors), 0) 570 571 def test_alias_sdk_container_to_worker_harness(self): 572 runner = MockRunners.DataflowRunner() 573 test_image = "SDK_IMAGE" 574 options = PipelineOptions([ 575 '--sdk_container_image=%s' % test_image, 576 '--project=example:example', 577 '--temp_location=gs://foo/bar', 578 ]) 579 validator = PipelineOptionsValidator(options, runner) 580 errors = validator.validate() 581 self.assertEqual(len(errors), 0) 582 self.assertEqual( 583 options.view_as(WorkerOptions).worker_harness_container_image, 584 test_image) 585 self.assertEqual( 586 options.view_as(WorkerOptions).sdk_container_image, test_image) 587 588 def test_alias_worker_harness_sdk_container_image(self): 589 runner = MockRunners.DataflowRunner() 590 test_image = "WORKER_HARNESS" 591 options = PipelineOptions([ 592 '--worker_harness_container_image=%s' % test_image, 593 '--project=example:example', 594 '--temp_location=gs://foo/bar', 595 ]) 596 validator = PipelineOptionsValidator(options, runner) 597 errors = validator.validate() 598 self.assertEqual(len(errors), 0) 599 self.assertEqual( 600 options.view_as(WorkerOptions).worker_harness_container_image, 601 test_image) 602 self.assertEqual( 603 options.view_as(WorkerOptions).sdk_container_image, test_image) 604 605 def test_worker_harness_sdk_container_image_mutually_exclusive(self): 606 runner = MockRunners.DataflowRunner() 607 options = PipelineOptions([ 608 '--worker_harness_container_image=WORKER', 609 '--sdk_container_image=SDK_ONLY', 610 '--project=example:example', 611 '--temp_location=gs://foo/bar', 612 ]) 613 validator = PipelineOptionsValidator(options, runner) 614 errors = validator.validate() 615 self.assertEqual(len(errors), 1) 616 self.assertIn('sdk_container_image', errors[0]) 617 self.assertIn('worker_harness_container_image', errors[0]) 618 619 def test_prebuild_sdk_container_base_image_disallowed(self): 620 runner = MockRunners.DataflowRunner() 621 options = PipelineOptions([ 622 '--project=example:example', 623 '--temp_location=gs://foo/bar', 624 '--prebuild_sdk_container_base_image=gcr.io/foo:bar' 625 ]) 626 validator = PipelineOptionsValidator(options, runner) 627 errors = validator.validate() 628 self.assertEqual(len(errors), 1) 629 self.assertIn('prebuild_sdk_container_base_image', errors[0]) 630 self.assertIn('sdk_container_image', errors[0]) 631 632 def test_prebuild_sdk_container_base_allowed_if_matches_custom_image(self): 633 runner = MockRunners.DataflowRunner() 634 options = PipelineOptions([ 635 '--project=example:example', 636 '--temp_location=gs://foo/bar', 637 '--sdk_container_image=gcr.io/foo:bar', 638 '--prebuild_sdk_container_base_image=gcr.io/foo:bar' 639 ]) 640 validator = PipelineOptionsValidator(options, runner) 641 errors = validator.validate() 642 self.assertEqual(len(errors), 0) 643 644 def test_test_matcher(self): 645 def get_validator(matcher): 646 options = [ 647 '--project=example:example', 648 '--job_name=job', 649 '--staging_location=gs://foo/bar', 650 '--temp_location=gs://foo/bar', 651 ] 652 if matcher: 653 options.append('%s=%s' % ('--on_success_matcher', matcher.decode())) 654 655 pipeline_options = PipelineOptions(options) 656 runner = MockRunners.TestDataflowRunner() 657 return PipelineOptionsValidator(pipeline_options, runner) 658 659 test_case = [ 660 { 661 'on_success_matcher': None, 'errors': [] 662 }, 663 { 664 'on_success_matcher': pickler.dumps(AlwaysPassMatcher()), 665 'errors': [] 666 }, 667 { 668 'on_success_matcher': b'abc', 'errors': ['on_success_matcher'] 669 }, 670 { 671 'on_success_matcher': pickler.dumps(object), 672 'errors': ['on_success_matcher'] 673 }, 674 ] 675 676 for case in test_case: 677 errors = get_validator(case['on_success_matcher']).validate() 678 self.assertEqual( 679 self.check_errors_for_arguments(errors, case['errors']), []) 680 681 def test_transform_name_mapping_without_update(self): 682 options = [ 683 '--project=example:example', 684 '--staging_location=gs://foo/bar', 685 '--temp_location=gs://foo/bar', 686 '--transform_name_mapping={\"fromPardo\":\"toPardo\"}' 687 ] 688 689 pipeline_options = PipelineOptions(options) 690 runner = MockRunners.DataflowRunner() 691 validator = PipelineOptionsValidator(pipeline_options, runner) 692 errors = validator.validate() 693 assert_that( 694 errors, 695 only_contains( 696 contains_string( 697 'Transform name mapping option is only useful when ' 698 '--update and --streaming is specified'))) 699 700 def test_transform_name_mapping_invalid_format(self): 701 options = [ 702 '--project=example:example', 703 '--staging_location=gs://foo/bar', 704 '--temp_location=gs://foo/bar', 705 '--update', 706 '--job_name=test', 707 '--streaming', 708 '--transform_name_mapping={\"fromPardo\":123}' 709 ] 710 711 pipeline_options = PipelineOptions(options) 712 runner = MockRunners.DataflowRunner() 713 validator = PipelineOptionsValidator(pipeline_options, runner) 714 errors = validator.validate() 715 assert_that( 716 errors, 717 only_contains( 718 contains_string('Invalid transform name mapping format.'))) 719 720 def test_type_check_additional(self): 721 runner = MockRunners.OtherRunner() 722 options = PipelineOptions(['--type_check_additional=all']) 723 validator = PipelineOptionsValidator(options, runner) 724 errors = validator.validate() 725 self.assertFalse(errors) 726 727 options = PipelineOptions(['--type_check_additional=']) 728 validator = PipelineOptionsValidator(options, runner) 729 errors = validator.validate() 730 self.assertFalse(errors) 731 732 def test_type_check_additional_unrecognized_feature(self): 733 runner = MockRunners.OtherRunner() 734 options = PipelineOptions(['--type_check_additional=all,dfgdf']) 735 validator = PipelineOptionsValidator(options, runner) 736 errors = validator.validate() 737 self.assertTrue(errors) 738 739 def test_environment_options(self): 740 test_cases = [ 741 { 742 'options': ['--environment_type=dOcKeR'], 'errors': [] 743 }, 744 { 745 'options': [ 746 '--environment_type=dOcKeR', 747 '--environment_options=docker_container_image=foo' 748 ], 749 'errors': [] 750 }, 751 { 752 'options': [ 753 '--environment_type=dOcKeR', '--environment_config=foo' 754 ], 755 'errors': [] 756 }, 757 { 758 'options': [ 759 '--environment_type=dOcKeR', 760 '--environment_options=docker_container_image=foo', 761 '--environment_config=foo' 762 ], 763 'errors': ['environment_config'] 764 }, 765 { 766 'options': [ 767 '--environment_type=dOcKeR', 768 '--environment_options=process_command=foo', 769 '--environment_options=process_variables=foo=bar', 770 '--environment_options=external_service_address=foo' 771 ], 772 'errors': [ 773 'process_command', 774 'process_variables', 775 'external_service_address' 776 ] 777 }, 778 { 779 'options': ['--environment_type=pRoCeSs'], 780 'errors': ['process_command'] 781 }, 782 { 783 'options': [ 784 '--environment_type=pRoCeSs', 785 '--environment_options=process_command=foo' 786 ], 787 'errors': [] 788 }, 789 { 790 'options': [ 791 '--environment_type=pRoCeSs', '--environment_config=foo' 792 ], 793 'errors': [] 794 }, 795 { 796 'options': [ 797 '--environment_type=pRoCeSs', 798 '--environment_options=process_command=foo', 799 '--environment_config=foo' 800 ], 801 'errors': ['environment_config'] 802 }, 803 { 804 'options': [ 805 '--environment_type=pRoCeSs', 806 '--environment_options=process_command=foo', 807 '--environment_options=process_variables=foo=bar', 808 '--environment_options=docker_container_image=foo', 809 '--environment_options=external_service_address=foo' 810 ], 811 'errors': ['docker_container_image', 'external_service_address'] 812 }, 813 { 814 'options': ['--environment_type=eXtErNaL'], 815 'errors': ['external_service_address'] 816 }, 817 { 818 'options': [ 819 '--environment_type=eXtErNaL', 820 '--environment_options=external_service_address=foo' 821 ], 822 'errors': [] 823 }, 824 { 825 'options': [ 826 '--environment_type=eXtErNaL', '--environment_config=foo' 827 ], 828 'errors': [] 829 }, 830 { 831 'options': [ 832 '--environment_type=eXtErNaL', 833 '--environment_options=external_service_address=foo', 834 '--environment_config=foo' 835 ], 836 'errors': ['environment_config'] 837 }, 838 { 839 'options': [ 840 '--environment_type=eXtErNaL', 841 '--environment_options=external_service_address=foo', 842 '--environment_options=process_command=foo', 843 '--environment_options=process_variables=foo=bar', 844 '--environment_options=docker_container_image=foo', 845 ], 846 'errors': [ 847 'process_command', 848 'process_variables', 849 'docker_container_image' 850 ] 851 }, 852 { 853 'options': ['--environment_type=lOoPbACk'], 'errors': [] 854 }, 855 { 856 'options': [ 857 '--environment_type=lOoPbACk', '--environment_config=foo' 858 ], 859 'errors': ['environment_config'] 860 }, 861 { 862 'options': [ 863 '--environment_type=lOoPbACk', 864 '--environment_options=docker_container_image=foo', 865 '--environment_options=process_command=foo', 866 '--environment_options=process_variables=foo=bar', 867 '--environment_options=external_service_address=foo', 868 ], 869 'errors': [ 870 'docker_container_image', 871 'process_command', 872 'process_variables', 873 'external_service_address' 874 ] 875 }, 876 { 877 'options': ['--environment_type=beam:env:foo:v1'], 'errors': [] 878 }, 879 { 880 'options': [ 881 '--environment_type=beam:env:foo:v1', 882 '--environment_config=foo' 883 ], 884 'errors': [] 885 }, 886 { 887 'options': [ 888 '--environment_type=beam:env:foo:v1', 889 '--environment_options=docker_container_image=foo', 890 '--environment_options=process_command=foo', 891 '--environment_options=process_variables=foo=bar', 892 '--environment_options=external_service_address=foo', 893 ], 894 'errors': [ 895 'docker_container_image', 896 'process_command', 897 'process_variables', 898 'external_service_address' 899 ] 900 }, 901 { 902 'options': [ 903 '--environment_options=docker_container_image=foo', 904 '--environment_options=process_command=foo', 905 '--environment_options=process_variables=foo=bar', 906 '--environment_options=external_service_address=foo', 907 ], 908 'errors': [ 909 'docker_container_image', 910 'process_command', 911 'process_variables', 912 'external_service_address' 913 ] 914 }, 915 ] 916 errors = [] 917 for case in test_cases: 918 validator = PipelineOptionsValidator( 919 PipelineOptions(case['options']), MockRunners.OtherRunner()) 920 validation_result = validator.validate() 921 validation_errors = self.check_errors_for_arguments( 922 validation_result, case['errors']) 923 if validation_errors: 924 errors.append( 925 'Options "%s" had unexpected validation results: "%s"' % 926 (' '.join(case['options']), ' '.join(validation_errors))) 927 self.assertEqual(errors, []) 928 929 930 if __name__ == '__main__': 931 logging.getLogger().setLevel(logging.INFO) 932 unittest.main()