github.com/apache/beam/sdks/v2@v2.48.2/python/test-suites/direct/common.gradle (about)

     1  /*
     2   * Licensed to the Apache Software Foundation (ASF) under one
     3   * or more contributor license agreements.  See the NOTICE file
     4   * distributed with this work for additional information
     5   * regarding copyright ownership.  The ASF licenses this file
     6   * to you under the Apache License, Version 2.0 (the
     7   * License); you may not use this file except in compliance
     8   * with the License.  You may obtain a copy of the License at
     9   *
    10   *     http://www.apache.org/licenses/LICENSE-2.0
    11   *
    12   * Unless required by applicable law or agreed to in writing, software
    13   * distributed under the License is distributed on an AS IS BASIS,
    14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15   * See the License for the specific language governing permissions and
    16   * limitations under the License.
    17   */
    18  evaluationDependsOn(':sdks:python:test-suites:xlang')
    19  
    20  def pythonVersionSuffix = project.ext.pythonVersion.replace('.', '')
    21  def pythonContainerVersion = project.ext.pythonVersion
    22  def runScriptsDir = "${rootDir}/sdks/python/scripts"
    23  // Basic test options for ITs running on Jenkins.
    24  def basicTestOpts = [
    25      "--capture=no",  // print stdout instantly
    26      "--numprocesses=8",  // run tests in parallel
    27      "--timeout=4500", // timeout of whole command execution
    28      "--color=yes", // console color
    29      "--log-cli-level=INFO" //log level info
    30  ]
    31  
    32  tasks.register("postCommitIT") {
    33    dependsOn 'installGcpTest'
    34  
    35    // Run IT tests with TestDirectRunner in batch in Python 3.
    36    doLast {
    37      def batchTests = [
    38          "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it",
    39          "apache_beam/io/gcp/pubsub_integration_test.py::PubSubIntegrationTest",
    40          "apache_beam/io/gcp/big_query_query_to_table_it_test.py::BigQueryQueryToTableIT",
    41          "apache_beam/io/gcp/bigquery_io_read_it_test.py",
    42          "apache_beam/io/gcp/bigquery_read_it_test.py",
    43          "apache_beam/io/gcp/bigquery_write_it_test.py",
    44          "apache_beam/io/gcp/datastore/v1new/datastore_write_it_test.py",
    45      ]
    46      def testOpts = basicTestOpts + ["${batchTests.join(' ')}"]
    47      def argMap = ["runner": "TestDirectRunner",
    48                    "test_opts": testOpts,
    49                    "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
    50                    ]
    51      def batchCmdArgs = mapToArgString(argMap)
    52      exec {
    53        executable 'sh'
    54        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $batchCmdArgs"
    55      }
    56    }
    57  }
    58  
    59  tasks.register("spannerioIT") {
    60    dependsOn 'installGcpTest'
    61  
    62    // Run Spanner IO IT tests with TestDirectRunner in batch in Python 3.
    63    doLast {
    64      def batchTests = [
    65          "apache_beam/io/gcp/experimental/spannerio_read_it_test.py",
    66          "apache_beam/io/gcp/experimental/spannerio_write_it_test.py",
    67      ]
    68      def testOpts = basicTestOpts + ["${batchTests.join(' ')}"]
    69      def argMap = ["runner": "TestDirectRunner",
    70                    "test_opts": testOpts,
    71                    "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
    72                    ]
    73      def batchCmdArgs = mapToArgString(argMap)
    74      exec {
    75        executable 'sh'
    76        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $batchCmdArgs"
    77      }
    78    }
    79  }
    80  
    81  tasks.register("examples") {
    82    dependsOn 'installGcpTest'
    83    dependsOn ':sdks:python:sdist'
    84  
    85    doLast {
    86      def testOpts = ["--log-cli-level=INFO"]
    87      def argMap = ["runner": "TestDirectRunner",
    88                    "test_opts": testOpts,
    89                    "suite": "postCommitExamples-direct-py${pythonVersionSuffix}",
    90                    "collect": "examples_postcommit and not sickbay_direct"
    91      ]
    92      def batchCmdArgs = mapToArgString(argMap)
    93      exec {
    94        executable 'sh'
    95        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $batchCmdArgs"
    96      }
    97    }
    98  }
    99  
   100  
   101  tasks.register("mongodbioIT") {
   102    dependsOn 'setupVirtualenv'
   103  
   104    Random r = new Random()
   105    def port = r.nextInt(1000) + 27017
   106    def containerName = "mongoioit" + port
   107  
   108    def options = [
   109            "--mongo_uri=mongodb://localhost:" + port
   110    ]
   111  
   112    // Pull the latest mongodb docker image and run
   113    doFirst {
   114      exec {
   115        executable 'sh'
   116        // TODO(https://github.com/apache/beam/issues/25359) switch back to use latest container when issue resolved.
   117        args '-c', "docker pull mongo && docker run --name ${containerName} -p ${port}:27017 -d mongo:6.0.3"
   118      }
   119    }
   120  
   121    doLast {
   122      exec {
   123        executable 'sh'
   124        args '-c', ". ${envdir}/bin/activate && pip install -e ${rootDir}/sdks/python/[test] && python -m apache_beam.io.mongodbio_it_test ${options.join(' ')}"
   125      }
   126      exec {
   127        executable 'sh'
   128        args '-c', "docker stop ${containerName} && docker rm ${containerName}"
   129      }
   130    }
   131  }
   132  
   133  tasks.register("directRunnerIT") {
   134    dependsOn 'installGcpTest'
   135  
   136    // Run IT tests with TestDirectRunner in batch.
   137    doLast {
   138      def tests = [
   139          "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it",
   140          "apache_beam/io/gcp/pubsub_integration_test.py::PubSubIntegrationTest",
   141          "apache_beam/io/gcp/big_query_query_to_table_it_test.py::BigQueryQueryToTableIT",
   142          "apache_beam/io/gcp/bigquery_io_read_it_test.py",
   143          "apache_beam/io/gcp/bigquery_read_it_test.py",
   144          "apache_beam/io/gcp/bigquery_write_it_test.py",
   145          "apache_beam/io/gcp/datastore/v1new/datastore_write_it_test.py",
   146      ]
   147      def batchTestOpts = basicTestOpts + ["${tests.join(' ')}"]
   148      def argMap = ["runner": "TestDirectRunner",
   149                    "test_opts": batchTestOpts,
   150                    "suite": "directRunnerIT-batch",
   151                    ]
   152      def batchCmdArgs = mapToArgString(argMap)
   153      exec {
   154        executable 'sh'
   155        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $batchCmdArgs"
   156      }
   157    }
   158  
   159    // Run IT tests with TestDirectRunner in streaming.
   160    doLast {
   161      def tests = [
   162          "apache_beam/examples/wordcount_it_test.py::WordCountIT::test_wordcount_it",
   163          "apache_beam/io/gcp/pubsub_integration_test.py::PubSubIntegrationTest",
   164          "apache_beam/io/gcp/bigquery_test.py::BigQueryStreamingInsertTransformIntegrationTests::test_multiple_destinations_transform",
   165          "apache_beam/io/gcp/bigquery_test.py::PubSubBigQueryIT",
   166          "apache_beam/io/gcp/bigquery_file_loads_test.py::BigQueryFileLoadsIT::test_bqfl_streaming",
   167          "apache_beam/io/gcp/bigquery_file_loads_test.py::BigQueryFileLoadsIT::test_bqfl_streaming_with_dynamic_destinations",
   168          "apache_beam/io/gcp/bigquery_file_loads_test.py::BigQueryFileLoadsIT::test_bqfl_streaming_with_copy_jobs",
   169      ]
   170      def streamingTestOpts = basicTestOpts + ["${tests.join(' ')}"]
   171      def argMap = ["runner": "TestDirectRunner",
   172                    "streaming": "true",
   173                    "test_opts": streamingTestOpts,
   174                    "suite": "directRunnerIT-streaming",
   175                  ]
   176      def streamingCmdArgs = mapToArgString(argMap)
   177      exec {
   178        executable 'sh'
   179        args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $streamingCmdArgs"
   180      }
   181    }
   182  }
   183  
   184  tasks.register("hdfsIntegrationTest") {
   185    doLast {
   186      exec {
   187        executable 'sh'
   188        args '-c', "${rootDir}/sdks/python/apache_beam/io/hdfs_integration_test/hdfs_integration_test.sh python:${pythonContainerVersion}"
   189      }
   190    }
   191  }
   192  
   193  tasks.register("azureIntegrationTest") {
   194    doLast {
   195      exec {
   196        executable 'sh'
   197        args '-c', "${rootDir}/sdks/python/apache_beam/io/azure/integration_test/azure_integration_test.sh python:${pythonContainerVersion}"
   198      }
   199    }
   200  }
   201  
   202  // Pytorch RunInference IT tests
   203  task torchInferenceTest {
   204    dependsOn 'installGcpTest'
   205    dependsOn ':sdks:python:sdist'
   206    def requirementsFile = "${rootDir}/sdks/python/apache_beam/ml/inference/torch_tests_requirements.txt"
   207    doFirst {
   208        exec {
   209          executable 'sh'
   210          args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile"
   211        }
   212      }
   213    doLast {
   214        def testOpts = basicTestOpts
   215        def argMap = [
   216            "test_opts": testOpts,
   217            "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
   218            "collect": "uses_pytorch and it_postcommit",
   219            "runner": "TestDirectRunner"
   220        ]
   221        def cmdArgs = mapToArgString(argMap)
   222        exec {
   223          executable 'sh'
   224          args '-c', ". ${envdir}/bin/activate && export FORCE_TORCH_IT=1 && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
   225        }
   226      }
   227  
   228  }
   229  
   230  // Scikit-learn RunInference IT tests
   231  task sklearnInferenceTest {
   232    dependsOn 'installGcpTest'
   233    dependsOn ':sdks:python:sdist'
   234    // TODO(https://github.com/apache/beam/issues/24787)
   235    // Scikit learn tests fails from version 1.2.0
   236    doFirst {
   237        exec {
   238          executable 'sh'
   239          args '-c', ". ${envdir}/bin/activate && pip install 'scikit-learn<1.2.0'"
   240        }
   241      }
   242    doLast {
   243        def testOpts = basicTestOpts
   244        def argMap = [
   245            "test_opts": testOpts,
   246            "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
   247            "collect": "uses_sklearn and it_postcommit" ,
   248            "runner": "TestDirectRunner"
   249        ]
   250        def cmdArgs = mapToArgString(argMap)
   251        exec {
   252          executable 'sh'
   253          args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
   254        }
   255      }
   256  }
   257  
   258  // TensorFlow Extended (TFX) RunInference IT tests
   259  task tfxInferenceTest {
   260    dependsOn 'installGcpTest'
   261    dependsOn ':sdks:python:sdist'
   262    def requirementsFile = "${rootDir}/sdks/python/apache_beam/examples/inference/tfx_bsl/requirements.txt"
   263    doFirst {
   264        exec {
   265          executable 'sh'
   266          args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile"
   267        }
   268      }
   269    doLast {
   270        def testOpts = basicTestOpts
   271        def argMap = [
   272            "test_opts": testOpts,
   273            "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
   274            "collect": "uses_tensorflow and it_postcommit" ,
   275            "runner": "TestDirectRunner"
   276        ]
   277        def cmdArgs = mapToArgString(argMap)
   278        exec {
   279          executable 'sh'
   280          args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
   281        }
   282      }
   283  }
   284  
   285  // TensorFlow RunInference IT tests
   286  task tensorflowInferenceTest {
   287    dependsOn 'installGcpTest'
   288    dependsOn ':sdks:python:sdist'
   289    def requirementsFile = "${rootDir}/sdks/python/apache_beam/ml/inference/tensorflow_tests_requirements.txt"
   290    doFirst {
   291        exec {
   292          executable 'sh'
   293          args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile"
   294        }
   295      }
   296    doLast {
   297        def testOpts = basicTestOpts
   298        def argMap = [
   299            "test_opts": testOpts,
   300            "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
   301            "collect": "uses_tf and it_postcommit" ,
   302            "runner": "TestDirectRunner"
   303        ]
   304        def cmdArgs = mapToArgString(argMap)
   305        exec {
   306          executable 'sh'
   307          args '-c', ". ${envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
   308        }
   309      }
   310  }
   311  
   312  // XGBoost RunInference IT tests
   313  task xgboostInferenceTest {
   314    dependsOn 'installGcpTest'
   315    dependsOn ':sdks:python:sdist'
   316    def requirementsFile = "${rootDir}/sdks/python/apache_beam/ml/inference/xgboost_tests_requirements.txt"
   317    doFirst {
   318        exec {
   319          executable 'sh'
   320          args '-c', ". ${envdir}/bin/activate && pip install -r $requirementsFile"
   321        }
   322      }
   323    doLast {
   324        def testOpts = basicTestOpts
   325        def argMap = [
   326            "test_opts": testOpts,
   327            "suite": "postCommitIT-direct-py${pythonVersionSuffix}",
   328            "collect": "uses_xgboost and it_postcommit",
   329            "runner": "TestDirectRunner"
   330        ]
   331        def cmdArgs = mapToArgString(argMap)
   332        exec {
   333          executable 'sh'
   334          args '-c', ". ${envdir}/bin/activate && export FORCE_XGBOOST_IT=1 && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
   335        }
   336      }
   337  
   338  }
   339  
   340  // Add all the RunInference framework IT tests to this gradle task that runs on Direct Runner Post commit suite.
   341  project.tasks.register("inferencePostCommitIT") {
   342    dependsOn = [
   343    'torchInferenceTest',
   344    'sklearnInferenceTest',
   345    'tensorflowInferenceTest',
   346    'xgboostInferenceTest'
   347    // (TODO) https://github.com/apache/beam/issues/25799
   348     // uncomment tfx bsl tests once tfx supports protobuf 4.x
   349    // 'tfxInferenceTest',
   350    ]
   351  }
   352  
   353  // Create cross-language tasks for running tests against Java expansion service(s)
   354  def gcpProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
   355  
   356  project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
   357      createCrossLanguageUsingJavaExpansionTask(
   358        name: taskMetadata.name,
   359        expansionProjectPath: taskMetadata.expansionProjectPath,
   360        collectMarker: taskMetadata.collectMarker,
   361        startJobServer: taskMetadata.startJobServer,
   362        cleanupJobServer: taskMetadata.cleanupJobServer,
   363        numParallelTests: 1,
   364        pythonPipelineOptions: [
   365          "--runner=TestDirectRunner",
   366          "--project=${gcpProject}",
   367        ],
   368        pytestOptions: [
   369          "--capture=no",  // print stdout instantly
   370          "--timeout=4500", // timeout of whole command execution
   371          "--color=yes", // console color
   372          "--log-cli-level=INFO" //log level info
   373        ]
   374      )
   375  }