github.com/apache/beam/sdks/v2@v2.48.2/java/extensions/sql/build.gradle (about)

     1  /*
     2   * Licensed to the Apache Software Foundation (ASF) under one
     3   * or more contributor license agreements.  See the NOTICE file
     4   * distributed with this work for additional information
     5   * regarding copyright ownership.  The ASF licenses this file
     6   * to you under the Apache License, Version 2.0 (the
     7   * License); you may not use this file except in compliance
     8   * with the License.  You may obtain a copy of the License at
     9   *
    10   *     http://www.apache.org/licenses/LICENSE-2.0
    11   *
    12   * Unless required by applicable law or agreed to in writing, software
    13   * distributed under the License is distributed on an AS IS BASIS,
    14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15   * See the License for the specific language governing permissions and
    16   * limitations under the License.
    17   */
    18  
    19  import groovy.json.JsonOutput
    20  import java.util.stream.Collectors
    21  
    22  plugins {
    23    id 'org.apache.beam.module'
    24    id 'ca.coglinc.javacc'
    25  }
    26  applyJavaNature(
    27    generatedClassPatterns: [
    28      /^org\.apache\.beam\.sdk\.extensions\.sql\.impl\.parser\.impl.*/,
    29    ],
    30    automaticModuleName: 'org.apache.beam.sdk.extensions.sql',
    31    classesTriggerCheckerBugs: [
    32      // TODO(https://github.com/apache/beam/issues/21068): This currently crashes with checkerframework 3.10.0
    33      // when compiling :sdks:java:extensions:sql:compileJava with:
    34      // message: class file for com.google.datastore.v1.Entity not found
    35      // ; The Checker Framework crashed.  Please report the crash.
    36      // Compilation unit: /usr/local/google/home/lcwik/git/beam/sdks/java/extensions/sql/src/main/java/org/apache/beam/sdk/extensions/sql/meta/provider/datastore/DataStoreV1TableProvider.java
    37      // Last visited tree at line 49 column 1:
    38      // @AutoService(TableProvider.class)
    39      // Exception: com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.datastore.v1.Entity not found; com.sun.tools.javac.code.Symbol$CompletionFailure: class file for com.google.datastore.v1.Entity not found
    40      'DataStoreV1TableProvider': 'TODO(https://github.com/apache/beam/issues/21068): Report the crash if still occurring on newest version',
    41    ],
    42    // javacc generated code produces lint warnings
    43    disableLintWarnings: ['dep-ann', 'rawtypes'],
    44  )
    45  
    46  description = "Apache Beam :: SDKs :: Java :: Extensions :: SQL"
    47  ext.summary = "Beam SQL provides a new interface to generate a Beam pipeline from SQL statement"
    48  
    49  configurations {
    50    // Create an fmppTask configuration representing the dependencies
    51    // required to define and execute the Ant FMPP task.
    52    // TODO: Migrate to a FMPP plugin once one exists
    53    fmppTask
    54    fmppTemplates
    55  }
    56  
    57  def hadoopVersions = [
    58      "285": "2.8.5",
    59      "292": "2.9.2",
    60      "2102": "2.10.2",
    61      "324": "3.2.4",
    62  ]
    63  
    64  hadoopVersions.each {kv -> configurations.create("hadoopVersion$kv.key")}
    65  
    66  dependencies {
    67    implementation enforcedPlatform(library.java.google_cloud_platform_libraries_bom)
    68  
    69    // TODO(https://github.com/apache/beam/issues/21156): Determine how to build without this dependency
    70    provided "org.immutables:value:2.8.8"
    71    permitUnusedDeclared "org.immutables:value:2.8.8"
    72    javacc "net.java.dev.javacc:javacc:4.0"
    73    fmppTask "com.googlecode.fmpp-maven-plugin:fmpp-maven-plugin:1.0"
    74    fmppTask "org.freemarker:freemarker:2.3.31"
    75    fmppTemplates library.java.vendored_calcite_1_28_0
    76    implementation project(path: ":sdks:java:core", configuration: "shadow")
    77    implementation project(":sdks:java:extensions:avro")
    78    implementation project(":sdks:java:extensions:join-library")
    79    permitUnusedDeclared project(":sdks:java:extensions:join-library") // BEAM-11761
    80    implementation project(":sdks:java:extensions:sql:udf")
    81    implementation project(path: ":runners:direct-java", configuration: "shadow")
    82    implementation library.java.commons_codec
    83    implementation library.java.commons_csv
    84    implementation library.java.jackson_databind
    85    implementation library.java.joda_time
    86    implementation library.java.vendored_calcite_1_28_0
    87    implementation "com.alibaba:fastjson:1.2.69"
    88    implementation "org.codehaus.janino:janino:3.0.11"
    89    implementation "org.codehaus.janino:commons-compiler:3.0.11"
    90    implementation library.java.jackson_core
    91    implementation library.java.mongo_java_driver
    92    implementation library.java.slf4j_api
    93    implementation library.java.joda_time
    94    implementation library.java.vendored_guava_26_0_jre
    95    provided project(":sdks:java:io:kafka")
    96    implementation project(":sdks:java:extensions:google-cloud-platform-core")
    97    permitUnusedDeclared project(":sdks:java:extensions:google-cloud-platform-core")
    98    implementation project(":sdks:java:io:google-cloud-platform")
    99    implementation library.java.proto_google_cloud_bigtable_v2
   100    implementation library.java.google_api_services_bigquery
   101    permitUnusedDeclared library.java.google_api_services_bigquery
   102    implementation library.java.proto_google_cloud_pubsublite_v1
   103    implementation library.java.google_cloud_pubsublite
   104    implementation project(":sdks:java:io:mongodb")
   105    implementation library.java.avro
   106    implementation library.java.protobuf_java
   107    implementation library.java.protobuf_java_util
   108    provided project(":sdks:java:io:parquet")
   109    provided library.java.jackson_dataformat_xml
   110    permitUnusedDeclared library.java.jackson_dataformat_xml
   111    provided library.java.hadoop_client
   112    permitUnusedDeclared library.java.hadoop_client
   113    provided library.java.kafka_clients
   114    testImplementation library.java.vendored_calcite_1_28_0
   115    testImplementation library.java.vendored_guava_26_0_jre
   116    testImplementation library.java.junit
   117    testImplementation library.java.quickcheck_core
   118    testImplementation library.java.testcontainers_kafka
   119    testImplementation library.java.google_cloud_bigtable
   120    testImplementation library.java.google_cloud_bigtable_client_core_config
   121    testImplementation library.java.google_cloud_bigtable_emulator
   122    testImplementation library.java.proto_google_cloud_bigtable_admin_v2
   123    testImplementation library.java.proto_google_cloud_datastore_v1
   124    testImplementation library.java.google_cloud_datastore_v1_proto_client
   125    testImplementation library.java.kafka_clients
   126    testImplementation project(":sdks:java:io:kafka")
   127    testImplementation project(path: ":sdks:java:io:mongodb", configuration: "testRuntimeMigration")
   128    testImplementation project(path: ":sdks:java:io:thrift", configuration: "testRuntimeMigration")
   129    testImplementation project(path: ":sdks:java:extensions:protobuf", configuration: "testRuntimeMigration")
   130    testCompileOnly project(":sdks:java:extensions:sql:udf-test-provider")
   131    testRuntimeOnly library.java.slf4j_jdk14
   132    hadoopVersions.each {kv ->
   133      "hadoopVersion$kv.key" "org.apache.hadoop:hadoop-client:$kv.value"
   134    }
   135  }
   136  
   137  hadoopVersions.each {kv ->
   138    configurations."hadoopVersion$kv.key" {
   139      resolutionStrategy {
   140        force "org.apache.hadoop:hadoop-client:$kv.value"
   141      }
   142    }
   143  }
   144  
   145  // Copy Calcite templates and our own template into the build directory
   146  // so we have one location for the FMPP task to parse.
   147  task copyFmppTemplatesFromSrc(type: Copy) {
   148    from "src/main/codegen"
   149    into "${project.buildDir}/templates-fmpp/codegen"
   150  }
   151  task copyFmppTemplatesFromCalciteCore(type: Copy) {
   152    dependsOn configurations.fmppTemplates
   153    File calciteCoreJar = files(configurations.fmppTemplates.files).filter {
   154      it.name.startsWith("beam-vendor-calcite")
   155    }.singleFile
   156    from zipTree(calciteCoreJar)
   157    include "**/Parser.jj"
   158    into "${project.buildDir}/templates-fmpp"
   159    filter{
   160      line ->
   161        line.replace('import org.apache.calcite.', 'import org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.')
   162    }
   163    filter{
   164      line ->
   165        line.replace('import static org.apache.calcite.', 'import static org.apache.beam.vendor.calcite.v1_28_0.org.apache.calcite.')
   166    }
   167  }
   168  
   169  // Generate the FMPP sources from the FMPP templates.
   170  def generateFmppOutputDir = "${project.buildDir}/generated/fmpp"
   171  task generateFmppSources {
   172    dependsOn configurations.fmppTask
   173    dependsOn copyFmppTemplatesFromSrc
   174    dependsOn copyFmppTemplatesFromCalciteCore
   175    doLast {
   176      ant.taskdef(name: "fmpp", classname: "fmpp.tools.AntTask", classpath: configurations.fmppTask.asPath)
   177      ant.fmpp(configuration: "src/main/codegen/config.fmpp", sourceRoot: "${project.buildDir}/templates-fmpp/codegen/templates", outputRoot: generateFmppOutputDir)
   178    }
   179  }
   180  
   181  // Match the output directory for generated code with the package, to be more tool-friendly
   182  def generateFmppJavaccRoot = "${generateFmppOutputDir}/javacc"
   183  def generatedJavaccSourceDir = "${project.buildDir}/generated/javacc"
   184  def generatedJavaccPackageDir = "${generatedJavaccSourceDir}/org/apache/beam/sdk/extensions/sql/impl/parser/impl"
   185  compileJavacc {
   186    dependsOn generateFmppSources
   187    inputDirectory = file(generateFmppJavaccRoot)
   188    outputDirectory = file(generatedJavaccPackageDir)
   189    arguments = [static: "false", lookahead: "2"]
   190  }
   191  
   192  // Help IntelliJ find the fmpp bits
   193  idea {
   194    module {
   195      sourceDirs += file(generateFmppOutputDir)
   196      generatedSourceDirs += file(generateFmppOutputDir)
   197  
   198      sourceDirs += file(generatedJavaccSourceDir)
   199      generatedSourceDirs += file(generatedJavaccSourceDir)
   200    }
   201  }
   202  
   203  // Run basic SQL example
   204  task runBasicExample(type: JavaExec) {
   205    description = "Run basic SQL example"
   206    mainClass = "org.apache.beam.sdk.extensions.sql.example.BeamSqlExample"
   207    classpath = sourceSets.main.runtimeClasspath
   208    args = ["--runner=DirectRunner"]
   209  }
   210  
   211  // Run SQL example on POJO inputs
   212  task runPojoExample(type: JavaExec) {
   213    description = "Run SQL example for PCollections of POJOs"
   214    mainClass = "org.apache.beam.sdk.extensions.sql.example.BeamSqlPojoExample"
   215    classpath = sourceSets.main.runtimeClasspath
   216    args = ["--runner=DirectRunner"]
   217  }
   218  
   219  task integrationTest(type: Test) {
   220    def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
   221    def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/'
   222  
   223    // Disable Gradle cache (it should not be used because the IT's won't run).
   224    outputs.upToDateWhen { false }
   225  
   226    def pipelineOptions = [
   227            "--project=${gcpProject}",
   228            "--tempLocation=${gcsTempRoot}",
   229            "--blockOnRun=false"]
   230  
   231    systemProperty "beamTestPipelineOptions", JsonOutput.toJson(pipelineOptions)
   232  
   233    include '**/*IT.class'
   234  
   235    maxParallelForks 4
   236    classpath = project(":sdks:java:extensions:sql")
   237            .sourceSets
   238            .test
   239            .runtimeClasspath
   240    testClassesDirs = files(project(":sdks:java:extensions:sql").sourceSets.test.output.classesDirs)
   241    useJUnit { }
   242  }
   243  
   244  task postCommit {
   245    group = "Verification"
   246    description = "Various integration tests"
   247    dependsOn integrationTest
   248  }
   249  
   250  task emptyJar(type: Jar) {
   251    archiveBaseName = "${project.archivesBaseName}-empty-jar"
   252    from fileTree(dir: getTemporaryDir().createNewFile().toString())
   253  }
   254  
   255  task hadoopVersionsTest(group: "Verification") {
   256    description = "Runs SQL tests with different Hadoop versions"
   257    def taskNames = hadoopVersions.keySet().stream()
   258        .map{num -> "hadoopVersion${num}Test"}
   259        .collect(Collectors.toList())
   260    dependsOn taskNames
   261  }
   262  
   263  hadoopVersions.each { kv ->
   264    task "hadoopVersion${kv.key}Test"(type: Test, group: "Verification") {
   265      description = "Runs SQL tests with Hadoop version $kv.value"
   266      classpath = configurations."hadoopVersion$kv.key" + sourceSets.test.runtimeClasspath
   267      include '**/*Test.class'
   268      dependsOn emptyJar
   269      // Pass jars used by Java UDF tests via system properties.
   270      evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath.
   271      systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath
   272      systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath
   273    }
   274  }
   275  
   276  test {
   277    dependsOn emptyJar
   278    // Pass jars used by Java UDF tests via system properties.
   279    evaluationDependsOn(":sdks:java:extensions:sql:udf-test-provider") // Needed to resolve jarPath.
   280    systemProperty "beam.sql.udf.test.jar_path", project(":sdks:java:extensions:sql:udf-test-provider").jarPath
   281    systemProperty "beam.sql.udf.test.empty_jar_path", emptyJar.archivePath
   282  }