github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/test/spark/docker-compose.yaml (about)

     1  x-lakefs-common:
     2    &lakefs-common
     3    image: "${REPO:-treeverse}/lakefs:${TAG:-latest}"
     4    environment:
     5      - LAKEFS_GATEWAYS_S3_DOMAIN_NAME=s3.docker.lakefs.io,s3.local.lakefs.io
     6      - LAKEFS_AUTH_ENCRYPT_SECRET_KEY=some random secret string
     7      - LAKEFS_DATABASE_TYPE=postgres
     8      - LAKEFS_DATABASE_POSTGRES_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
     9      - LAKEFS_LOGGING_LEVEL=DEBUG
    10      - LAKEFS_STATS_ENABLED=false
    11      - LAKEFS_BLOCKSTORE_LOCAL_PATH=/home/lakefs
    12      - LAKEFS_BLOCKSTORE_TYPE
    13      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID
    14      - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY
    15      - LAKEFS_BLOCKSTORE_GS_CREDENTIALS_JSON
    16      - AWS_REGION
    17      - AWS_ACCESS_KEY_ID
    18      - AWS_SECRET_ACCESS_KEY
    19      - TESTER_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
    20      - TESTER_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
    21  
    22  services:
    23    postgres:
    24      image: "postgres:11"
    25      ports:
    26        - "5432:5432"
    27      environment:
    28        POSTGRES_USER: lakefs
    29        POSTGRES_PASSWORD: lakefs
    30    lakefs:
    31      <<: *lakefs-common
    32      ports:
    33        - "8000:8000"
    34      networks:
    35        default:
    36          ipv4_address: 10.5.0.55
    37      depends_on:
    38        - "postgres"
    39      volumes:
    40        - ./lakectl-tester.yaml:/home/lakefs/.lakectl.yaml
    41        - .:/local
    42      healthcheck:
    43        test: /app/wait-for localhost:8000
    44        timeout: 3s
    45        start_period: 0s
    46        interval: 3s
    47        retries: 10
    48      entrypoint: ["/app/wait-for", "postgres:5432", "--", "/app/lakefs", "run"]
    49    lakefs-setup:
    50      <<: *lakefs-common
    51      depends_on:
    52        lakefs:
    53          condition: service_healthy
    54      entrypoint: ["sh", "-c", "/app/lakefs setup --user-name tester --access-key-id $${TESTER_ACCESS_KEY_ID} --secret-access-key $${TESTER_SECRET_ACCESS_KEY}"]
    55    spark:
    56      image: docker.io/${SPARK_BASE:-bitnami/spark}:${SPARK_TAG:-3}
    57      environment:
    58        - SPARK_MODE=master
    59        - SPARK_MASTER_HOST=spark
    60        - SPARK_RPC_AUTHENTICATION_ENABLED=no
    61        - SPARK_RPC_ENCRYPTION_ENABLED=no
    62        - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
    63        - SPARK_SSL_ENABLED=no
    64        - AWS_ACCESS_KEY_ID
    65        - AWS_SECRET_ACCESS_KEY
    66        - METACLIENT_JAR
    67        - EXPORT_LOCATION
    68        - TESTER_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
    69        - TESTER_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
    70      ports:
    71        - 18080:8080
    72      extra_hosts:
    73        - "s3.docker.lakefs.io:10.5.0.55"
    74        - "example.s3.docker.lakefs.io:10.5.0.55"
    75        - "gateway-test-spark2.s3.docker.lakefs.io:10.5.0.55"
    76        - "gateway-test-spark3.s3.docker.lakefs.io:10.5.0.55"
    77        - "thick-client-test.s3.docker.lakefs.io:10.5.0.55"
    78    spark-worker:
    79      image: docker.io/${SPARK_BASE:-bitnami/spark}:${SPARK_TAG:-3}
    80      ports:
    81        - 8081
    82      environment:
    83        - SPARK_MODE=worker
    84        - SPARK_MASTER_URL=spark://spark:7077
    85        - SPARK_WORKER_MEMORY=4G
    86        - SPARK_WORKER_CORES=8
    87        - SPARK_RPC_AUTHENTICATION_ENABLED=no
    88        - SPARK_RPC_ENCRYPTION_ENABLED=no
    89        - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
    90        - SPARK_SSL_ENABLED=no
    91        - AWS_ACCESS_KEY_ID
    92        - AWS_SECRET_ACCESS_KEY
    93        - AWS_REGION
    94        - TESTER_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
    95        - TESTER_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
    96      extra_hosts:
    97        - "s3.docker.lakefs.io:10.5.0.55"
    98        - "docker.lakefs.io:10.5.0.55"
    99        - "example.s3.docker.lakefs.io:10.5.0.55"
   100        - "gateway-test-spark2.s3.docker.lakefs.io:10.5.0.55"
   101        - "gateway-test-spark3.s3.docker.lakefs.io:10.5.0.55"
   102        - "thick-client-test.s3.docker.lakefs.io:10.5.0.55"
   103    spark-submit:
   104      image: docker.io/${SPARK_BASE:-bitnami/spark}:${SPARK_TAG:-3}
   105      profiles: ["command"]
   106      volumes:
   107        - ./:/local
   108        - ../../clients/hadoopfs/target:/target
   109      environment:
   110        - input
   111        - output
   112        - s3input
   113        - s3output
   114        - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID
   115        - CLIENT_JAR
   116        - EXPORT_LOCATION
   117        - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY
   118        - AWS_ACCESS_KEY_ID
   119        - AWS_SECRET_ACCESS_KEY
   120        - AWS_REGION
   121        - TESTER_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
   122        - TESTER_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
   123      extra_hosts:
   124        - "s3.docker.lakefs.io:10.5.0.55"
   125        - "docker.lakefs.io:10.5.0.55"
   126        - "example.s3.docker.lakefs.io:10.5.0.55"
   127        - "gateway-test-spark2.s3.docker.lakefs.io:10.5.0.55"
   128        - "gateway-test-spark3.s3.docker.lakefs.io:10.5.0.55"
   129        - "thick-client-test.s3.docker.lakefs.io:10.5.0.55"
   130  
   131  networks:
   132    default:
   133      driver: bridge
   134      ipam:
   135       config:
   136         - subnet: 10.5.0.0/16
   137           gateway: 10.5.0.1
   138