github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/clients/hadoopfs/src/test/java/io/lakefs/S3FSTestBase.java (about)

     1  package io.lakefs;
     2  
     3  import org.slf4j.Logger;
     4  import org.slf4j.LoggerFactory;
     5  import com.amazonaws.ClientConfiguration;
     6  import com.amazonaws.auth.AWSCredentials;
     7  import com.amazonaws.auth.BasicAWSCredentials;
     8  import com.amazonaws.services.s3.AmazonS3;
     9  import com.amazonaws.services.s3.AmazonS3Client;
    10  import com.amazonaws.services.s3.S3ClientOptions;
    11  import com.amazonaws.services.s3.model.*;
    12  import com.aventrix.jnanoid.jnanoid.NanoIdUtils;
    13  import com.google.common.collect.ImmutableList;
    14  import com.google.common.collect.Lists;
    15  
    16  import org.apache.commons.io.IOUtils;
    17  
    18  import io.lakefs.clients.sdk.model.*;
    19  
    20  import org.junit.Assert;
    21  import org.junit.Before;
    22  import org.junit.Rule;
    23  import org.junit.Test;
    24  import org.slf4j.Logger;
    25  import org.slf4j.LoggerFactory;
    26  import org.testcontainers.containers.GenericContainer;
    27  import org.testcontainers.containers.output.Slf4jLogConsumer;
    28  import org.testcontainers.utility.DockerImageName;
    29  
    30  import java.util.List;
    31  
    32  /**
    33   * Base for all LakeFSFilesystem tests that need to access S3.  It adds a
    34   * MinIO container to FSTestBase, and configures to use it.
    35   * The visibility of this class is public as it's being used by other libraries for testing purposes
    36   */
    37  public abstract class S3FSTestBase extends FSTestBase {
    38      static private final Logger LOG = LoggerFactory.getLogger(S3FSTestBase.class);
    39  
    40      protected String s3Endpoint;
    41      protected AmazonS3 s3Client;
    42  
    43      private static final DockerImageName MINIO = DockerImageName.parse("minio/minio:RELEASE.2021-06-07T21-40-51Z");
    44  
    45      @Rule
    46      public final GenericContainer s3 = new GenericContainer(MINIO.toString()).
    47          withCommand("minio", "server", "/data").
    48          withEnv("MINIO_ROOT_USER", S3_ACCESS_KEY_ID).
    49          withEnv("MINIO_ROOT_PASSWORD", S3_SECRET_ACCESS_KEY).
    50          withEnv("MINIO_DOMAIN", "s3.local.lakefs.io").
    51          withEnv("MINIO_UPDATE", "off").
    52          withExposedPorts(9000);
    53  
    54      @Before
    55      public void logS3Container() {
    56          Logger s3Logger = LoggerFactory.getLogger("s3 container");
    57          Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(s3Logger)
    58              .withMdc("container", "s3")
    59              .withSeparateOutputStreams();
    60          s3.followOutput(logConsumer);
    61      }
    62  
    63      public void s3ClientSetup() {
    64          AWSCredentials creds = new BasicAWSCredentials(S3_ACCESS_KEY_ID, S3_SECRET_ACCESS_KEY);
    65  
    66          ClientConfiguration clientConfiguration = new ClientConfiguration()
    67                  .withSignerOverride("AWSS3V4SignerType");
    68          s3Endpoint = String.format("http://s3.local.lakefs.io:%d", s3.getMappedPort(9000));
    69  
    70          s3Client = new AmazonS3Client(creds, clientConfiguration);
    71  
    72          S3ClientOptions s3ClientOptions = new S3ClientOptions()
    73              .withPathStyleAccess(true);
    74          s3Client.setS3ClientOptions(s3ClientOptions);
    75          s3Client.setEndpoint(s3Endpoint);
    76  
    77          s3Bucket = makeS3BucketName();
    78          s3Base = String.format("s3://%s/", s3Bucket);
    79          LOG.info("S3: bucket {} => base URL {}", s3Bucket, s3Base);
    80  
    81          CreateBucketRequest cbr = new CreateBucketRequest(s3Bucket);
    82          s3Client.createBucket(cbr);
    83      }
    84  
    85      /**
    86       * @return all pathnames under s3Prefix that start with prefix.  (Obvious not scalable!)
    87       */
    88      protected List<String> getS3FilesByPrefix(String prefix) {
    89  
    90          ListObjectsRequest req = new ListObjectsRequest()
    91              .withBucketName(s3Bucket)
    92              .withPrefix(prefix)
    93              .withDelimiter(null);
    94  
    95          ObjectListing listing = s3Client.listObjects(req);
    96          List<S3ObjectSummary> summaries = listing.getObjectSummaries();
    97          if (listing.isTruncated()) {
    98              Assert.fail(String.format("[internal] no support for test that creates >%d S3 objects", listing.getMaxKeys()));
    99          }
   100  
   101          return Lists.transform(summaries, S3ObjectSummary::getKey);
   102      }
   103  
   104      protected void assertS3Object(StagingLocation stagingLocation, String contents) {
   105          String s3Key = getS3Key(stagingLocation);
   106          List<String> actualFiles = ImmutableList.of("<not yet listed>");
   107          try (S3Object obj =
   108               s3Client.getObject(new GetObjectRequest(s3Bucket, "/" + s3Key))) {
   109              actualFiles = getS3FilesByPrefix("");
   110              String actual = IOUtils.toString(obj.getObjectContent());
   111              Assert.assertEquals(contents, actual);
   112  
   113              Assert.assertEquals(ImmutableList.of(s3Key), actualFiles);
   114          } catch (Exception e) {
   115              throw new RuntimeException("Files " + actualFiles +
   116                                         "; read key " + s3Key + " failed", e);
   117          }
   118      }
   119  
   120      protected void moreHadoopSetup() {
   121          s3ClientSetup();
   122  
   123          conf.set("fs.s3a.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem");
   124          conf.set(org.apache.hadoop.fs.s3a.Constants.ACCESS_KEY, S3_ACCESS_KEY_ID);
   125          conf.set(org.apache.hadoop.fs.s3a.Constants.SECRET_KEY, S3_SECRET_ACCESS_KEY);
   126          conf.set(org.apache.hadoop.fs.s3a.Constants.ENDPOINT, s3Endpoint);
   127          conf.set(org.apache.hadoop.fs.s3a.Constants.BUFFER_DIR, "/tmp/s3a");
   128      }
   129  }