github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/clients/hadoopfs/src/main/java/io/lakefs/storage/PresignedStorageAccessStrategy.java (about)

     1  package io.lakefs.storage;
     2  
     3  import java.io.IOException;
     4  import java.io.OutputStream;
     5  import java.net.HttpURLConnection;
     6  import java.net.URL;
     7  
     8  import org.apache.hadoop.fs.FSDataInputStream;
     9  import org.apache.hadoop.fs.FSDataOutputStream;
    10  
    11  import io.lakefs.LakeFSClient;
    12  import io.lakefs.LakeFSFileSystem;
    13  import io.lakefs.LakeFSLinker;
    14  import io.lakefs.clients.sdk.ApiException;
    15  import io.lakefs.clients.sdk.ObjectsApi;
    16  import io.lakefs.clients.sdk.StagingApi;
    17  import io.lakefs.clients.sdk.model.ObjectStats;
    18  import io.lakefs.clients.sdk.model.StagingLocation;
    19  import io.lakefs.utils.ObjectLocation;
    20  
    21  public class PresignedStorageAccessStrategy implements StorageAccessStrategy {
    22  
    23      private final LakeFSFileSystem lakeFSFileSystem;
    24      private final LakeFSClient lfsClient;
    25  
    26      public PresignedStorageAccessStrategy(LakeFSFileSystem lakeFSFileSystem,
    27              LakeFSClient lfsClient) {
    28          this.lakeFSFileSystem = lakeFSFileSystem;
    29          this.lfsClient = lfsClient;
    30      }
    31      
    32      @Override
    33      public FSDataOutputStream createDataOutputStream(ObjectLocation objectLocation,
    34              CreateOutputStreamParams params, boolean overwrite) throws ApiException, IOException {
    35          StagingApi stagingApi = lfsClient.getStagingApi();
    36          StagingLocation stagingLocation =
    37              stagingApi.getPhysicalAddress(objectLocation.getRepository(),
    38                                            objectLocation.getRef(), objectLocation.getPath())
    39              .presign(true).execute();
    40          URL presignedUrl = new URL(stagingLocation.getPresignedUrl());
    41          HttpURLConnection connection = (HttpURLConnection) presignedUrl.openConnection();
    42          connection.setDoOutput(true);
    43          connection.setRequestProperty("Content-Type", "application/octet-stream");
    44          connection.setRequestMethod("PUT");
    45          LakeFSLinker linker = new LakeFSLinker(lakeFSFileSystem, lfsClient, objectLocation, stagingLocation, overwrite);
    46          OutputStream out = new LakeFSFileSystemOutputStream(connection, linker);
    47          // TODO(ariels): add fs.FileSystem.Statistics here to keep track.
    48          return new FSDataOutputStream(out, null);
    49      }
    50  
    51      @Override
    52      public FSDataInputStream createDataInputStream(ObjectLocation objectLocation, int bufSize)
    53              throws ApiException, IOException {
    54          ObjectsApi objectsApi = lfsClient.getObjectsApi();
    55          ObjectStats stats = objectsApi.statObject(objectLocation.getRepository(),
    56                                                    objectLocation.getRef(),
    57                                                    objectLocation.getPath())
    58              .userMetadata(false).presign(true)
    59              .execute();
    60          return new FSDataInputStream(new HttpRangeInputStream(stats.getPhysicalAddress(), bufSize));
    61      }
    62  }