github.com/grailbio/base@v0.0.11/file/s3file/versions_leaf.go (about)

     1  package s3file
     2  
     3  import (
     4  	"context"
     5  	"os"
     6  	"sync/atomic"
     7  	"unsafe"
     8  
     9  	"github.com/aws/aws-sdk-go/service/s3/s3iface"
    10  	"github.com/grailbio/base/errors"
    11  	"github.com/grailbio/base/file"
    12  	"github.com/grailbio/base/file/fsnode"
    13  	"github.com/grailbio/base/grail/biofs/biofseventlog"
    14  	"github.com/grailbio/base/ioctx"
    15  	"github.com/grailbio/base/ioctx/fsctx"
    16  )
    17  
    18  type (
    19  	versionsLeaf struct {
    20  		fsnode.FileInfo
    21  		s3Query
    22  		versionID string
    23  	}
    24  	versionsFile struct {
    25  		versionsLeaf
    26  
    27  		// readOffset is the cursor for Read().
    28  		readOffset int64
    29  
    30  		reader chunkReaderCache
    31  	}
    32  )
    33  
    34  var (
    35  	_ fsnode.Leaf    = versionsLeaf{}
    36  	_ fsctx.File     = (*versionsFile)(nil)
    37  	_ ioctx.ReaderAt = (*versionsFile)(nil)
    38  )
    39  
    40  func (n versionsLeaf) FSNodeT() {}
    41  
    42  func (n versionsLeaf) OpenFile(ctx context.Context, flag int) (fsctx.File, error) {
    43  	biofseventlog.UsedFeature("s3.versions.open")
    44  	return &versionsFile{versionsLeaf: n}, nil
    45  }
    46  
    47  func (f *versionsFile) Stat(ctx context.Context) (os.FileInfo, error) {
    48  	return f.FileInfo, nil
    49  }
    50  
    51  func (f *versionsFile) Read(ctx context.Context, dst []byte) (int, error) {
    52  	n, err := f.ReadAt(ctx, dst, f.readOffset)
    53  	f.readOffset += int64(n)
    54  	return n, err
    55  }
    56  
    57  func (f *versionsFile) ReadAt(ctx context.Context, dst []byte, offset int64) (int, error) {
    58  	reader, cleanUp, err := f.reader.getOrCreate(ctx, func() (*chunkReaderAt, error) {
    59  		clients, err := f.impl.clientsForAction(ctx, "GetObjectVersion", f.bucket, f.key)
    60  		if err != nil {
    61  			return nil, errors.E(err, "getting clients")
    62  		}
    63  		return &chunkReaderAt{
    64  			name: f.path(), bucket: f.bucket, key: f.key, versionID: f.versionID,
    65  			newRetryPolicy: func() retryPolicy {
    66  				return newBackoffPolicy(append([]s3iface.S3API{}, clients...), file.Opts{})
    67  			},
    68  		}, nil
    69  	})
    70  	if err != nil {
    71  		return 0, err
    72  	}
    73  	defer cleanUp()
    74  	// TODO: Consider checking s3Info for ETag changes.
    75  	n, _, err := reader.ReadAt(ctx, dst, offset)
    76  	return n, err
    77  }
    78  
    79  func (f *versionsFile) Close(ctx context.Context) error {
    80  	f.reader.close()
    81  	return nil
    82  }
    83  
    84  type chunkReaderCache struct {
    85  	// available is idle (for some goroutine to use). Goroutines set available = nil before
    86  	// using it to "acquire" it, then return it after their operation (if available == nil then).
    87  	// If the caller only uses one thread, we'll end up creating and reusing just one
    88  	// *chunkReaderAt for all operations.
    89  	available unsafe.Pointer // *chunkReaderAt
    90  }
    91  
    92  // get constructs a reader. cleanUp must be called iff error is nil.
    93  func (c *chunkReaderCache) getOrCreate(
    94  	ctx context.Context, create func() (*chunkReaderAt, error),
    95  ) (
    96  	reader *chunkReaderAt, cleanUp func(), err error,
    97  ) {
    98  	trySaveReader := func() {
    99  		if atomic.CompareAndSwapPointer(&c.available, nil, unsafe.Pointer(reader)) {
   100  			return
   101  		}
   102  		reader.Close()
   103  	}
   104  
   105  	reader = (*chunkReaderAt)(atomic.SwapPointer(&c.available, nil))
   106  	if reader != nil {
   107  		return reader, trySaveReader, nil
   108  	}
   109  
   110  	reader, err = create()
   111  	if err != nil {
   112  		if reader != nil {
   113  			reader.Close()
   114  		}
   115  		return nil, nil, err
   116  	}
   117  
   118  	return reader, trySaveReader, nil
   119  }
   120  
   121  func (c *chunkReaderCache) close() {
   122  	reader := (*chunkReaderAt)(atomic.SwapPointer(&c.available, nil))
   123  	if reader != nil {
   124  		reader.Close()
   125  	}
   126  }