github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/actions/lua/storage/gcloud/gcs.go (about)

     1  package gcloud
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"net/url"
     8  	"strings"
     9  
    10  	"cloud.google.com/go/storage"
    11  	"github.com/Shopify/go-lua"
    12  	"github.com/treeverse/lakefs/pkg/actions/lua/path"
    13  	"github.com/treeverse/lakefs/pkg/actions/lua/util"
    14  	"golang.org/x/oauth2/google"
    15  	"google.golang.org/api/option"
    16  )
    17  
    18  const (
    19  	// googleAuthCloudPlatform - Cloud Storage authentication https://cloud.google.com/storage/docs/authentication
    20  	googleAuthCloudPlatform = "https://www.googleapis.com/auth/cloud-platform"
    21  )
    22  
    23  var ErrInvalidGCSURI = errors.New("invalid Google Cloud Storage URI")
    24  
    25  func Open(l *lua.State, ctx context.Context) {
    26  	open := func(l *lua.State) int {
    27  		lua.NewLibrary(l, []lua.RegistryFunction{
    28  			{Name: "gs_client", Function: newGSClient(ctx)},
    29  		})
    30  		return 1
    31  	}
    32  	lua.Require(l, "gcloud", open, false)
    33  	l.Pop(1)
    34  }
    35  
    36  func newGSClient(ctx context.Context) lua.Function {
    37  	return func(l *lua.State) int {
    38  		json := lua.CheckString(l, 1)
    39  		c := &GSClient{
    40  			JSON: json,
    41  			ctx:  ctx,
    42  		}
    43  		l.NewTable()
    44  		for name, goFn := range functions {
    45  			// -1: tbl
    46  			l.PushGoFunction(goFn(c))
    47  			// -1: fn, -2:tbl
    48  			l.SetField(-2, name)
    49  		}
    50  		return 1
    51  	}
    52  }
    53  
    54  type GSClient struct {
    55  	JSON string
    56  	ctx  context.Context
    57  }
    58  
    59  func (c *GSClient) client() (*storage.Client, error) {
    60  	cred, err := google.CredentialsFromJSON(c.ctx, []byte(c.JSON), googleAuthCloudPlatform)
    61  	if err != nil {
    62  		return nil, err
    63  	}
    64  	return storage.NewClient(c.ctx, option.WithCredentials(cred))
    65  }
    66  
    67  var functions = map[string]func(client *GSClient) lua.Function{
    68  	"write_fuse_symlink": writeFuseSymlink,
    69  }
    70  
    71  func writeFuseSymlink(c *GSClient) lua.Function {
    72  	return func(l *lua.State) int {
    73  		// convert the relative physical address with the mount point
    74  		physicalAddress := lua.CheckString(l, 1)
    75  		outputAddress := lua.CheckString(l, 2)
    76  		mountInfo, err := util.PullStringTable(l, 3)
    77  		if err != nil {
    78  			lua.Errorf(l, "could not read mount info: %s", err.Error())
    79  		}
    80  
    81  		// let's resolve the path:
    82  		if fromValue, removeFrom := mountInfo["from"]; removeFrom {
    83  			physicalAddress = strings.TrimPrefix(physicalAddress, fromValue)
    84  		}
    85  		if toValue, prependTo := mountInfo["to"]; prependTo {
    86  			physicalAddress = path.Join("/", toValue, physicalAddress)
    87  		}
    88  
    89  		// write the object
    90  		client, err := c.client()
    91  		if err != nil {
    92  			lua.Errorf(l, "could not initialize google storage client: %s", err.Error())
    93  		}
    94  		defer func() { _ = client.Close() }()
    95  
    96  		obj, err := asObject(client, outputAddress)
    97  		if err != nil {
    98  			lua.Errorf(l, "could not parse destination object \"%s\": %s", outputAddress, err.Error())
    99  		}
   100  
   101  		w := obj.NewWriter(c.ctx)
   102  		w.ObjectAttrs.Metadata = map[string]string{
   103  			"gcsfuse_symlink_target": physicalAddress,
   104  		}
   105  		err = w.Close()
   106  		if err != nil {
   107  			lua.Errorf(l, "could not close object \"%s\": %s", outputAddress, err.Error())
   108  		}
   109  		return 0
   110  	}
   111  }
   112  
   113  func asObject(client *storage.Client, gsURI string) (*storage.ObjectHandle, error) {
   114  	parsed, err := url.Parse(gsURI)
   115  	if err != nil {
   116  		return nil, fmt.Errorf("unable to parse URI: %s: %w", gsURI, ErrInvalidGCSURI)
   117  	}
   118  	if parsed.Scheme != "gs" {
   119  		return nil, ErrInvalidGCSURI
   120  	}
   121  	bucket := parsed.Host
   122  	objectPath := strings.TrimPrefix(parsed.Path, path.SEPARATOR)
   123  	return client.Bucket(bucket).Object(objectPath), nil
   124  }