github.com/go4org/go4@v0.0.0-20200104003542-c7e774b10ea0/wkfs/gcs/gcs.go (about)

     1  /*
     2  Copyright 2014 The Perkeep Authors
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8       http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  // Package gcs registers a Google Cloud Storage filesystem at the
    18  // well-known /gcs/ filesystem path if the current machine is running
    19  // on Google Compute Engine.
    20  //
    21  // It was initially only meant for small files, and as such, it can only
    22  // read files smaller than 1MB for now.
    23  package gcs // import "go4.org/wkfs/gcs"
    24  
    25  import (
    26  	"bytes"
    27  	"fmt"
    28  	"io"
    29  	"io/ioutil"
    30  	"os"
    31  	"path"
    32  	"strings"
    33  	"time"
    34  
    35  	"cloud.google.com/go/compute/metadata"
    36  	"cloud.google.com/go/storage"
    37  	"go4.org/wkfs"
    38  	"golang.org/x/net/context"
    39  	"golang.org/x/oauth2"
    40  	"golang.org/x/oauth2/google"
    41  	"google.golang.org/api/option"
    42  )
    43  
    44  // Max size for all files read, because we use a bytes.Reader as our file
    45  // reader, instead of storage.NewReader. This is because we get all wkfs.File
    46  // methods for free by embedding a bytes.Reader. This filesystem was only supposed
    47  // to be for configuration data only, so this is ok for now.
    48  const maxSize = 1 << 20
    49  
    50  func init() {
    51  	if !metadata.OnGCE() {
    52  		return
    53  	}
    54  	hc, err := google.DefaultClient(oauth2.NoContext)
    55  	if err != nil {
    56  		registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err))
    57  		return
    58  	}
    59  	ctx := context.Background()
    60  	sc, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
    61  	if err != nil {
    62  		registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err))
    63  		return
    64  	}
    65  	wkfs.RegisterFS("/gcs/", &gcsFS{
    66  		ctx: ctx,
    67  		sc:  sc,
    68  	})
    69  }
    70  
    71  type gcsFS struct {
    72  	ctx context.Context
    73  	sc  *storage.Client
    74  	err error // sticky error
    75  }
    76  
    77  func registerBrokenFS(err error) {
    78  	wkfs.RegisterFS("/gcs/", &gcsFS{
    79  		err: err,
    80  	})
    81  }
    82  
    83  func (fs *gcsFS) parseName(name string) (bucket, fileName string, err error) {
    84  	if fs.err != nil {
    85  		return "", "", fs.err
    86  	}
    87  	name = strings.TrimPrefix(name, "/gcs/")
    88  	i := strings.Index(name, "/")
    89  	if i < 0 {
    90  		return name, "", nil
    91  	}
    92  	return name[:i], name[i+1:], nil
    93  }
    94  
    95  // Open opens the named file for reading. It returns an error if the file size
    96  // is larger than 1 << 20.
    97  func (fs *gcsFS) Open(name string) (wkfs.File, error) {
    98  	bucket, fileName, err := fs.parseName(name)
    99  	if err != nil {
   100  		return nil, err
   101  	}
   102  	obj := fs.sc.Bucket(bucket).Object(fileName)
   103  	attrs, err := obj.Attrs(fs.ctx)
   104  	if err != nil {
   105  		return nil, err
   106  	}
   107  	size := attrs.Size
   108  	if size > maxSize {
   109  		return nil, fmt.Errorf("file %s too large (%d bytes) for /gcs/ filesystem", name, size)
   110  	}
   111  	rc, err := obj.NewReader(fs.ctx)
   112  	if err != nil {
   113  		return nil, err
   114  	}
   115  	defer rc.Close()
   116  
   117  	slurp, err := ioutil.ReadAll(io.LimitReader(rc, size))
   118  	if err != nil {
   119  		return nil, err
   120  	}
   121  	return &file{
   122  		name:   name,
   123  		Reader: bytes.NewReader(slurp),
   124  	}, nil
   125  }
   126  
   127  func (fs *gcsFS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) }
   128  func (fs *gcsFS) Lstat(name string) (os.FileInfo, error) {
   129  	bucket, fileName, err := fs.parseName(name)
   130  	if err != nil {
   131  		return nil, err
   132  	}
   133  	attrs, err := fs.sc.Bucket(bucket).Object(fileName).Attrs(fs.ctx)
   134  	if err == storage.ErrObjectNotExist {
   135  		return nil, os.ErrNotExist
   136  	}
   137  	if err != nil {
   138  		return nil, err
   139  	}
   140  	return &statInfo{
   141  		name: attrs.Name,
   142  		size: attrs.Size,
   143  	}, nil
   144  }
   145  
   146  func (fs *gcsFS) MkdirAll(path string, perm os.FileMode) error { return nil }
   147  
   148  func (fs *gcsFS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) {
   149  	bucket, fileName, err := fs.parseName(name)
   150  	if err != nil {
   151  		return nil, err
   152  	}
   153  	switch flag {
   154  	case os.O_WRONLY | os.O_CREATE | os.O_EXCL:
   155  	case os.O_WRONLY | os.O_CREATE | os.O_TRUNC:
   156  	default:
   157  		return nil, fmt.Errorf("Unsupported OpenFlag flag mode %d on Google Cloud Storage", flag)
   158  	}
   159  	if flag&os.O_EXCL != 0 {
   160  		if _, err := fs.Stat(name); err == nil {
   161  			return nil, os.ErrExist
   162  		}
   163  	}
   164  	// TODO(mpl): consider adding perm to the object's ObjectAttrs.Metadata
   165  	return fs.sc.Bucket(bucket).Object(fileName).NewWriter(fs.ctx), nil
   166  }
   167  
   168  func (fs *gcsFS) Remove(name string) error {
   169  	bucket, fileName, err := fs.parseName(name)
   170  	if err != nil {
   171  		return err
   172  	}
   173  	return fs.sc.Bucket(bucket).Object(fileName).Delete(fs.ctx)
   174  }
   175  
   176  type statInfo struct {
   177  	name    string
   178  	size    int64
   179  	isDir   bool
   180  	modtime time.Time
   181  }
   182  
   183  func (si *statInfo) IsDir() bool        { return si.isDir }
   184  func (si *statInfo) ModTime() time.Time { return si.modtime }
   185  func (si *statInfo) Mode() os.FileMode  { return 0644 }
   186  func (si *statInfo) Name() string       { return path.Base(si.name) }
   187  func (si *statInfo) Size() int64        { return si.size }
   188  func (si *statInfo) Sys() interface{}   { return nil }
   189  
   190  type file struct {
   191  	name string
   192  	*bytes.Reader
   193  }
   194  
   195  func (*file) Close() error   { return nil }
   196  func (f *file) Name() string { return path.Base(f.name) }
   197  func (f *file) Stat() (os.FileInfo, error) {
   198  	panic("Stat not implemented on /gcs/ files yet")
   199  }