github.com/go4org/go4@v0.0.0-20200104003542-c7e774b10ea0/wkfs/gcs/gcs.go (about) 1 /* 2 Copyright 2014 The Perkeep Authors 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package gcs registers a Google Cloud Storage filesystem at the 18 // well-known /gcs/ filesystem path if the current machine is running 19 // on Google Compute Engine. 20 // 21 // It was initially only meant for small files, and as such, it can only 22 // read files smaller than 1MB for now. 23 package gcs // import "go4.org/wkfs/gcs" 24 25 import ( 26 "bytes" 27 "fmt" 28 "io" 29 "io/ioutil" 30 "os" 31 "path" 32 "strings" 33 "time" 34 35 "cloud.google.com/go/compute/metadata" 36 "cloud.google.com/go/storage" 37 "go4.org/wkfs" 38 "golang.org/x/net/context" 39 "golang.org/x/oauth2" 40 "golang.org/x/oauth2/google" 41 "google.golang.org/api/option" 42 ) 43 44 // Max size for all files read, because we use a bytes.Reader as our file 45 // reader, instead of storage.NewReader. This is because we get all wkfs.File 46 // methods for free by embedding a bytes.Reader. This filesystem was only supposed 47 // to be for configuration data only, so this is ok for now. 48 const maxSize = 1 << 20 49 50 func init() { 51 if !metadata.OnGCE() { 52 return 53 } 54 hc, err := google.DefaultClient(oauth2.NoContext) 55 if err != nil { 56 registerBrokenFS(fmt.Errorf("could not get http client for context: %v", err)) 57 return 58 } 59 ctx := context.Background() 60 sc, err := storage.NewClient(ctx, option.WithHTTPClient(hc)) 61 if err != nil { 62 registerBrokenFS(fmt.Errorf("could not get cloud storage client: %v", err)) 63 return 64 } 65 wkfs.RegisterFS("/gcs/", &gcsFS{ 66 ctx: ctx, 67 sc: sc, 68 }) 69 } 70 71 type gcsFS struct { 72 ctx context.Context 73 sc *storage.Client 74 err error // sticky error 75 } 76 77 func registerBrokenFS(err error) { 78 wkfs.RegisterFS("/gcs/", &gcsFS{ 79 err: err, 80 }) 81 } 82 83 func (fs *gcsFS) parseName(name string) (bucket, fileName string, err error) { 84 if fs.err != nil { 85 return "", "", fs.err 86 } 87 name = strings.TrimPrefix(name, "/gcs/") 88 i := strings.Index(name, "/") 89 if i < 0 { 90 return name, "", nil 91 } 92 return name[:i], name[i+1:], nil 93 } 94 95 // Open opens the named file for reading. It returns an error if the file size 96 // is larger than 1 << 20. 97 func (fs *gcsFS) Open(name string) (wkfs.File, error) { 98 bucket, fileName, err := fs.parseName(name) 99 if err != nil { 100 return nil, err 101 } 102 obj := fs.sc.Bucket(bucket).Object(fileName) 103 attrs, err := obj.Attrs(fs.ctx) 104 if err != nil { 105 return nil, err 106 } 107 size := attrs.Size 108 if size > maxSize { 109 return nil, fmt.Errorf("file %s too large (%d bytes) for /gcs/ filesystem", name, size) 110 } 111 rc, err := obj.NewReader(fs.ctx) 112 if err != nil { 113 return nil, err 114 } 115 defer rc.Close() 116 117 slurp, err := ioutil.ReadAll(io.LimitReader(rc, size)) 118 if err != nil { 119 return nil, err 120 } 121 return &file{ 122 name: name, 123 Reader: bytes.NewReader(slurp), 124 }, nil 125 } 126 127 func (fs *gcsFS) Stat(name string) (os.FileInfo, error) { return fs.Lstat(name) } 128 func (fs *gcsFS) Lstat(name string) (os.FileInfo, error) { 129 bucket, fileName, err := fs.parseName(name) 130 if err != nil { 131 return nil, err 132 } 133 attrs, err := fs.sc.Bucket(bucket).Object(fileName).Attrs(fs.ctx) 134 if err == storage.ErrObjectNotExist { 135 return nil, os.ErrNotExist 136 } 137 if err != nil { 138 return nil, err 139 } 140 return &statInfo{ 141 name: attrs.Name, 142 size: attrs.Size, 143 }, nil 144 } 145 146 func (fs *gcsFS) MkdirAll(path string, perm os.FileMode) error { return nil } 147 148 func (fs *gcsFS) OpenFile(name string, flag int, perm os.FileMode) (wkfs.FileWriter, error) { 149 bucket, fileName, err := fs.parseName(name) 150 if err != nil { 151 return nil, err 152 } 153 switch flag { 154 case os.O_WRONLY | os.O_CREATE | os.O_EXCL: 155 case os.O_WRONLY | os.O_CREATE | os.O_TRUNC: 156 default: 157 return nil, fmt.Errorf("Unsupported OpenFlag flag mode %d on Google Cloud Storage", flag) 158 } 159 if flag&os.O_EXCL != 0 { 160 if _, err := fs.Stat(name); err == nil { 161 return nil, os.ErrExist 162 } 163 } 164 // TODO(mpl): consider adding perm to the object's ObjectAttrs.Metadata 165 return fs.sc.Bucket(bucket).Object(fileName).NewWriter(fs.ctx), nil 166 } 167 168 func (fs *gcsFS) Remove(name string) error { 169 bucket, fileName, err := fs.parseName(name) 170 if err != nil { 171 return err 172 } 173 return fs.sc.Bucket(bucket).Object(fileName).Delete(fs.ctx) 174 } 175 176 type statInfo struct { 177 name string 178 size int64 179 isDir bool 180 modtime time.Time 181 } 182 183 func (si *statInfo) IsDir() bool { return si.isDir } 184 func (si *statInfo) ModTime() time.Time { return si.modtime } 185 func (si *statInfo) Mode() os.FileMode { return 0644 } 186 func (si *statInfo) Name() string { return path.Base(si.name) } 187 func (si *statInfo) Size() int64 { return si.size } 188 func (si *statInfo) Sys() interface{} { return nil } 189 190 type file struct { 191 name string 192 *bytes.Reader 193 } 194 195 func (*file) Close() error { return nil } 196 func (f *file) Name() string { return path.Base(f.name) } 197 func (f *file) Stat() (os.FileInfo, error) { 198 panic("Stat not implemented on /gcs/ files yet") 199 }