github.com/GoogleCloudPlatform/testgrid@v0.0.174/util/gcs/local_gcs.go (about)

     1  /*
     2  Copyright 2021 The TestGrid Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package gcs
    18  
    19  import (
    20  	"context"
    21  	"io"
    22  	"io/ioutil"
    23  	"net/url"
    24  	"os"
    25  	"path/filepath"
    26  	"regexp"
    27  	"strings"
    28  
    29  	"cloud.google.com/go/storage"
    30  	"google.golang.org/api/iterator"
    31  )
    32  
    33  var (
    34  	_ Client = &localClient{} // Ensure this implements interface
    35  )
    36  
    37  type localIterator struct {
    38  	files []os.FileInfo
    39  	dir   string
    40  	index int
    41  }
    42  
    43  func convertIsNotExistsErr(err error) error {
    44  	if os.IsNotExist(err) {
    45  		return storage.ErrObjectNotExist
    46  	}
    47  	return err
    48  }
    49  
    50  // See https://en.wikipedia.org/wiki/File_URI_scheme#How_many_slashes
    51  var fileRegex = regexp.MustCompile(`file:\/+`)
    52  
    53  func cleanFilepath(path Path) string {
    54  	p := fileRegex.ReplaceAllString(path.String(), "/")
    55  	// TODO(michelle192837): Handle URLs vs. filepaths gracefully.
    56  	p, err := url.PathUnescape(p)
    57  	if err != nil {
    58  		return ""
    59  	}
    60  	return p
    61  }
    62  
    63  func (li *localIterator) Next() (*storage.ObjectAttrs, error) {
    64  	defer func() { li.index++ }()
    65  	if li.index >= len(li.files) {
    66  		return nil, iterator.Done
    67  	}
    68  	info := li.files[li.index]
    69  	p, err := NewPath(filepath.Join(li.dir, info.Name()))
    70  	if err != nil {
    71  		return nil, err
    72  	}
    73  	return objectAttrs(info, *p), nil
    74  }
    75  
    76  // NewLocalClient returns a GCSUploadClient for the storage.Client.
    77  func NewLocalClient() ConditionalClient {
    78  	return localClient{nil, nil}
    79  }
    80  
    81  type localClient struct {
    82  	readCond  *storage.Conditions
    83  	writeCond *storage.Conditions
    84  }
    85  
    86  func (lc localClient) If(_, _ *storage.Conditions) ConditionalClient {
    87  	return NewLocalClient()
    88  }
    89  
    90  func (lc localClient) Copy(ctx context.Context, from, to Path) (*storage.ObjectAttrs, error) {
    91  	buf, err := ioutil.ReadFile(cleanFilepath(from))
    92  	if err != nil {
    93  		return nil, convertIsNotExistsErr(err)
    94  	}
    95  	return lc.Upload(ctx, to, buf, false, "")
    96  }
    97  
    98  func (lc localClient) Open(ctx context.Context, path Path) (io.ReadCloser, *storage.ReaderObjectAttrs, error) {
    99  	r, err := os.Open(cleanFilepath(path))
   100  	return r, &storage.ReaderObjectAttrs{}, convertIsNotExistsErr(err)
   101  }
   102  
   103  func (lc localClient) Objects(ctx context.Context, path Path, delimiter, startOffset string) Iterator {
   104  	p := cleanFilepath(path)
   105  	if !strings.HasSuffix(p, "/") {
   106  		p += "/"
   107  	}
   108  	files, err := ioutil.ReadDir(p)
   109  	if err != nil {
   110  		return &localIterator{}
   111  	}
   112  	return &localIterator{
   113  		dir:   filepath.Dir(p),
   114  		files: files,
   115  	}
   116  }
   117  
   118  func (lc localClient) Upload(ctx context.Context, path Path, buf []byte, _ bool, _ string) (*storage.ObjectAttrs, error) {
   119  	cleanPath := cleanFilepath(path)
   120  	dirName := filepath.Dir(cleanPath)
   121  	// swallow this error; dir may already exist, but it must exist to WriteFile for local storage only
   122  	os.MkdirAll(dirName, os.ModePerm)
   123  
   124  	err := ioutil.WriteFile(cleanPath, buf, 0666)
   125  	if err != nil {
   126  		return nil, convertIsNotExistsErr(err)
   127  	}
   128  	return lc.Stat(ctx, path)
   129  }
   130  
   131  func (lc localClient) Stat(ctx context.Context, path Path) (*storage.ObjectAttrs, error) {
   132  	info, err := os.Stat(cleanFilepath(path))
   133  	if err != nil {
   134  		return nil, convertIsNotExistsErr(err)
   135  	}
   136  	return objectAttrs(info, path), nil
   137  }
   138  
   139  func objectAttrs(info os.FileInfo, path Path) *storage.ObjectAttrs {
   140  	return &storage.ObjectAttrs{
   141  		Bucket:  path.Bucket(),
   142  		Name:    path.Object(),
   143  		Size:    info.Size(),
   144  		Updated: info.ModTime(),
   145  	}
   146  }