github.com/esnet/gdg@v0.6.1-0.20240412190737-6b6eba9c14d8/internal/service/storage_cloud.go (about) 1 package service 2 3 import ( 4 "context" 5 "errors" 6 "fmt" 7 "github.com/aws/aws-sdk-go/aws" 8 "github.com/aws/aws-sdk-go/aws/credentials" 9 "github.com/aws/aws-sdk-go/aws/session" 10 "github.com/aws/aws-sdk-go/service/s3" 11 "gocloud.dev/blob" 12 "gocloud.dev/blob/s3blob" 13 "log" 14 "log/slog" 15 "os" 16 "path" 17 "path/filepath" 18 "strings" 19 "sync" 20 ) 21 22 type CloudStorage struct { 23 BucketRef *blob.Bucket 24 BucketName string 25 Prefix string 26 StorageName string 27 } 28 29 const ( 30 CloudType = "cloud_type" 31 BucketName = "bucket_name" 32 Prefix = "prefix" 33 Kind = "kind" 34 Custom = "custom" 35 AccessId = "access_id" 36 SecretKey = "secret_key" 37 Endpoint = "endpoint" 38 Region = "region" 39 SSLEnabled = "ssl_enabled" 40 InitBucket = "init_bucket" 41 ) 42 43 var ( 44 stringEmpty = func(key string) bool { 45 return key == "" 46 } 47 initBucketOnce sync.Once 48 ) 49 50 // getCloudLocation appends prefix to path 51 func (s *CloudStorage) getCloudLocation(fileName string) string { 52 if s.Prefix == "<nil>" { 53 s.Prefix = "" 54 } 55 //Skip if prefix is already in Path. 56 if len(s.Prefix) > 0 && strings.Contains(fileName, s.Prefix) { 57 return fileName 58 } 59 if fileName[0] != '/' && s.Prefix != "" { 60 return path.Join(s.Prefix, "/", fileName) 61 } 62 return path.Join(s.Prefix, fileName) 63 } 64 65 // ReadFile read file from Cloud Provider and return byte array 66 func (s *CloudStorage) ReadFile(filename string) ([]byte, error) { 67 if s.BucketRef == nil { 68 return nil, errors.New("unable to find valid bucket to read file from") 69 } 70 ctx := context.Background() 71 return s.BucketRef.ReadAll(ctx, s.getCloudLocation(filename)) 72 73 } 74 75 // WriteFile persists data to Cloud Provider Storage returning error if operation failed 76 func (s *CloudStorage) WriteFile(filename string, data []byte) error { 77 if s.BucketRef == nil { 78 return errors.New("unable to get valid bucket ") 79 } 80 return s.BucketRef.WriteAll(context.Background(), s.getCloudLocation(filename), data, nil) 81 } 82 83 func (s *CloudStorage) Name() string { 84 return s.StorageName 85 } 86 87 func (s *CloudStorage) FindAllFiles(folder string, fullPath bool) ([]string, error) { 88 if s.BucketRef == nil { 89 return nil, errors.New("unable to find valid bucket to list files from") 90 } 91 folderName := s.getCloudLocation(folder) 92 93 var fileList []string 94 opts := blob.ListOptions{} 95 if s.Prefix != "" { 96 opts.Prefix = folderName 97 } 98 99 iterator := s.BucketRef.List(&opts) 100 for { 101 obj, err := iterator.Next(context.Background()) 102 if err != nil { 103 break 104 } 105 if fullPath { 106 if strings.Contains(obj.Key, folderName) { 107 fileList = append(fileList, obj.Key) 108 } else { 109 slog.Debug("key does not match folder path", "key", obj.Key) 110 } 111 } else { 112 fileList = append(fileList, filepath.Base(obj.Key)) 113 } 114 } 115 116 return fileList, nil 117 } 118 119 func NewCloudStorage(c context.Context) (Storage, error) { 120 var ( 121 err error 122 bucketObj *blob.Bucket 123 errorMsg string 124 ) 125 126 contextVal := c.Value(StorageContext) 127 if contextVal == nil { 128 return nil, errors.New("cannot configure GCP storage, context missing") 129 } 130 appData, ok := contextVal.(map[string]string) 131 if !ok { 132 return nil, errors.New("cannot convert appData to string map") 133 } 134 135 //Pattern specifically for Self hosted S3 compatible instances Minio / Ceph 136 if boolStrCheck(getMapValue(Custom, "false", stringEmpty, appData)) { 137 var sess *session.Session 138 creds := credentials.NewStaticCredentials( 139 getMapValue(AccessId, os.Getenv("AWS_ACCESS_KEY"), stringEmpty, appData), 140 getMapValue(SecretKey, os.Getenv("AWS_SECRET_KEY"), stringEmpty, appData), "") 141 sess, err = session.NewSession(&aws.Config{ 142 Credentials: creds, 143 Endpoint: aws.String(getMapValue(Endpoint, "http://localhost:9000", stringEmpty, appData)), 144 DisableSSL: aws.Bool(getMapValue(SSLEnabled, "false", stringEmpty, appData) != "true"), 145 S3ForcePathStyle: aws.Bool(true), 146 Region: aws.String(getMapValue(Region, "us-east-1", stringEmpty, appData)), 147 }) 148 if err != nil { 149 errorMsg = err.Error() 150 } 151 bucketObj, err = s3blob.OpenBucket(context.Background(), sess, appData["bucket_name"], nil) 152 if err != nil { 153 errorMsg = err.Error() 154 } 155 if err == nil && boolStrCheck(getMapValue(InitBucket, "false", stringEmpty, appData)) { 156 //Attempts to initiate bucket 157 initBucketOnce.Do(func() { 158 client := s3.New(sess) 159 m := s3.CreateBucketInput{ 160 Bucket: aws.String(appData[BucketName]), 161 } 162 //attempt to create bucket 163 _, err := client.CreateBucket(&m) 164 if err != nil { 165 slog.Warn("bucket already exists or cannot be created", "bucket", *m.Bucket) 166 } else { 167 slog.Info("bucket has been created", "bucket", *m.Bucket) 168 } 169 }) 170 171 } 172 173 } else { 174 var cloudURL = fmt.Sprintf("%s://%s", appData["cloud_type"], appData["bucket_name"]) 175 bucketObj, err = blob.OpenBucket(c, cloudURL) 176 errorMsg = fmt.Sprintf("failed to open bucket %s", cloudURL) 177 } 178 179 if err != nil { 180 log.Fatalf("unable to connect to cloud provider, err: %v, message: %s", err, errorMsg) 181 } 182 183 entity := &CloudStorage{ 184 BucketName: appData[BucketName], 185 BucketRef: bucketObj, 186 } 187 188 if val, ok := appData[Prefix]; ok { 189 entity.Prefix = val 190 } 191 192 return entity, nil 193 } 194 195 // boolStrCheck does a more intelligent bool check as yaml values are converted to "1" or "true" depending 196 // on how the user configures quotes the value. 197 func boolStrCheck(val string) bool { 198 return strings.ToLower(val) == "true" || val == "1" 199 200 } 201 202 // getMapValue a generic utility that will get a value from a map and return a default if key does not exist 203 func getMapValue[T comparable](key, defaultValue T, emptyTest func(key T) bool, data map[T]T) T { 204 val, ok := data[key] 205 if ok && !emptyTest(val) { 206 return val 207 } 208 return defaultValue 209 }