github.com/sentienttechnologies/studio-go-runner@v0.0.0-20201118202441-6d21f2ced8ee/internal/runner/minio_local.go (about) 1 // Copyright 2018-2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License. 2 3 package runner 4 5 // This file contains a skeleton wrapper for running a minio 6 // server in-situ and is principally useful for when testing 7 // is being done and a mocked S3 is needed, this case 8 // we provide a full implementation as minio offers a full 9 // implementation 10 11 import ( 12 "bufio" 13 "context" 14 "encoding/json" 15 "flag" 16 "fmt" 17 "io" 18 "io/ioutil" 19 "os" 20 "os/exec" 21 "path" 22 "path/filepath" 23 "sync" 24 "time" 25 26 "github.com/go-stack/stack" 27 "github.com/jjeffery/kv" // MIT License 28 29 "go.uber.org/atomic" 30 31 minio "github.com/minio/minio-go" 32 "github.com/rs/xid" // MIT 33 ) 34 35 // MinioTestServer encapsulates all of the data needed to run 36 // a test minio server instance 37 // 38 type MinioTestServer struct { 39 AccessKeyId string 40 SecretAccessKeyId string 41 Address string 42 Client *minio.Client 43 Ready atomic.Bool 44 } 45 46 func init() { 47 MinioTest = &MinioTestServer{ 48 AccessKeyId: xid.New().String(), 49 SecretAccessKeyId: xid.New().String(), 50 } 51 52 MinioTest.Ready.Store(false) 53 } 54 55 // MinioCfgJson stores configuration information to be written to a disk based configuration 56 // file prior to starting a test minio instance 57 // 58 type MinioCfgJson struct { 59 Version string `json:"version"` 60 Credential struct { 61 AccessKey string `json:"accessKey"` 62 SecretKey string `json:"secretKey"` 63 } `json:"credential"` 64 Region string `json:"region"` 65 Browser string `json:"browser"` 66 Worm string `json:"worm"` 67 Domain string `json:"domain"` 68 Storageclass struct { 69 Standard string `json:"standard"` 70 Rrs string `json:"rrs"` 71 } `json:"storageclass"` 72 Cache struct { 73 Drives []interface{} `json:"drives"` 74 Expiry int `json:"expiry"` 75 Maxuse int `json:"maxuse"` 76 Exclude []interface{} `json:"exclude"` 77 } `json:"cache"` 78 } 79 80 var ( 81 // MinioTest encapsulates a running minio instance 82 MinioTest *MinioTestServer 83 84 minioAccessKey = flag.String("minio-access-key", "", "Specifies an AWS access key for a minio server used during testing, accepts ${} env var expansion") 85 minioSecretKey = flag.String("minio-secret-key", "", "Specifies an AWS secret access key for a minio server used during testing, accepts ${} env var expansion") 86 minioTestServer = flag.String("minio-test-server", "", "Specifies an existing minio server that is available for testing purposes, accepts ${} env var expansion") 87 ) 88 89 // TmpDirFile creates a temporary file of a given size and passes back the directory it 90 // was generated in along with its name 91 func TmpDirFile(size int64) (dir string, fn string, err kv.Error) { 92 93 tmpDir, errGo := ioutil.TempDir("", xid.New().String()) 94 if errGo != nil { 95 return "", "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 96 } 97 98 fn = path.Join(tmpDir, xid.New().String()) 99 f, errGo := os.Create(fn) 100 if errGo != nil { 101 return "", "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 102 } 103 defer func() { _ = f.Close() }() 104 105 if errGo = f.Truncate(size); errGo != nil { 106 return "", "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 107 } 108 109 return tmpDir, fn, nil 110 } 111 112 // UploadTestFile will create and upload a file of a given size to the MinioTest server to 113 // allow test cases to exercise functionality based on S3 114 // 115 func (mts *MinioTestServer) UploadTestFile(bucket string, key string, size int64) (err kv.Error) { 116 tmpDir, fn, err := TmpDirFile(size) 117 if err != nil { 118 return err 119 } 120 defer func() { 121 if errGo := os.RemoveAll(tmpDir); errGo != nil { 122 fmt.Printf("%s %#v", tmpDir, errGo) 123 } 124 }() 125 126 // Get the Minio Test Server instance and sent it some random data while generating 127 // a hash 128 return mts.Upload(bucket, key, fn) 129 } 130 131 // SetPublic can be used to enable public access to a bucket 132 // 133 func (mts *MinioTestServer) SetPublic(bucket string) (err kv.Error) { 134 if !mts.Ready.Load() { 135 return kv.NewError("server not ready").With("host", mts.Address).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()) 136 } 137 policy := `{ 138 "Version": "2012-10-17", 139 "Statement": [ 140 { 141 "Action": [ 142 "s3:GetObject" 143 ], 144 "Effect": "Allow", 145 "Principal": { 146 "AWS": [ 147 "*" 148 ] 149 }, 150 "Resource": [ 151 "arn:aws:s3:::%s/*" 152 ], 153 "Sid": "" 154 } 155 ] 156 }` 157 158 if errGo := mts.Client.SetBucketPolicy(bucket, fmt.Sprintf(policy, bucket)); errGo != nil { 159 return kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()) 160 } 161 return nil 162 } 163 164 // RemoveBucketAll empties the identified bucket on the minio test server 165 // identified by the mtx receiver variable 166 // 167 func (mts *MinioTestServer) RemoveBucketAll(bucket string) (errs []kv.Error) { 168 169 if !mts.Ready.Load() { 170 errs = append(errs, kv.NewError("server not ready").With("host", mts.Address).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 171 return errs 172 } 173 174 exists, errGo := mts.Client.BucketExists(bucket) 175 if errGo != nil { 176 errs = append(errs, kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 177 return errs 178 } 179 if !exists { 180 return nil 181 } 182 183 doneC := make(chan struct{}) 184 defer close(doneC) 185 186 // This channel is used to send keys on that will be deleted in the background. 187 // We dont yet have large buckets that need deleting so the asynchronous 188 // features of this are not used but they very well could be used in the future. 189 keysC := make(chan string) 190 errLock := sync.Mutex{} 191 192 // Send object names that are needed to be removed though a worker style channel 193 // that might be a little slower, but for our case with small buckets is not 194 // yet an issue so leave things as they are 195 go func() { 196 defer close(keysC) 197 198 // List all objects from a bucket-name with a matching prefix. 199 for object := range mts.Client.ListObjectsV2(bucket, "", true, doneC) { 200 if object.Err != nil { 201 errLock.Lock() 202 errs = append(errs, kv.Wrap(object.Err).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 203 errLock.Unlock() 204 continue 205 } 206 select { 207 case keysC <- object.Key: 208 case <-time.After(2 * time.Second): 209 errLock.Lock() 210 errs = append(errs, kv.NewError("object delete timeout").With("key", object.Key).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 211 errLock.Unlock() 212 // Giveup deleting an object if it blocks everything 213 } 214 } 215 for object := range mts.Client.ListIncompleteUploads(bucket, "", true, doneC) { 216 if object.Err != nil { 217 errLock.Lock() 218 errs = append(errs, kv.Wrap(object.Err).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 219 errLock.Unlock() 220 continue 221 } 222 select { 223 case keysC <- object.Key: 224 case <-time.After(2 * time.Second): 225 errLock.Lock() 226 errs = append(errs, kv.NewError("partial upload delete timeout").With("key", object.Key).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 227 errLock.Unlock() 228 // Giveup deleting an object if it blocks everything 229 } 230 } 231 }() 232 233 for errMinio := range mts.Client.RemoveObjects(bucket, keysC) { 234 if errMinio.Err.Error() == "EOF" { 235 break 236 } 237 errLock.Lock() 238 errs = append(errs, kv.NewError(errMinio.Err.Error()).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 239 errLock.Unlock() 240 } 241 242 errGo = mts.Client.RemoveBucket(bucket) 243 if errGo != nil { 244 errs = append(errs, kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())) 245 } 246 return errs 247 } 248 249 // Upload will take the nominated file, file parameter, and will upload it to the bucket and key 250 // pair on the server identified by the mtx receiver variable 251 // 252 func (mts *MinioTestServer) Upload(bucket string, key string, file string) (err kv.Error) { 253 254 if !mts.Ready.Load() { 255 return kv.NewError("server not ready").With("host", mts.Address).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()) 256 } 257 258 f, errGo := os.Open(filepath.Clean(file)) 259 if errGo != nil { 260 return kv.Wrap(errGo, "Upload passed a non-existent file name").With("file", file).With("stack", stack.Trace().TrimRuntime()) 261 } 262 defer f.Close() 263 264 exists, errGo := mts.Client.BucketExists(bucket) 265 if errGo != nil { 266 return kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()) 267 } 268 if !exists { 269 if errGo = mts.Client.MakeBucket(bucket, ""); errGo != nil { 270 return kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()) 271 } 272 } 273 274 _, errGo = mts.Client.PutObject(bucket, key, bufio.NewReader(f), -1, 275 minio.PutObjectOptions{ 276 ContentType: "application/octet-stream", 277 CacheControl: "max-age=600", 278 }) 279 280 if errGo != nil { 281 return kv.Wrap(errGo).With("bucket", bucket).With("key", key).With("file", file).With("stack", stack.Trace().TrimRuntime()) 282 } 283 284 return nil 285 } 286 287 func writeCfg(mts *MinioTestServer) (cfgDir string, err kv.Error) { 288 // Initialize a configuration directory for the minio server 289 // complete with the json configuration containing the credentials 290 // for the test server 291 cfgDir, errGo := ioutil.TempDir("", xid.New().String()) 292 if errGo != nil { 293 return "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 294 } 295 cfg := MinioCfgJson{} 296 cfg.Version = "26" 297 cfg.Credential.AccessKey = mts.AccessKeyId 298 cfg.Credential.SecretKey = mts.SecretAccessKeyId 299 cfg.Worm = "off" 300 301 result, errGo := json.MarshalIndent(cfg, "", " ") 302 if errGo != nil { 303 return "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 304 } 305 if errGo = ioutil.WriteFile(path.Join(cfgDir, "config.json"), result, 0666); errGo != nil { 306 return "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 307 } 308 return cfgDir, nil 309 } 310 311 // startLocalMinio will fork off a running minio server with an empty data store 312 // that can be used for testing purposes. This function does not block, 313 // however it does start a go routine 314 // 315 func startLocalMinio(ctx context.Context, retainWorkingDirs bool, errC chan kv.Error) { 316 317 // Default to the case that another pod for external host has a running minio server for us 318 // to use during testing 319 if len(*minioTestServer) != 0 { 320 MinioTest.Address = os.ExpandEnv(*minioTestServer) 321 } 322 if len(*minioAccessKey) != 0 { 323 MinioTest.AccessKeyId = os.ExpandEnv(*minioAccessKey) 324 } 325 if len(*minioSecretKey) != 0 { 326 MinioTest.SecretAccessKeyId = os.ExpandEnv(*minioSecretKey) 327 } 328 329 // If we dont have a k8s based minio server specified for our test try try using a local 330 // minio instance within the container or machine the test is run on 331 // 332 if len(*minioTestServer) == 0 { 333 // First check that the minio executable is present on the test system 334 // 335 // We are using the executable because the dependency hierarchy of minio 336 // is very tangled and so it is very hard to embeed for now, Go 1.10.3 337 execPath, errGo := exec.LookPath("minio") 338 if errGo != nil { 339 errC <- kv.Wrap(errGo, "please install minio into your path").With("path", os.Getenv("PATH")).With("stack", stack.Trace().TrimRuntime()) 340 return 341 } 342 343 // Get a free server listening port for our test 344 port, err := GetFreePort("127.0.0.1:0") 345 if err != nil { 346 errC <- err 347 return 348 } 349 350 MinioTest.Address = fmt.Sprintf("127.0.0.1:%d", port) 351 352 // Initialize the data directory for the file server 353 storageDir, errGo := ioutil.TempDir("", xid.New().String()) 354 if errGo != nil { 355 errC <- kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 356 return 357 } 358 filepath.Clean(storageDir) 359 360 if errGo = os.Chmod(storageDir, 0700); errGo != nil { 361 errC <- kv.Wrap(errGo).With("storageDir", storageDir).With("stack", stack.Trace().TrimRuntime()) 362 os.RemoveAll(storageDir) 363 return 364 } 365 366 // If we see no credentials were supplied for a local test, the typical case 367 // then supply some defaults 368 if len(MinioTest.AccessKeyId) == 0 { 369 MinioTest.AccessKeyId = "UserUser" 370 } 371 if len(MinioTest.SecretAccessKeyId) == 0 { 372 MinioTest.SecretAccessKeyId = "PasswordPassword" 373 } 374 375 // Now write a cfg file out for our desired minio 376 // configuration 377 cfgDir, err := writeCfg(MinioTest) 378 if err != nil { 379 errC <- err 380 return 381 } 382 cfgDir = filepath.Clean(cfgDir) 383 384 go func() { 385 cmdCtx, cancel := context.WithCancel(ctx) 386 // When the main process stops kill our cmd runner for minio 387 defer cancel() 388 389 // #nosec 390 cmd := exec.CommandContext(cmdCtx, filepath.Clean(execPath), 391 "server", 392 "--address", MinioTest.Address, 393 "--config-dir", cfgDir, 394 storageDir, 395 ) 396 397 stdout, errGo := cmd.StdoutPipe() 398 if errGo != nil { 399 errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime()) 400 } 401 stderr, errGo := cmd.StderrPipe() 402 if errGo != nil { 403 errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime()) 404 } 405 // Non-blockingly echo command output to terminal 406 go io.Copy(os.Stdout, stdout) 407 go io.Copy(os.Stderr, stderr) 408 409 if errGo = cmd.Start(); errGo != nil { 410 errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime()) 411 } 412 413 if errGo = cmd.Wait(); errGo != nil { 414 if errGo.Error() != "signal: killed" { 415 errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime()) 416 } 417 } 418 419 fmt.Printf("%v\n", kv.NewError("minio terminated").With("cfgDir", cfgDir, "storageDir", storageDir).With("stack", stack.Trace().TrimRuntime())) 420 421 if !retainWorkingDirs { 422 os.RemoveAll(storageDir) 423 os.RemoveAll(cfgDir) 424 } 425 }() 426 } 427 428 startMinioClient(ctx, errC) 429 } 430 431 func startMinioClient(ctx context.Context, errC chan kv.Error) { 432 // Wait for the server to start by checking the listen port using 433 // TCP 434 check := time.NewTicker(time.Second) 435 defer check.Stop() 436 437 for { 438 select { 439 case <-check.C: 440 client, errGo := minio.New(MinioTest.Address, MinioTest.AccessKeyId, 441 MinioTest.SecretAccessKeyId, false) 442 if errGo != nil { 443 errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime()) 444 continue 445 } 446 MinioTest.Client = client 447 MinioTest.Ready.Store(true) 448 return 449 case <-ctx.Done(): 450 return 451 } 452 } 453 } 454 455 // IsAlive is used to test if the expected minio local test server is alive 456 // 457 func (mts *MinioTestServer) IsAlive(ctx context.Context) (alive bool, err kv.Error) { 458 459 check := time.NewTicker(5 * time.Second) 460 defer check.Stop() 461 462 for { 463 select { 464 case <-ctx.Done(): 465 return false, err 466 case <-check.C: 467 if !mts.Ready.Load() || mts.Client == nil { 468 continue 469 } 470 _, errGo := mts.Client.BucketExists(xid.New().String()) 471 if errGo == nil { 472 return true, nil 473 } 474 err = kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime()) 475 } 476 } 477 } 478 479 // InitTestingMinio will fork a minio server that can he used for staging and test 480 // in a manner that also wraps an error reporting channel and a means of 481 // stopping it 482 // 483 func InitTestingMinio(ctx context.Context, retainWorkingDirs bool) (errC chan kv.Error) { 484 errC = make(chan kv.Error, 5) 485 486 startLocalMinio(ctx, retainWorkingDirs, errC) 487 488 return errC 489 }