github.com/sentienttechnologies/studio-go-runner@v0.0.0-20201118202441-6d21f2ced8ee/internal/runner/minio_local.go (about)

     1  // Copyright 2018-2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License.
     2  
     3  package runner
     4  
     5  // This file contains a skeleton wrapper for running a minio
     6  // server in-situ and is principally useful for when testing
     7  // is being done and a mocked S3 is needed, this case
     8  // we provide a full implementation as minio offers a full
     9  // implementation
    10  
    11  import (
    12  	"bufio"
    13  	"context"
    14  	"encoding/json"
    15  	"flag"
    16  	"fmt"
    17  	"io"
    18  	"io/ioutil"
    19  	"os"
    20  	"os/exec"
    21  	"path"
    22  	"path/filepath"
    23  	"sync"
    24  	"time"
    25  
    26  	"github.com/go-stack/stack"
    27  	"github.com/jjeffery/kv" // MIT License
    28  
    29  	"go.uber.org/atomic"
    30  
    31  	minio "github.com/minio/minio-go"
    32  	"github.com/rs/xid" // MIT
    33  )
    34  
    35  // MinioTestServer encapsulates all of the data needed to run
    36  // a test minio server instance
    37  //
    38  type MinioTestServer struct {
    39  	AccessKeyId       string
    40  	SecretAccessKeyId string
    41  	Address           string
    42  	Client            *minio.Client
    43  	Ready             atomic.Bool
    44  }
    45  
    46  func init() {
    47  	MinioTest = &MinioTestServer{
    48  		AccessKeyId:       xid.New().String(),
    49  		SecretAccessKeyId: xid.New().String(),
    50  	}
    51  
    52  	MinioTest.Ready.Store(false)
    53  }
    54  
    55  // MinioCfgJson stores configuration information to be written to a disk based configuration
    56  // file prior to starting a test minio instance
    57  //
    58  type MinioCfgJson struct {
    59  	Version    string `json:"version"`
    60  	Credential struct {
    61  		AccessKey string `json:"accessKey"`
    62  		SecretKey string `json:"secretKey"`
    63  	} `json:"credential"`
    64  	Region       string `json:"region"`
    65  	Browser      string `json:"browser"`
    66  	Worm         string `json:"worm"`
    67  	Domain       string `json:"domain"`
    68  	Storageclass struct {
    69  		Standard string `json:"standard"`
    70  		Rrs      string `json:"rrs"`
    71  	} `json:"storageclass"`
    72  	Cache struct {
    73  		Drives  []interface{} `json:"drives"`
    74  		Expiry  int           `json:"expiry"`
    75  		Maxuse  int           `json:"maxuse"`
    76  		Exclude []interface{} `json:"exclude"`
    77  	} `json:"cache"`
    78  }
    79  
    80  var (
    81  	// MinioTest encapsulates a running minio instance
    82  	MinioTest *MinioTestServer
    83  
    84  	minioAccessKey  = flag.String("minio-access-key", "", "Specifies an AWS access key for a minio server used during testing, accepts ${} env var expansion")
    85  	minioSecretKey  = flag.String("minio-secret-key", "", "Specifies an AWS secret access key for a minio server used during testing, accepts ${} env var expansion")
    86  	minioTestServer = flag.String("minio-test-server", "", "Specifies an existing minio server that is available for testing purposes, accepts ${} env var expansion")
    87  )
    88  
    89  // TmpDirFile creates a temporary file of a given size and passes back the directory it
    90  // was generated in along with its name
    91  func TmpDirFile(size int64) (dir string, fn string, err kv.Error) {
    92  
    93  	tmpDir, errGo := ioutil.TempDir("", xid.New().String())
    94  	if errGo != nil {
    95  		return "", "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
    96  	}
    97  
    98  	fn = path.Join(tmpDir, xid.New().String())
    99  	f, errGo := os.Create(fn)
   100  	if errGo != nil {
   101  		return "", "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   102  	}
   103  	defer func() { _ = f.Close() }()
   104  
   105  	if errGo = f.Truncate(size); errGo != nil {
   106  		return "", "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   107  	}
   108  
   109  	return tmpDir, fn, nil
   110  }
   111  
   112  // UploadTestFile will create and upload a file of a given size to the MinioTest server to
   113  // allow test cases to exercise functionality based on S3
   114  //
   115  func (mts *MinioTestServer) UploadTestFile(bucket string, key string, size int64) (err kv.Error) {
   116  	tmpDir, fn, err := TmpDirFile(size)
   117  	if err != nil {
   118  		return err
   119  	}
   120  	defer func() {
   121  		if errGo := os.RemoveAll(tmpDir); errGo != nil {
   122  			fmt.Printf("%s %#v", tmpDir, errGo)
   123  		}
   124  	}()
   125  
   126  	// Get the Minio Test Server instance and sent it some random data while generating
   127  	// a hash
   128  	return mts.Upload(bucket, key, fn)
   129  }
   130  
   131  // SetPublic can be used to enable public access to a bucket
   132  //
   133  func (mts *MinioTestServer) SetPublic(bucket string) (err kv.Error) {
   134  	if !mts.Ready.Load() {
   135  		return kv.NewError("server not ready").With("host", mts.Address).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())
   136  	}
   137  	policy := `{
   138    "Version": "2012-10-17",
   139    "Statement": [
   140      {
   141        "Action": [
   142          "s3:GetObject"
   143        ],
   144        "Effect": "Allow",
   145        "Principal": {
   146          "AWS": [
   147            "*"
   148          ]
   149        },
   150        "Resource": [
   151          "arn:aws:s3:::%s/*"
   152        ],
   153        "Sid": ""
   154      }
   155    ]
   156  }`
   157  
   158  	if errGo := mts.Client.SetBucketPolicy(bucket, fmt.Sprintf(policy, bucket)); errGo != nil {
   159  		return kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())
   160  	}
   161  	return nil
   162  }
   163  
   164  // RemoveBucketAll empties the identified bucket on the minio test server
   165  // identified by the mtx receiver variable
   166  //
   167  func (mts *MinioTestServer) RemoveBucketAll(bucket string) (errs []kv.Error) {
   168  
   169  	if !mts.Ready.Load() {
   170  		errs = append(errs, kv.NewError("server not ready").With("host", mts.Address).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   171  		return errs
   172  	}
   173  
   174  	exists, errGo := mts.Client.BucketExists(bucket)
   175  	if errGo != nil {
   176  		errs = append(errs, kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   177  		return errs
   178  	}
   179  	if !exists {
   180  		return nil
   181  	}
   182  
   183  	doneC := make(chan struct{})
   184  	defer close(doneC)
   185  
   186  	// This channel is used to send keys on that will be deleted in the background.
   187  	// We dont yet have large buckets that need deleting so the asynchronous
   188  	// features of this are not used but they very well could be used in the future.
   189  	keysC := make(chan string)
   190  	errLock := sync.Mutex{}
   191  
   192  	// Send object names that are needed to be removed though a worker style channel
   193  	// that might be a little slower, but for our case with small buckets is not
   194  	// yet an issue so leave things as they are
   195  	go func() {
   196  		defer close(keysC)
   197  
   198  		// List all objects from a bucket-name with a matching prefix.
   199  		for object := range mts.Client.ListObjectsV2(bucket, "", true, doneC) {
   200  			if object.Err != nil {
   201  				errLock.Lock()
   202  				errs = append(errs, kv.Wrap(object.Err).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   203  				errLock.Unlock()
   204  				continue
   205  			}
   206  			select {
   207  			case keysC <- object.Key:
   208  			case <-time.After(2 * time.Second):
   209  				errLock.Lock()
   210  				errs = append(errs, kv.NewError("object delete timeout").With("key", object.Key).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   211  				errLock.Unlock()
   212  				// Giveup deleting an object if it blocks everything
   213  			}
   214  		}
   215  		for object := range mts.Client.ListIncompleteUploads(bucket, "", true, doneC) {
   216  			if object.Err != nil {
   217  				errLock.Lock()
   218  				errs = append(errs, kv.Wrap(object.Err).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   219  				errLock.Unlock()
   220  				continue
   221  			}
   222  			select {
   223  			case keysC <- object.Key:
   224  			case <-time.After(2 * time.Second):
   225  				errLock.Lock()
   226  				errs = append(errs, kv.NewError("partial upload delete timeout").With("key", object.Key).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   227  				errLock.Unlock()
   228  				// Giveup deleting an object if it blocks everything
   229  			}
   230  		}
   231  	}()
   232  
   233  	for errMinio := range mts.Client.RemoveObjects(bucket, keysC) {
   234  		if errMinio.Err.Error() == "EOF" {
   235  			break
   236  		}
   237  		errLock.Lock()
   238  		errs = append(errs, kv.NewError(errMinio.Err.Error()).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   239  		errLock.Unlock()
   240  	}
   241  
   242  	errGo = mts.Client.RemoveBucket(bucket)
   243  	if errGo != nil {
   244  		errs = append(errs, kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime()))
   245  	}
   246  	return errs
   247  }
   248  
   249  // Upload will take the nominated file, file parameter, and will upload it to the bucket and key
   250  // pair on the server identified by the mtx receiver variable
   251  //
   252  func (mts *MinioTestServer) Upload(bucket string, key string, file string) (err kv.Error) {
   253  
   254  	if !mts.Ready.Load() {
   255  		return kv.NewError("server not ready").With("host", mts.Address).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())
   256  	}
   257  
   258  	f, errGo := os.Open(filepath.Clean(file))
   259  	if errGo != nil {
   260  		return kv.Wrap(errGo, "Upload passed a non-existent file name").With("file", file).With("stack", stack.Trace().TrimRuntime())
   261  	}
   262  	defer f.Close()
   263  
   264  	exists, errGo := mts.Client.BucketExists(bucket)
   265  	if errGo != nil {
   266  		return kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())
   267  	}
   268  	if !exists {
   269  		if errGo = mts.Client.MakeBucket(bucket, ""); errGo != nil {
   270  			return kv.Wrap(errGo).With("bucket", bucket).With("stack", stack.Trace().TrimRuntime())
   271  		}
   272  	}
   273  
   274  	_, errGo = mts.Client.PutObject(bucket, key, bufio.NewReader(f), -1,
   275  		minio.PutObjectOptions{
   276  			ContentType:  "application/octet-stream",
   277  			CacheControl: "max-age=600",
   278  		})
   279  
   280  	if errGo != nil {
   281  		return kv.Wrap(errGo).With("bucket", bucket).With("key", key).With("file", file).With("stack", stack.Trace().TrimRuntime())
   282  	}
   283  
   284  	return nil
   285  }
   286  
   287  func writeCfg(mts *MinioTestServer) (cfgDir string, err kv.Error) {
   288  	// Initialize a configuration directory for the minio server
   289  	// complete with the json configuration containing the credentials
   290  	// for the test server
   291  	cfgDir, errGo := ioutil.TempDir("", xid.New().String())
   292  	if errGo != nil {
   293  		return "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   294  	}
   295  	cfg := MinioCfgJson{}
   296  	cfg.Version = "26"
   297  	cfg.Credential.AccessKey = mts.AccessKeyId
   298  	cfg.Credential.SecretKey = mts.SecretAccessKeyId
   299  	cfg.Worm = "off"
   300  
   301  	result, errGo := json.MarshalIndent(cfg, "", "    ")
   302  	if errGo != nil {
   303  		return "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   304  	}
   305  	if errGo = ioutil.WriteFile(path.Join(cfgDir, "config.json"), result, 0666); errGo != nil {
   306  		return "", kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   307  	}
   308  	return cfgDir, nil
   309  }
   310  
   311  // startLocalMinio will fork off a running minio server with an empty data store
   312  // that can be used for testing purposes.  This function does not block,
   313  // however it does start a go routine
   314  //
   315  func startLocalMinio(ctx context.Context, retainWorkingDirs bool, errC chan kv.Error) {
   316  
   317  	// Default to the case that another pod for external host has a running minio server for us
   318  	// to use during testing
   319  	if len(*minioTestServer) != 0 {
   320  		MinioTest.Address = os.ExpandEnv(*minioTestServer)
   321  	}
   322  	if len(*minioAccessKey) != 0 {
   323  		MinioTest.AccessKeyId = os.ExpandEnv(*minioAccessKey)
   324  	}
   325  	if len(*minioSecretKey) != 0 {
   326  		MinioTest.SecretAccessKeyId = os.ExpandEnv(*minioSecretKey)
   327  	}
   328  
   329  	// If we dont have a k8s based minio server specified for our test try try using a local
   330  	// minio instance within the container or machine the test is run on
   331  	//
   332  	if len(*minioTestServer) == 0 {
   333  		// First check that the minio executable is present on the test system
   334  		//
   335  		// We are using the executable because the dependency hierarchy of minio
   336  		// is very tangled and so it is very hard to embeed for now, Go 1.10.3
   337  		execPath, errGo := exec.LookPath("minio")
   338  		if errGo != nil {
   339  			errC <- kv.Wrap(errGo, "please install minio into your path").With("path", os.Getenv("PATH")).With("stack", stack.Trace().TrimRuntime())
   340  			return
   341  		}
   342  
   343  		// Get a free server listening port for our test
   344  		port, err := GetFreePort("127.0.0.1:0")
   345  		if err != nil {
   346  			errC <- err
   347  			return
   348  		}
   349  
   350  		MinioTest.Address = fmt.Sprintf("127.0.0.1:%d", port)
   351  
   352  		// Initialize the data directory for the file server
   353  		storageDir, errGo := ioutil.TempDir("", xid.New().String())
   354  		if errGo != nil {
   355  			errC <- kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   356  			return
   357  		}
   358  		filepath.Clean(storageDir)
   359  
   360  		if errGo = os.Chmod(storageDir, 0700); errGo != nil {
   361  			errC <- kv.Wrap(errGo).With("storageDir", storageDir).With("stack", stack.Trace().TrimRuntime())
   362  			os.RemoveAll(storageDir)
   363  			return
   364  		}
   365  
   366  		// If we see no credentials were supplied for a local test, the typical case
   367  		// then supply some defaults
   368  		if len(MinioTest.AccessKeyId) == 0 {
   369  			MinioTest.AccessKeyId = "UserUser"
   370  		}
   371  		if len(MinioTest.SecretAccessKeyId) == 0 {
   372  			MinioTest.SecretAccessKeyId = "PasswordPassword"
   373  		}
   374  
   375  		// Now write a cfg file out for our desired minio
   376  		// configuration
   377  		cfgDir, err := writeCfg(MinioTest)
   378  		if err != nil {
   379  			errC <- err
   380  			return
   381  		}
   382  		cfgDir = filepath.Clean(cfgDir)
   383  
   384  		go func() {
   385  			cmdCtx, cancel := context.WithCancel(ctx)
   386  			// When the main process stops kill our cmd runner for minio
   387  			defer cancel()
   388  
   389  			// #nosec
   390  			cmd := exec.CommandContext(cmdCtx, filepath.Clean(execPath),
   391  				"server",
   392  				"--address", MinioTest.Address,
   393  				"--config-dir", cfgDir,
   394  				storageDir,
   395  			)
   396  
   397  			stdout, errGo := cmd.StdoutPipe()
   398  			if errGo != nil {
   399  				errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime())
   400  			}
   401  			stderr, errGo := cmd.StderrPipe()
   402  			if errGo != nil {
   403  				errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime())
   404  			}
   405  			// Non-blockingly echo command output to terminal
   406  			go io.Copy(os.Stdout, stdout)
   407  			go io.Copy(os.Stderr, stderr)
   408  
   409  			if errGo = cmd.Start(); errGo != nil {
   410  				errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime())
   411  			}
   412  
   413  			if errGo = cmd.Wait(); errGo != nil {
   414  				if errGo.Error() != "signal: killed" {
   415  					errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime())
   416  				}
   417  			}
   418  
   419  			fmt.Printf("%v\n", kv.NewError("minio terminated").With("cfgDir", cfgDir, "storageDir", storageDir).With("stack", stack.Trace().TrimRuntime()))
   420  
   421  			if !retainWorkingDirs {
   422  				os.RemoveAll(storageDir)
   423  				os.RemoveAll(cfgDir)
   424  			}
   425  		}()
   426  	}
   427  
   428  	startMinioClient(ctx, errC)
   429  }
   430  
   431  func startMinioClient(ctx context.Context, errC chan kv.Error) {
   432  	// Wait for the server to start by checking the listen port using
   433  	// TCP
   434  	check := time.NewTicker(time.Second)
   435  	defer check.Stop()
   436  
   437  	for {
   438  		select {
   439  		case <-check.C:
   440  			client, errGo := minio.New(MinioTest.Address, MinioTest.AccessKeyId,
   441  				MinioTest.SecretAccessKeyId, false)
   442  			if errGo != nil {
   443  				errC <- kv.Wrap(errGo, "minio failed").With("stack", stack.Trace().TrimRuntime())
   444  				continue
   445  			}
   446  			MinioTest.Client = client
   447  			MinioTest.Ready.Store(true)
   448  			return
   449  		case <-ctx.Done():
   450  			return
   451  		}
   452  	}
   453  }
   454  
   455  // IsAlive is used to test if the expected minio local test server is alive
   456  //
   457  func (mts *MinioTestServer) IsAlive(ctx context.Context) (alive bool, err kv.Error) {
   458  
   459  	check := time.NewTicker(5 * time.Second)
   460  	defer check.Stop()
   461  
   462  	for {
   463  		select {
   464  		case <-ctx.Done():
   465  			return false, err
   466  		case <-check.C:
   467  			if !mts.Ready.Load() || mts.Client == nil {
   468  				continue
   469  			}
   470  			_, errGo := mts.Client.BucketExists(xid.New().String())
   471  			if errGo == nil {
   472  				return true, nil
   473  			}
   474  			err = kv.Wrap(errGo).With("stack", stack.Trace().TrimRuntime())
   475  		}
   476  	}
   477  }
   478  
   479  // InitTestingMinio will fork a minio server that can he used for staging and test
   480  // in a manner that also wraps an error reporting channel and a means of
   481  // stopping it
   482  //
   483  func InitTestingMinio(ctx context.Context, retainWorkingDirs bool) (errC chan kv.Error) {
   484  	errC = make(chan kv.Error, 5)
   485  
   486  	startLocalMinio(ctx, retainWorkingDirs, errC)
   487  
   488  	return errC
   489  }