storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/test-utils_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bufio"
    21  	"bytes"
    22  	"context"
    23  	"crypto/ecdsa"
    24  	"crypto/hmac"
    25  	crand "crypto/rand"
    26  	"crypto/rsa"
    27  	"crypto/sha1"
    28  	"crypto/tls"
    29  	"crypto/x509"
    30  	"crypto/x509/pkix"
    31  	"encoding/base64"
    32  	"encoding/hex"
    33  	"encoding/json"
    34  	"encoding/pem"
    35  	"encoding/xml"
    36  	"errors"
    37  	"flag"
    38  	"fmt"
    39  	"io"
    40  	"io/ioutil"
    41  	"math/big"
    42  	"math/rand"
    43  	"net"
    44  	"net/http"
    45  	"net/http/httptest"
    46  	"net/url"
    47  	"os"
    48  	"reflect"
    49  	"sort"
    50  	"strconv"
    51  	"strings"
    52  	"sync"
    53  	"testing"
    54  	"time"
    55  
    56  	"github.com/fatih/color"
    57  
    58  	"github.com/gorilla/mux"
    59  	"github.com/minio/minio-go/v7/pkg/s3utils"
    60  	"github.com/minio/minio-go/v7/pkg/signer"
    61  
    62  	"storj.io/minio/cmd/config"
    63  	"storj.io/minio/cmd/crypto"
    64  	xhttp "storj.io/minio/cmd/http"
    65  	"storj.io/minio/cmd/logger"
    66  	"storj.io/minio/cmd/rest"
    67  	"storj.io/minio/pkg/auth"
    68  	"storj.io/minio/pkg/bucket/policy"
    69  	"storj.io/minio/pkg/hash"
    70  )
    71  
    72  // TestMain to set up global env.
    73  func TestMain(m *testing.M) {
    74  	flag.Parse()
    75  
    76  	globalActiveCred = auth.Credentials{
    77  		AccessKey: auth.DefaultAccessKey,
    78  		SecretKey: auth.DefaultSecretKey,
    79  	}
    80  
    81  	globalConfigEncrypted = true
    82  
    83  	// disable ENVs which interfere with tests.
    84  	for _, env := range []string{
    85  		crypto.EnvKMSAutoEncryption,
    86  		config.EnvAccessKey,
    87  		config.EnvSecretKey,
    88  	} {
    89  		os.Unsetenv(env)
    90  	}
    91  
    92  	// Set as non-distributed.
    93  	globalIsDistErasure = false
    94  
    95  	if !testing.Verbose() {
    96  		// Disable printing console messages during tests.
    97  		color.Output = ioutil.Discard
    98  		logger.Disable = true
    99  	}
   100  	// Uncomment the following line to see trace logs during unit tests.
   101  	// logger.AddTarget(console.New())
   102  
   103  	// Set system resources to maximum.
   104  	setMaxResources()
   105  
   106  	// Initialize globalConsoleSys system
   107  	globalConsoleSys = NewConsoleLogger(context.Background())
   108  
   109  	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second, logger.LogOnceIf)
   110  
   111  	globalInternodeTransport = newInternodeHTTPTransport(nil, rest.DefaultTimeout)()
   112  
   113  	initHelp()
   114  
   115  	resetTestGlobals()
   116  
   117  	os.Setenv("MINIO_CI_CD", "ci")
   118  
   119  	os.Exit(m.Run())
   120  }
   121  
   122  // concurrency level for certain parallel tests.
   123  const testConcurrencyLevel = 10
   124  
   125  ///
   126  /// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258
   127  ///
   128  ///  User-Agent:
   129  ///
   130  ///      This is ignored from signing because signing this causes problems with generating pre-signed URLs
   131  ///      (that are executed by other agents) or when customers pass requests through proxies, which may
   132  ///      modify the user-agent.
   133  ///
   134  ///  Authorization:
   135  ///
   136  ///      Is skipped for obvious reasons
   137  ///
   138  var ignoredHeaders = map[string]bool{
   139  	"Authorization": true,
   140  	"User-Agent":    true,
   141  }
   142  
   143  // Headers to ignore in streaming v4
   144  var ignoredStreamingHeaders = map[string]bool{
   145  	"Authorization": true,
   146  	"Content-Type":  true,
   147  	"Content-Md5":   true,
   148  	"User-Agent":    true,
   149  }
   150  
   151  // calculateSignedChunkLength - calculates the length of chunk metadata
   152  func calculateSignedChunkLength(chunkDataSize int64) int64 {
   153  	return int64(len(fmt.Sprintf("%x", chunkDataSize))) +
   154  		17 + // ";chunk-signature="
   155  		64 + // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2"
   156  		2 + // CRLF
   157  		chunkDataSize +
   158  		2 // CRLF
   159  }
   160  
   161  func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *PutObjReader {
   162  	hr, err := hash.NewReader(data, size, md5hex, sha256hex, size)
   163  	if err != nil {
   164  		t.Fatal(err)
   165  	}
   166  	return NewPutObjReader(hr)
   167  }
   168  
   169  // calculateSignedChunkLength - calculates the length of the overall stream (data + metadata)
   170  func calculateStreamContentLength(dataLen, chunkSize int64) int64 {
   171  	if dataLen <= 0 {
   172  		return 0
   173  	}
   174  	chunksCount := dataLen / chunkSize
   175  	remainingBytes := dataLen % chunkSize
   176  	var streamLen int64
   177  	streamLen += chunksCount * calculateSignedChunkLength(chunkSize)
   178  	if remainingBytes > 0 {
   179  		streamLen += calculateSignedChunkLength(remainingBytes)
   180  	}
   181  	streamLen += calculateSignedChunkLength(0)
   182  	return streamLen
   183  }
   184  
   185  func prepareFS() (ObjectLayer, string, error) {
   186  	nDisks := 1
   187  	fsDirs, err := getRandomDisks(nDisks)
   188  	if err != nil {
   189  		return nil, "", err
   190  	}
   191  	obj, err := NewFSObjectLayer(fsDirs[0])
   192  	if err != nil {
   193  		return nil, "", err
   194  	}
   195  	return obj, fsDirs[0], nil
   196  }
   197  
   198  func prepareErasureSets32(ctx context.Context) (ObjectLayer, []string, error) {
   199  	return prepareErasure(ctx, 32)
   200  }
   201  
   202  func prepareErasure(ctx context.Context, nDisks int) (ObjectLayer, []string, error) {
   203  	fsDirs, err := getRandomDisks(nDisks)
   204  	if err != nil {
   205  		return nil, nil, err
   206  	}
   207  	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
   208  	if err != nil {
   209  		removeRoots(fsDirs)
   210  		return nil, nil, err
   211  	}
   212  	return obj, fsDirs, nil
   213  }
   214  
   215  func prepareErasure16(ctx context.Context) (ObjectLayer, []string, error) {
   216  	return prepareErasure(ctx, 16)
   217  }
   218  
   219  // Initialize FS objects.
   220  func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) {
   221  	var err error
   222  	obj, err = NewFSObjectLayer(disk)
   223  	if err != nil {
   224  		t.Fatal(err)
   225  	}
   226  	newTestConfig(globalMinioDefaultRegion, obj)
   227  	return obj
   228  }
   229  
   230  // TestErrHandler - Go testing.T satisfy this interface.
   231  // This makes it easy to run the TestServer from any of the tests.
   232  // Using this interface, functionalities to be used in tests can be
   233  // made generalized, and can be integrated in benchmarks/unit tests/go check suite tests.
   234  type TestErrHandler interface {
   235  	testing.TB
   236  }
   237  
   238  const (
   239  	// FSTestStr is the string which is used as notation for Single node ObjectLayer in the unit tests.
   240  	FSTestStr string = "FS"
   241  
   242  	// ErasureTestStr is the string which is used as notation for Erasure ObjectLayer in the unit tests.
   243  	ErasureTestStr string = "Erasure"
   244  
   245  	// ErasureSetsTestStr is the string which is used as notation for Erasure sets object layer in the unit tests.
   246  	ErasureSetsTestStr string = "ErasureSet"
   247  )
   248  
   249  const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
   250  const (
   251  	letterIdxBits = 6                    // 6 bits to represent a letter index
   252  	letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
   253  	letterIdxMax  = 63 / letterIdxBits   // # of letter indices fitting in 63 bits
   254  )
   255  
   256  // Random number state.
   257  // We generate random temporary file names so that there's a good
   258  // chance the file doesn't exist yet.
   259  var randN uint32
   260  var randmu sync.Mutex
   261  
   262  // Temp files created in default Tmp dir
   263  var globalTestTmpDir = os.TempDir()
   264  
   265  // reseed - returns a new seed every time the function is called.
   266  func reseed() uint32 {
   267  	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
   268  }
   269  
   270  // nextSuffix - provides a new unique suffix every time the function is called.
   271  func nextSuffix() string {
   272  	randmu.Lock()
   273  	r := randN
   274  	// Initial seed required, generate one.
   275  	if r == 0 {
   276  		r = reseed()
   277  	}
   278  	// constants from Numerical Recipes
   279  	r = r*1664525 + 1013904223
   280  	randN = r
   281  	randmu.Unlock()
   282  	return strconv.Itoa(int(1e9 + r%1e9))[1:]
   283  }
   284  
   285  // isSameType - compares two object types via reflect.TypeOf
   286  func isSameType(obj1, obj2 interface{}) bool {
   287  	return reflect.TypeOf(obj1) == reflect.TypeOf(obj2)
   288  }
   289  
   290  // TestServer encapsulates an instantiation of a MinIO instance with a temporary backend.
   291  // Example usage:
   292  //   s := StartTestServer(t,"Erasure")
   293  //   defer s.Stop()
   294  type TestServer struct {
   295  	Root      string
   296  	Disks     EndpointServerPools
   297  	AccessKey string
   298  	SecretKey string
   299  	Server    *httptest.Server
   300  	Obj       ObjectLayer
   301  	cancel    context.CancelFunc
   302  }
   303  
   304  // UnstartedTestServer - Configures a temp FS/Erasure backend,
   305  // initializes the endpoints and configures the test server.
   306  // The server should be started using the Start() method.
   307  func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
   308  	ctx, cancel := context.WithCancel(context.Background())
   309  	// create an instance of TestServer.
   310  	testServer := TestServer{cancel: cancel}
   311  	// return FS/Erasure object layer and temp backend.
   312  	objLayer, disks, err := prepareTestBackend(ctx, instanceType)
   313  	if err != nil {
   314  		t.Fatal(err)
   315  	}
   316  
   317  	// set the server configuration.
   318  	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
   319  		t.Fatalf("%s", err)
   320  	}
   321  
   322  	// Test Server needs to start before formatting of disks.
   323  	// Get credential.
   324  	credentials := globalActiveCred
   325  
   326  	testServer.Obj = objLayer
   327  	testServer.Disks = mustGetPoolEndpoints(disks...)
   328  	testServer.AccessKey = credentials.AccessKey
   329  	testServer.SecretKey = credentials.SecretKey
   330  
   331  	httpHandler, err := configureServerHandler(testServer.Disks)
   332  	if err != nil {
   333  		t.Fatalf("Failed to configure one of the RPC services <ERROR> %s", err)
   334  	}
   335  
   336  	// Run TestServer.
   337  	testServer.Server = httptest.NewUnstartedServer(criticalErrorHandler{corsHandler(httpHandler)})
   338  
   339  	globalObjLayerMutex.Lock()
   340  	globalObjectAPI = objLayer
   341  	globalObjLayerMutex.Unlock()
   342  
   343  	// initialize peer rpc
   344  	host, port := mustSplitHostPort(testServer.Server.Listener.Addr().String())
   345  	globalMinioHost = host
   346  	globalMinioPort = port
   347  	globalMinioAddr = getEndpointsLocalAddr(testServer.Disks)
   348  
   349  	newAllSubsystems()
   350  
   351  	initAllSubsystems(ctx, objLayer)
   352  
   353  	return testServer
   354  }
   355  
   356  // testServerCertPEM and testServerKeyPEM are generated by
   357  //  https://golang.org/src/crypto/tls/generate_cert.go
   358  //    $ go run generate_cert.go -ca --host 127.0.0.1
   359  // The generated certificate contains IP SAN, that way we don't need
   360  // to enable InsecureSkipVerify in TLS config
   361  
   362  // Starts the test server and returns the TestServer with TLS configured instance.
   363  func StartTestTLSServer(t TestErrHandler, instanceType string, cert, key []byte) TestServer {
   364  	// Fetch TLS key and pem files from test-data/ directory.
   365  	//	dir, _ := os.Getwd()
   366  	//	testDataDir := filepath.Join(filepath.Dir(dir), "test-data")
   367  	//
   368  	//	pemFile := filepath.Join(testDataDir, "server.pem")
   369  	//	keyFile := filepath.Join(testDataDir, "server.key")
   370  	cer, err := tls.X509KeyPair(cert, key)
   371  	if err != nil {
   372  		t.Fatalf("Failed to load certificate: %v", err)
   373  	}
   374  	config := &tls.Config{Certificates: []tls.Certificate{cer}}
   375  
   376  	testServer := UnstartedTestServer(t, instanceType)
   377  	testServer.Server.TLS = config
   378  	testServer.Server.StartTLS()
   379  	return testServer
   380  }
   381  
   382  // Starts the test server and returns the TestServer instance.
   383  func StartTestServer(t TestErrHandler, instanceType string) TestServer {
   384  	// create an instance of TestServer.
   385  	testServer := UnstartedTestServer(t, instanceType)
   386  	testServer.Server.Start()
   387  	return testServer
   388  }
   389  
   390  // Sets the global config path to empty string.
   391  func resetGlobalConfigPath() {
   392  	globalConfigDir = &ConfigDir{path: ""}
   393  }
   394  
   395  // sets globalObjectAPI to `nil`.
   396  func resetGlobalObjectAPI() {
   397  	globalObjLayerMutex.Lock()
   398  	globalObjectAPI = nil
   399  	globalObjLayerMutex.Unlock()
   400  }
   401  
   402  // reset the value of the Global server config.
   403  // set it to `nil`.
   404  func resetGlobalConfig() {
   405  	// hold the mutex lock before a new config is assigned.
   406  	globalServerConfigMu.Lock()
   407  	// Save the loaded config globally.
   408  	globalServerConfig = nil
   409  	globalServerConfigMu.Unlock()
   410  }
   411  
   412  func resetGlobalEndpoints() {
   413  	globalEndpoints = EndpointServerPools{}
   414  }
   415  
   416  func resetGlobalIsErasure() {
   417  	globalIsErasure = false
   418  }
   419  
   420  // reset global heal state
   421  func resetGlobalHealState() {
   422  	// Init global heal state
   423  	if globalAllHealState == nil {
   424  		globalAllHealState = newHealState(false)
   425  	} else {
   426  		globalAllHealState.Lock()
   427  		for _, v := range globalAllHealState.healSeqMap {
   428  			if !v.hasEnded() {
   429  				v.stop()
   430  			}
   431  		}
   432  		globalAllHealState.Unlock()
   433  	}
   434  
   435  	// Init background heal state
   436  	if globalBackgroundHealState == nil {
   437  		globalBackgroundHealState = newHealState(false)
   438  	} else {
   439  		globalBackgroundHealState.Lock()
   440  		for _, v := range globalBackgroundHealState.healSeqMap {
   441  			if !v.hasEnded() {
   442  				v.stop()
   443  			}
   444  		}
   445  		globalBackgroundHealState.Unlock()
   446  	}
   447  }
   448  
   449  // sets GlobalIAMSys to `nil`.
   450  func resetGlobalIAMSys() {
   451  	GlobalIAMSys = nil
   452  }
   453  
   454  // Resets all the globals used modified in tests.
   455  // Resetting ensures that the changes made to globals by one test doesn't affect others.
   456  func resetTestGlobals() {
   457  	// set globalObjectAPI to `nil`.
   458  	resetGlobalObjectAPI()
   459  	// Reset config path set.
   460  	resetGlobalConfigPath()
   461  	// Reset Global server config.
   462  	resetGlobalConfig()
   463  	// Reset global endpoints.
   464  	resetGlobalEndpoints()
   465  	// Reset global isErasure flag.
   466  	resetGlobalIsErasure()
   467  	// Reset global heal state
   468  	resetGlobalHealState()
   469  	// Reset GlobalIAMSys to `nil`
   470  	resetGlobalIAMSys()
   471  }
   472  
   473  // Configure the server for the test run.
   474  func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) {
   475  	// Initialize server config.
   476  	if err = newSrvConfig(obj); err != nil {
   477  		return err
   478  	}
   479  
   480  	// Set a default region.
   481  	config.SetRegion(globalServerConfig, bucketLocation)
   482  
   483  	// Save config.
   484  	return saveServerConfig(context.Background(), obj, globalServerConfig)
   485  }
   486  
   487  // Deleting the temporary backend and stopping the server.
   488  func (testServer TestServer) Stop() {
   489  	testServer.cancel()
   490  	testServer.Server.Close()
   491  	testServer.Obj.Shutdown(context.Background())
   492  	os.RemoveAll(testServer.Root)
   493  	for _, ep := range testServer.Disks {
   494  		for _, disk := range ep.Endpoints {
   495  			os.RemoveAll(disk.Path)
   496  		}
   497  	}
   498  }
   499  
   500  // Truncate request to simulate unexpected EOF for a request signed using streaming signature v4.
   501  func truncateChunkByHalfSigv4(req *http.Request) (*http.Request, error) {
   502  	bufReader := bufio.NewReader(req.Body)
   503  	hexChunkSize, chunkSignature, err := readChunkLine(bufReader)
   504  	if err != nil {
   505  		return nil, err
   506  	}
   507  
   508  	newChunkHdr := []byte(fmt.Sprintf("%s"+s3ChunkSignatureStr+"%s\r\n",
   509  		hexChunkSize, chunkSignature))
   510  	newChunk, err := ioutil.ReadAll(bufReader)
   511  	if err != nil {
   512  		return nil, err
   513  	}
   514  	newReq := req
   515  	newReq.Body = ioutil.NopCloser(
   516  		bytes.NewReader(bytes.Join([][]byte{newChunkHdr, newChunk[:len(newChunk)/2]},
   517  			[]byte(""))),
   518  	)
   519  	return newReq, nil
   520  }
   521  
   522  // Malform data given a request signed using streaming signature V4.
   523  func malformDataSigV4(req *http.Request, newByte byte) (*http.Request, error) {
   524  	bufReader := bufio.NewReader(req.Body)
   525  	hexChunkSize, chunkSignature, err := readChunkLine(bufReader)
   526  	if err != nil {
   527  		return nil, err
   528  	}
   529  
   530  	newChunkHdr := []byte(fmt.Sprintf("%s"+s3ChunkSignatureStr+"%s\r\n",
   531  		hexChunkSize, chunkSignature))
   532  	newChunk, err := ioutil.ReadAll(bufReader)
   533  	if err != nil {
   534  		return nil, err
   535  	}
   536  
   537  	newChunk[0] = newByte
   538  	newReq := req
   539  	newReq.Body = ioutil.NopCloser(
   540  		bytes.NewReader(bytes.Join([][]byte{newChunkHdr, newChunk},
   541  			[]byte(""))),
   542  	)
   543  
   544  	return newReq, nil
   545  }
   546  
   547  // Malform chunk size given a request signed using streaming signatureV4.
   548  func malformChunkSizeSigV4(req *http.Request, badSize int64) (*http.Request, error) {
   549  	bufReader := bufio.NewReader(req.Body)
   550  	_, chunkSignature, err := readChunkLine(bufReader)
   551  	if err != nil {
   552  		return nil, err
   553  	}
   554  
   555  	n := badSize
   556  	newHexChunkSize := []byte(fmt.Sprintf("%x", n))
   557  	newChunkHdr := []byte(fmt.Sprintf("%s"+s3ChunkSignatureStr+"%s\r\n",
   558  		newHexChunkSize, chunkSignature))
   559  	newChunk, err := ioutil.ReadAll(bufReader)
   560  	if err != nil {
   561  		return nil, err
   562  	}
   563  
   564  	newReq := req
   565  	newReq.Body = ioutil.NopCloser(
   566  		bytes.NewReader(bytes.Join([][]byte{newChunkHdr, newChunk},
   567  			[]byte(""))),
   568  	)
   569  
   570  	return newReq, nil
   571  }
   572  
   573  // Sign given request using Signature V4.
   574  func signStreamingRequest(req *http.Request, accessKey, secretKey string, currTime time.Time) (string, error) {
   575  	// Get hashed payload.
   576  	hashedPayload := req.Header.Get("x-amz-content-sha256")
   577  	if hashedPayload == "" {
   578  		return "", fmt.Errorf("Invalid hashed payload")
   579  	}
   580  
   581  	// Set x-amz-date.
   582  	req.Header.Set("x-amz-date", currTime.Format(iso8601Format))
   583  
   584  	// Get header map.
   585  	headerMap := make(map[string][]string)
   586  	for k, vv := range req.Header {
   587  		// If request header key is not in ignored headers, then add it.
   588  		if _, ok := ignoredStreamingHeaders[http.CanonicalHeaderKey(k)]; !ok {
   589  			headerMap[strings.ToLower(k)] = vv
   590  		}
   591  	}
   592  
   593  	// Get header keys.
   594  	headers := []string{"host"}
   595  	for k := range headerMap {
   596  		headers = append(headers, k)
   597  	}
   598  	sort.Strings(headers)
   599  
   600  	// Get canonical headers.
   601  	var buf bytes.Buffer
   602  	for _, k := range headers {
   603  		buf.WriteString(k)
   604  		buf.WriteByte(':')
   605  		switch {
   606  		case k == "host":
   607  			buf.WriteString(req.URL.Host)
   608  			fallthrough
   609  		default:
   610  			for idx, v := range headerMap[k] {
   611  				if idx > 0 {
   612  					buf.WriteByte(',')
   613  				}
   614  				buf.WriteString(v)
   615  			}
   616  			buf.WriteByte('\n')
   617  		}
   618  	}
   619  	canonicalHeaders := buf.String()
   620  
   621  	// Get signed headers.
   622  	signedHeaders := strings.Join(headers, ";")
   623  
   624  	// Get canonical query string.
   625  	req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
   626  
   627  	// Get canonical URI.
   628  	canonicalURI := s3utils.EncodePath(req.URL.Path)
   629  
   630  	// Get canonical request.
   631  	// canonicalRequest =
   632  	//  <HTTPMethod>\n
   633  	//  <CanonicalURI>\n
   634  	//  <CanonicalQueryString>\n
   635  	//  <CanonicalHeaders>\n
   636  	//  <SignedHeaders>\n
   637  	//  <HashedPayload>
   638  	//
   639  	canonicalRequest := strings.Join([]string{
   640  		req.Method,
   641  		canonicalURI,
   642  		req.URL.RawQuery,
   643  		canonicalHeaders,
   644  		signedHeaders,
   645  		hashedPayload,
   646  	}, "\n")
   647  
   648  	// Get scope.
   649  	scope := strings.Join([]string{
   650  		currTime.Format(yyyymmdd),
   651  		globalMinioDefaultRegion,
   652  		string(serviceS3),
   653  		"aws4_request",
   654  	}, SlashSeparator)
   655  
   656  	stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime.Format(iso8601Format) + "\n"
   657  	stringToSign = stringToSign + scope + "\n"
   658  	stringToSign = stringToSign + getSHA256Hash([]byte(canonicalRequest))
   659  
   660  	date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
   661  	region := sumHMAC(date, []byte(globalMinioDefaultRegion))
   662  	service := sumHMAC(region, []byte(string(serviceS3)))
   663  	signingKey := sumHMAC(service, []byte("aws4_request"))
   664  
   665  	signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
   666  
   667  	// final Authorization header
   668  	parts := []string{
   669  		"AWS4-HMAC-SHA256" + " Credential=" + accessKey + SlashSeparator + scope,
   670  		"SignedHeaders=" + signedHeaders,
   671  		"Signature=" + signature,
   672  	}
   673  	auth := strings.Join(parts, ", ")
   674  	req.Header.Set("Authorization", auth)
   675  
   676  	return signature, nil
   677  }
   678  
   679  // Returns new HTTP request object.
   680  func newTestStreamingRequest(method, urlStr string, dataLength, chunkSize int64, body io.ReadSeeker) (*http.Request, error) {
   681  	if method == "" {
   682  		method = http.MethodPost
   683  	}
   684  
   685  	req, err := http.NewRequest(method, urlStr, nil)
   686  	if err != nil {
   687  		return nil, err
   688  	}
   689  
   690  	if body == nil {
   691  		// this is added to avoid panic during ioutil.ReadAll(req.Body).
   692  		// th stack trace can be found here  https://github.com/minio/minio/pull/2074 .
   693  		// This is very similar to https://github.com/golang/go/issues/7527.
   694  		req.Body = ioutil.NopCloser(bytes.NewReader([]byte("")))
   695  	}
   696  
   697  	contentLength := calculateStreamContentLength(dataLength, chunkSize)
   698  
   699  	req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD")
   700  	req.Header.Set("content-encoding", "aws-chunked")
   701  	req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLength, 10))
   702  	req.Header.Set("content-length", strconv.FormatInt(contentLength, 10))
   703  
   704  	// Seek back to beginning.
   705  	body.Seek(0, 0)
   706  
   707  	// Add body
   708  	req.Body = ioutil.NopCloser(body)
   709  	req.ContentLength = contentLength
   710  
   711  	return req, nil
   712  }
   713  
   714  func assembleStreamingChunks(req *http.Request, body io.ReadSeeker, chunkSize int64,
   715  	secretKey, signature string, currTime time.Time) (*http.Request, error) {
   716  
   717  	regionStr := globalServerRegion
   718  	var stream []byte
   719  	var buffer []byte
   720  	body.Seek(0, 0)
   721  	for {
   722  		buffer = make([]byte, chunkSize)
   723  		n, err := body.Read(buffer)
   724  		if err != nil && err != io.EOF {
   725  			return nil, err
   726  		}
   727  
   728  		// Get scope.
   729  		scope := strings.Join([]string{
   730  			currTime.Format(yyyymmdd),
   731  			regionStr,
   732  			string(serviceS3),
   733  			"aws4_request",
   734  		}, SlashSeparator)
   735  
   736  		stringToSign := "AWS4-HMAC-SHA256-PAYLOAD" + "\n"
   737  		stringToSign = stringToSign + currTime.Format(iso8601Format) + "\n"
   738  		stringToSign = stringToSign + scope + "\n"
   739  		stringToSign = stringToSign + signature + "\n"
   740  		stringToSign = stringToSign + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + "\n" // hex(sum256(""))
   741  		stringToSign = stringToSign + getSHA256Hash(buffer[:n])
   742  
   743  		date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
   744  		region := sumHMAC(date, []byte(regionStr))
   745  		service := sumHMAC(region, []byte(serviceS3))
   746  		signingKey := sumHMAC(service, []byte("aws4_request"))
   747  
   748  		signature = hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
   749  
   750  		stream = append(stream, []byte(fmt.Sprintf("%x", n)+";chunk-signature="+signature+"\r\n")...)
   751  		stream = append(stream, buffer[:n]...)
   752  		stream = append(stream, []byte("\r\n")...)
   753  
   754  		if n <= 0 {
   755  			break
   756  		}
   757  
   758  	}
   759  	req.Body = ioutil.NopCloser(bytes.NewReader(stream))
   760  	return req, nil
   761  }
   762  
   763  func newTestStreamingSignedBadChunkDateRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) {
   764  	req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body)
   765  	if err != nil {
   766  		return nil, err
   767  	}
   768  
   769  	currTime := UTCNow()
   770  	signature, err := signStreamingRequest(req, accessKey, secretKey, currTime)
   771  	if err != nil {
   772  		return nil, err
   773  	}
   774  
   775  	// skew the time between the chunk signature calculation and seed signature.
   776  	currTime = currTime.Add(1 * time.Second)
   777  	req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime)
   778  	return req, err
   779  }
   780  
   781  func newTestStreamingSignedCustomEncodingRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey, contentEncoding string) (*http.Request, error) {
   782  	req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body)
   783  	if err != nil {
   784  		return nil, err
   785  	}
   786  
   787  	// Set custom encoding.
   788  	req.Header.Set("content-encoding", contentEncoding)
   789  
   790  	currTime := UTCNow()
   791  	signature, err := signStreamingRequest(req, accessKey, secretKey, currTime)
   792  	if err != nil {
   793  		return nil, err
   794  	}
   795  
   796  	req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime)
   797  	return req, err
   798  }
   799  
   800  // Returns new HTTP request object signed with streaming signature v4.
   801  func newTestStreamingSignedRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) {
   802  	req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body)
   803  	if err != nil {
   804  		return nil, err
   805  	}
   806  
   807  	currTime := UTCNow()
   808  	signature, err := signStreamingRequest(req, accessKey, secretKey, currTime)
   809  	if err != nil {
   810  		return nil, err
   811  	}
   812  
   813  	req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime)
   814  	return req, err
   815  }
   816  
   817  // preSignV4 presign the request, in accordance with
   818  // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html.
   819  func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires int64) error {
   820  	// Presign is not needed for anonymous credentials.
   821  	if accessKeyID == "" || secretAccessKey == "" {
   822  		return errors.New("Presign cannot be generated without access and secret keys")
   823  	}
   824  
   825  	region := globalServerRegion
   826  	date := UTCNow()
   827  	scope := getScope(date, region)
   828  	credential := fmt.Sprintf("%s/%s", accessKeyID, scope)
   829  
   830  	// Set URL query.
   831  	query := req.URL.Query()
   832  	query.Set("X-Amz-Algorithm", signV4Algorithm)
   833  	query.Set("X-Amz-Date", date.Format(iso8601Format))
   834  	query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
   835  	query.Set("X-Amz-SignedHeaders", "host")
   836  	query.Set("X-Amz-Credential", credential)
   837  	query.Set("X-Amz-Content-Sha256", unsignedPayload)
   838  
   839  	// "host" is the only header required to be signed for Presigned URLs.
   840  	extractedSignedHeaders := make(http.Header)
   841  	extractedSignedHeaders.Set("host", req.Host)
   842  
   843  	queryStr := strings.Replace(query.Encode(), "+", "%20", -1)
   844  	canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method)
   845  	stringToSign := getStringToSign(canonicalRequest, date, scope)
   846  	signingKey := getSigningKey(secretAccessKey, date, region, serviceS3)
   847  	signature := getSignature(signingKey, stringToSign)
   848  
   849  	req.URL.RawQuery = query.Encode()
   850  
   851  	// Add signature header to RawQuery.
   852  	req.URL.RawQuery += "&X-Amz-Signature=" + url.QueryEscape(signature)
   853  
   854  	// Construct the final presigned URL.
   855  	return nil
   856  }
   857  
   858  // preSignV2 - presign the request in following style.
   859  // https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
   860  func preSignV2(req *http.Request, accessKeyID, secretAccessKey string, expires int64) error {
   861  	// Presign is not needed for anonymous credentials.
   862  	if accessKeyID == "" || secretAccessKey == "" {
   863  		return errors.New("Presign cannot be generated without access and secret keys")
   864  	}
   865  
   866  	// FIXME: Remove following portion of code after fixing a bug in minio-go preSignV2.
   867  
   868  	d := UTCNow()
   869  	// Find epoch expires when the request will expire.
   870  	epochExpires := d.Unix() + expires
   871  
   872  	// Add expires header if not present.
   873  	expiresStr := req.Header.Get("Expires")
   874  	if expiresStr == "" {
   875  		expiresStr = strconv.FormatInt(epochExpires, 10)
   876  		req.Header.Set("Expires", expiresStr)
   877  	}
   878  
   879  	// url.RawPath will be valid if path has any encoded characters, if not it will
   880  	// be empty - in which case we need to consider url.Path (bug in net/http?)
   881  	encodedResource := req.URL.RawPath
   882  	encodedQuery := req.URL.RawQuery
   883  	if encodedResource == "" {
   884  		splits := strings.SplitN(req.URL.Path, "?", 2)
   885  		encodedResource = splits[0]
   886  		if len(splits) == 2 {
   887  			encodedQuery = splits[1]
   888  		}
   889  	}
   890  
   891  	unescapedQueries, err := unescapeQueries(encodedQuery)
   892  	if err != nil {
   893  		return err
   894  	}
   895  
   896  	// Get presigned string to sign.
   897  	stringToSign := getStringToSignV2(req.Method, encodedResource, strings.Join(unescapedQueries, "&"), req.Header, expiresStr)
   898  	hm := hmac.New(sha1.New, []byte(secretAccessKey))
   899  	hm.Write([]byte(stringToSign))
   900  
   901  	// Calculate signature.
   902  	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
   903  
   904  	query := req.URL.Query()
   905  	// Handle specially for Google Cloud Storage.
   906  	query.Set("AWSAccessKeyId", accessKeyID)
   907  	// Fill in Expires for presigned query.
   908  	query.Set("Expires", strconv.FormatInt(epochExpires, 10))
   909  
   910  	// Encode query and save.
   911  	req.URL.RawQuery = query.Encode()
   912  
   913  	// Save signature finally.
   914  	req.URL.RawQuery += "&Signature=" + url.QueryEscape(signature)
   915  	return nil
   916  }
   917  
   918  // Sign given request using Signature V2.
   919  func signRequestV2(req *http.Request, accessKey, secretKey string) error {
   920  	signer.SignV2(*req, accessKey, secretKey, false)
   921  	return nil
   922  }
   923  
   924  // Sign given request using Signature V4.
   925  func signRequestV4(req *http.Request, accessKey, secretKey string) error {
   926  	// Get hashed payload.
   927  	hashedPayload := req.Header.Get("x-amz-content-sha256")
   928  	if hashedPayload == "" {
   929  		return fmt.Errorf("Invalid hashed payload")
   930  	}
   931  
   932  	currTime := UTCNow()
   933  
   934  	// Set x-amz-date.
   935  	req.Header.Set("x-amz-date", currTime.Format(iso8601Format))
   936  
   937  	// Get header map.
   938  	headerMap := make(map[string][]string)
   939  	for k, vv := range req.Header {
   940  		// If request header key is not in ignored headers, then add it.
   941  		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; !ok {
   942  			headerMap[strings.ToLower(k)] = vv
   943  		}
   944  	}
   945  
   946  	// Get header keys.
   947  	headers := []string{"host"}
   948  	for k := range headerMap {
   949  		headers = append(headers, k)
   950  	}
   951  	sort.Strings(headers)
   952  
   953  	region := globalServerRegion
   954  
   955  	// Get canonical headers.
   956  	var buf bytes.Buffer
   957  	for _, k := range headers {
   958  		buf.WriteString(k)
   959  		buf.WriteByte(':')
   960  		switch {
   961  		case k == "host":
   962  			buf.WriteString(req.URL.Host)
   963  			fallthrough
   964  		default:
   965  			for idx, v := range headerMap[k] {
   966  				if idx > 0 {
   967  					buf.WriteByte(',')
   968  				}
   969  				buf.WriteString(v)
   970  			}
   971  			buf.WriteByte('\n')
   972  		}
   973  	}
   974  	canonicalHeaders := buf.String()
   975  
   976  	// Get signed headers.
   977  	signedHeaders := strings.Join(headers, ";")
   978  
   979  	// Get canonical query string.
   980  	req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
   981  
   982  	// Get canonical URI.
   983  	canonicalURI := s3utils.EncodePath(req.URL.Path)
   984  
   985  	// Get canonical request.
   986  	// canonicalRequest =
   987  	//  <HTTPMethod>\n
   988  	//  <CanonicalURI>\n
   989  	//  <CanonicalQueryString>\n
   990  	//  <CanonicalHeaders>\n
   991  	//  <SignedHeaders>\n
   992  	//  <HashedPayload>
   993  	//
   994  	canonicalRequest := strings.Join([]string{
   995  		req.Method,
   996  		canonicalURI,
   997  		req.URL.RawQuery,
   998  		canonicalHeaders,
   999  		signedHeaders,
  1000  		hashedPayload,
  1001  	}, "\n")
  1002  
  1003  	// Get scope.
  1004  	scope := strings.Join([]string{
  1005  		currTime.Format(yyyymmdd),
  1006  		region,
  1007  		string(serviceS3),
  1008  		"aws4_request",
  1009  	}, SlashSeparator)
  1010  
  1011  	stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime.Format(iso8601Format) + "\n"
  1012  	stringToSign = stringToSign + scope + "\n"
  1013  	stringToSign = stringToSign + getSHA256Hash([]byte(canonicalRequest))
  1014  
  1015  	date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd)))
  1016  	regionHMAC := sumHMAC(date, []byte(region))
  1017  	service := sumHMAC(regionHMAC, []byte(serviceS3))
  1018  	signingKey := sumHMAC(service, []byte("aws4_request"))
  1019  
  1020  	signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
  1021  
  1022  	// final Authorization header
  1023  	parts := []string{
  1024  		"AWS4-HMAC-SHA256" + " Credential=" + accessKey + SlashSeparator + scope,
  1025  		"SignedHeaders=" + signedHeaders,
  1026  		"Signature=" + signature,
  1027  	}
  1028  	auth := strings.Join(parts, ", ")
  1029  	req.Header.Set("Authorization", auth)
  1030  
  1031  	return nil
  1032  }
  1033  
  1034  // getCredentialString generate a credential string.
  1035  func getCredentialString(accessKeyID, location string, t time.Time) string {
  1036  	return accessKeyID + SlashSeparator + getScope(t, location)
  1037  }
  1038  
  1039  // getMD5HashBase64 returns MD5 hash in base64 encoding of given data.
  1040  func getMD5HashBase64(data []byte) string {
  1041  	return base64.StdEncoding.EncodeToString(getMD5Sum(data))
  1042  }
  1043  
  1044  // Returns new HTTP request object.
  1045  func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) {
  1046  	if method == "" {
  1047  		method = http.MethodPost
  1048  	}
  1049  
  1050  	// Save for subsequent use
  1051  	var hashedPayload string
  1052  	var md5Base64 string
  1053  	switch {
  1054  	case body == nil:
  1055  		hashedPayload = getSHA256Hash([]byte{})
  1056  	default:
  1057  		payloadBytes, err := ioutil.ReadAll(body)
  1058  		if err != nil {
  1059  			return nil, err
  1060  		}
  1061  		hashedPayload = getSHA256Hash(payloadBytes)
  1062  		md5Base64 = getMD5HashBase64(payloadBytes)
  1063  	}
  1064  	// Seek back to beginning.
  1065  	if body != nil {
  1066  		body.Seek(0, 0)
  1067  	} else {
  1068  		body = bytes.NewReader([]byte(""))
  1069  	}
  1070  	req, err := http.NewRequest(method, urlStr, body)
  1071  	if err != nil {
  1072  		return nil, err
  1073  	}
  1074  	if md5Base64 != "" {
  1075  		req.Header.Set("Content-Md5", md5Base64)
  1076  	}
  1077  	req.Header.Set("x-amz-content-sha256", hashedPayload)
  1078  
  1079  	// Add Content-Length
  1080  	req.ContentLength = contentLength
  1081  
  1082  	return req, nil
  1083  }
  1084  
  1085  // Various signature types we are supporting, currently
  1086  // two main signature types.
  1087  type signerType int
  1088  
  1089  const (
  1090  	signerV2 signerType = iota
  1091  	signerV4
  1092  )
  1093  
  1094  func newTestSignedRequest(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, signer signerType) (*http.Request, error) {
  1095  	if signer == signerV2 {
  1096  		return newTestSignedRequestV2(method, urlStr, contentLength, body, accessKey, secretKey, nil)
  1097  	}
  1098  	return newTestSignedRequestV4(method, urlStr, contentLength, body, accessKey, secretKey, nil)
  1099  }
  1100  
  1101  // Returns request with correct signature but with incorrect SHA256.
  1102  func newTestSignedBadSHARequest(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, signer signerType) (*http.Request, error) {
  1103  	req, err := newTestRequest(method, urlStr, contentLength, body)
  1104  	if err != nil {
  1105  		return nil, err
  1106  	}
  1107  
  1108  	// Anonymous request return early.
  1109  	if accessKey == "" || secretKey == "" {
  1110  		return req, nil
  1111  	}
  1112  
  1113  	if signer == signerV2 {
  1114  		err = signRequestV2(req, accessKey, secretKey)
  1115  		req.Header.Del("x-amz-content-sha256")
  1116  	} else {
  1117  		req.Header.Set("x-amz-content-sha256", "92b165232fbd011da355eca0b033db22b934ba9af0145a437a832d27310b89f9")
  1118  		err = signRequestV4(req, accessKey, secretKey)
  1119  	}
  1120  
  1121  	return req, err
  1122  }
  1123  
  1124  // Returns new HTTP request object signed with signature v2.
  1125  func newTestSignedRequestV2(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, headers map[string]string) (*http.Request, error) {
  1126  	req, err := newTestRequest(method, urlStr, contentLength, body)
  1127  	if err != nil {
  1128  		return nil, err
  1129  	}
  1130  	req.Header.Del("x-amz-content-sha256")
  1131  
  1132  	// Anonymous request return quickly.
  1133  	if accessKey == "" || secretKey == "" {
  1134  		return req, nil
  1135  	}
  1136  
  1137  	for k, v := range headers {
  1138  		req.Header.Set(k, v)
  1139  	}
  1140  
  1141  	err = signRequestV2(req, accessKey, secretKey)
  1142  	if err != nil {
  1143  		return nil, err
  1144  	}
  1145  
  1146  	return req, nil
  1147  }
  1148  
  1149  // Returns new HTTP request object signed with signature v4.
  1150  func newTestSignedRequestV4(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, headers map[string]string) (*http.Request, error) {
  1151  	req, err := newTestRequest(method, urlStr, contentLength, body)
  1152  	if err != nil {
  1153  		return nil, err
  1154  	}
  1155  
  1156  	// Anonymous request return quickly.
  1157  	if accessKey == "" || secretKey == "" {
  1158  		return req, nil
  1159  	}
  1160  
  1161  	for k, v := range headers {
  1162  		req.Header.Set(k, v)
  1163  	}
  1164  
  1165  	err = signRequestV4(req, accessKey, secretKey)
  1166  	if err != nil {
  1167  		return nil, err
  1168  	}
  1169  
  1170  	return req, nil
  1171  }
  1172  
  1173  // Return new WebRPC request object.
  1174  func newWebRPCRequest(methodRPC, authorization string, body io.ReadSeeker) (*http.Request, error) {
  1175  	req, err := http.NewRequest(http.MethodPost, "/minio/webrpc", nil)
  1176  	if err != nil {
  1177  		return nil, err
  1178  	}
  1179  	req.Header.Set("User-Agent", "Mozilla")
  1180  	req.Header.Set("Content-Type", "application/json")
  1181  	if authorization != "" {
  1182  		req.Header.Set("Authorization", "Bearer "+authorization)
  1183  	}
  1184  	// Seek back to beginning.
  1185  	if body != nil {
  1186  		body.Seek(0, 0)
  1187  		// Add body
  1188  		req.Body = ioutil.NopCloser(body)
  1189  	} else {
  1190  		// this is added to avoid panic during ioutil.ReadAll(req.Body).
  1191  		// th stack trace can be found here  https://github.com/minio/minio/pull/2074 .
  1192  		// This is very similar to https://github.com/golang/go/issues/7527.
  1193  		req.Body = ioutil.NopCloser(bytes.NewReader([]byte("")))
  1194  	}
  1195  	return req, nil
  1196  }
  1197  
  1198  // Marshal request and return a new HTTP request object to call the webrpc
  1199  func newTestWebRPCRequest(rpcMethod string, authorization string, data interface{}) (*http.Request, error) {
  1200  	type genericJSON struct {
  1201  		JSONRPC string      `json:"jsonrpc"`
  1202  		ID      string      `json:"id"`
  1203  		Method  string      `json:"method"`
  1204  		Params  interface{} `json:"params"`
  1205  	}
  1206  	encapsulatedData := genericJSON{JSONRPC: "2.0", ID: "1", Method: rpcMethod, Params: data}
  1207  	jsonData, err := json.Marshal(encapsulatedData)
  1208  	if err != nil {
  1209  		return nil, err
  1210  	}
  1211  	req, err := newWebRPCRequest(rpcMethod, authorization, bytes.NewReader(jsonData))
  1212  	if err != nil {
  1213  		return nil, err
  1214  	}
  1215  	return req, nil
  1216  }
  1217  
  1218  type ErrWebRPC struct {
  1219  	Code    int         `json:"code"`
  1220  	Message string      `json:"message"`
  1221  	Data    interface{} `json:"data"`
  1222  }
  1223  
  1224  // Unmarshal response and return the webrpc response
  1225  func getTestWebRPCResponse(resp *httptest.ResponseRecorder, data interface{}) error {
  1226  	type rpcReply struct {
  1227  		ID      string      `json:"id"`
  1228  		JSONRPC string      `json:"jsonrpc"`
  1229  		Result  interface{} `json:"result"`
  1230  		Error   *ErrWebRPC  `json:"error"`
  1231  	}
  1232  	reply := &rpcReply{Result: &data}
  1233  	err := json.NewDecoder(resp.Body).Decode(reply)
  1234  	if err != nil {
  1235  		return err
  1236  	}
  1237  	// For the moment, web handlers errors code are not meaningful
  1238  	// Return only the error message
  1239  	if reply.Error != nil {
  1240  		return errors.New(reply.Error.Message)
  1241  	}
  1242  	return nil
  1243  }
  1244  
  1245  // Function to generate random string for bucket/object names.
  1246  func randString(n int) string {
  1247  	src := rand.NewSource(UTCNow().UnixNano())
  1248  
  1249  	b := make([]byte, n)
  1250  	// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
  1251  	for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
  1252  		if remain == 0 {
  1253  			cache, remain = src.Int63(), letterIdxMax
  1254  		}
  1255  		if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
  1256  			b[i] = letterBytes[idx]
  1257  			i--
  1258  		}
  1259  		cache >>= letterIdxBits
  1260  		remain--
  1261  	}
  1262  	return string(b)
  1263  }
  1264  
  1265  // generate random object name.
  1266  func getRandomObjectName() string {
  1267  	return randString(16)
  1268  
  1269  }
  1270  
  1271  // generate random bucket name.
  1272  func getRandomBucketName() string {
  1273  	return randString(60)
  1274  
  1275  }
  1276  
  1277  // NewEOFWriter returns a Writer that writes to w,
  1278  // but returns EOF error after writing n bytes.
  1279  func NewEOFWriter(w io.Writer, n int64) io.Writer {
  1280  	return &EOFWriter{w, n}
  1281  }
  1282  
  1283  type EOFWriter struct {
  1284  	w io.Writer
  1285  	n int64
  1286  }
  1287  
  1288  // io.Writer implementation designed to error out with io.EOF after reading `n` bytes.
  1289  func (t *EOFWriter) Write(p []byte) (n int, err error) {
  1290  	if t.n <= 0 {
  1291  		return -1, io.EOF
  1292  	}
  1293  	// real write
  1294  	n = len(p)
  1295  	if int64(n) > t.n {
  1296  		n = int(t.n)
  1297  	}
  1298  	n, err = t.w.Write(p[0:n])
  1299  	t.n -= int64(n)
  1300  	if err == nil {
  1301  		n = len(p)
  1302  	}
  1303  	return
  1304  }
  1305  
  1306  // construct URL for http requests for bucket operations.
  1307  func makeTestTargetURL(endPoint, bucketName, objectName string, queryValues url.Values) string {
  1308  	urlStr := endPoint + SlashSeparator
  1309  	if bucketName != "" {
  1310  		urlStr = urlStr + bucketName + SlashSeparator
  1311  	}
  1312  	if objectName != "" {
  1313  		urlStr = urlStr + s3utils.EncodePath(objectName)
  1314  	}
  1315  	if len(queryValues) > 0 {
  1316  		urlStr = urlStr + "?" + queryValues.Encode()
  1317  	}
  1318  	return urlStr
  1319  }
  1320  
  1321  // return URL for uploading object into the bucket.
  1322  func getPutObjectURL(endPoint, bucketName, objectName string) string {
  1323  	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
  1324  }
  1325  
  1326  func getPutObjectPartURL(endPoint, bucketName, objectName, uploadID, partNumber string) string {
  1327  	queryValues := url.Values{}
  1328  	queryValues.Set("uploadId", uploadID)
  1329  	queryValues.Set("partNumber", partNumber)
  1330  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValues)
  1331  }
  1332  
  1333  func getCopyObjectPartURL(endPoint, bucketName, objectName, uploadID, partNumber string) string {
  1334  	queryValues := url.Values{}
  1335  	queryValues.Set("uploadId", uploadID)
  1336  	queryValues.Set("partNumber", partNumber)
  1337  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValues)
  1338  }
  1339  
  1340  // return URL for fetching object from the bucket.
  1341  func getGetObjectURL(endPoint, bucketName, objectName string) string {
  1342  	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
  1343  }
  1344  
  1345  // return URL for deleting the object from the bucket.
  1346  func getDeleteObjectURL(endPoint, bucketName, objectName string) string {
  1347  	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
  1348  }
  1349  
  1350  // return URL for deleting multiple objects from a bucket.
  1351  func getMultiDeleteObjectURL(endPoint, bucketName string) string {
  1352  	queryValue := url.Values{}
  1353  	queryValue.Set("delete", "")
  1354  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1355  
  1356  }
  1357  
  1358  // return URL for HEAD on the object.
  1359  func getHeadObjectURL(endPoint, bucketName, objectName string) string {
  1360  	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
  1361  }
  1362  
  1363  // return url to be used while copying the object.
  1364  func getCopyObjectURL(endPoint, bucketName, objectName string) string {
  1365  	return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{})
  1366  }
  1367  
  1368  // return URL for inserting bucket notification.
  1369  func getPutNotificationURL(endPoint, bucketName string) string {
  1370  	queryValue := url.Values{}
  1371  	queryValue.Set("notification", "")
  1372  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1373  }
  1374  
  1375  // return URL for inserting bucket policy.
  1376  func getPutPolicyURL(endPoint, bucketName string) string {
  1377  	queryValue := url.Values{}
  1378  	queryValue.Set("policy", "")
  1379  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1380  }
  1381  
  1382  // return URL for fetching bucket policy.
  1383  func getGetPolicyURL(endPoint, bucketName string) string {
  1384  	queryValue := url.Values{}
  1385  	queryValue.Set("policy", "")
  1386  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1387  }
  1388  
  1389  // return URL for deleting bucket policy.
  1390  func getDeletePolicyURL(endPoint, bucketName string) string {
  1391  	queryValue := url.Values{}
  1392  	queryValue.Set("policy", "")
  1393  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1394  }
  1395  
  1396  // return URL for creating the bucket.
  1397  func getMakeBucketURL(endPoint, bucketName string) string {
  1398  	return makeTestTargetURL(endPoint, bucketName, "", url.Values{})
  1399  }
  1400  
  1401  // return URL for listing buckets.
  1402  func getListBucketURL(endPoint string) string {
  1403  	return makeTestTargetURL(endPoint, "", "", url.Values{})
  1404  }
  1405  
  1406  // return URL for HEAD on the bucket.
  1407  func getHEADBucketURL(endPoint, bucketName string) string {
  1408  	return makeTestTargetURL(endPoint, bucketName, "", url.Values{})
  1409  }
  1410  
  1411  // return URL for deleting the bucket.
  1412  func getDeleteBucketURL(endPoint, bucketName string) string {
  1413  	return makeTestTargetURL(endPoint, bucketName, "", url.Values{})
  1414  }
  1415  
  1416  // return URL for deleting the bucket.
  1417  func getDeleteMultipleObjectsURL(endPoint, bucketName string) string {
  1418  	queryValue := url.Values{}
  1419  	queryValue.Set("delete", "")
  1420  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1421  }
  1422  
  1423  // return URL For fetching location of the bucket.
  1424  func getBucketLocationURL(endPoint, bucketName string) string {
  1425  	queryValue := url.Values{}
  1426  	queryValue.Set("location", "")
  1427  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1428  }
  1429  
  1430  // return URL For set/get lifecycle of the bucket.
  1431  func getBucketLifecycleURL(endPoint, bucketName string) (ret string) {
  1432  	queryValue := url.Values{}
  1433  	queryValue.Set("lifecycle", "")
  1434  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1435  }
  1436  
  1437  // return URL for listing objects in the bucket with V1 legacy API.
  1438  func getListObjectsV1URL(endPoint, bucketName, prefix, maxKeys, encodingType string) string {
  1439  	queryValue := url.Values{}
  1440  	if maxKeys != "" {
  1441  		queryValue.Set("max-keys", maxKeys)
  1442  	}
  1443  	if encodingType != "" {
  1444  		queryValue.Set("encoding-type", encodingType)
  1445  	}
  1446  	return makeTestTargetURL(endPoint, bucketName, prefix, queryValue)
  1447  }
  1448  
  1449  // return URL for listing objects in the bucket with V2 API.
  1450  func getListObjectsV2URL(endPoint, bucketName, prefix, maxKeys, fetchOwner, encodingType string) string {
  1451  	queryValue := url.Values{}
  1452  	queryValue.Set("list-type", "2") // Enables list objects V2 URL.
  1453  	if maxKeys != "" {
  1454  		queryValue.Set("max-keys", maxKeys)
  1455  	}
  1456  	if fetchOwner != "" {
  1457  		queryValue.Set("fetch-owner", fetchOwner)
  1458  	}
  1459  	if encodingType != "" {
  1460  		queryValue.Set("encoding-type", encodingType)
  1461  	}
  1462  	return makeTestTargetURL(endPoint, bucketName, prefix, queryValue)
  1463  }
  1464  
  1465  // return URL for a new multipart upload.
  1466  func getNewMultipartURL(endPoint, bucketName, objectName string) string {
  1467  	queryValue := url.Values{}
  1468  	queryValue.Set("uploads", "")
  1469  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValue)
  1470  }
  1471  
  1472  // return URL for a new multipart upload.
  1473  func getPartUploadURL(endPoint, bucketName, objectName, uploadID, partNumber string) string {
  1474  	queryValues := url.Values{}
  1475  	queryValues.Set("uploadId", uploadID)
  1476  	queryValues.Set("partNumber", partNumber)
  1477  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValues)
  1478  }
  1479  
  1480  // return URL for aborting multipart upload.
  1481  func getAbortMultipartUploadURL(endPoint, bucketName, objectName, uploadID string) string {
  1482  	queryValue := url.Values{}
  1483  	queryValue.Set("uploadId", uploadID)
  1484  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValue)
  1485  }
  1486  
  1487  // return URL for a listing pending multipart uploads.
  1488  func getListMultipartURL(endPoint, bucketName string) string {
  1489  	queryValue := url.Values{}
  1490  	queryValue.Set("uploads", "")
  1491  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1492  }
  1493  
  1494  // return URL for listing pending multipart uploads with parameters.
  1495  func getListMultipartUploadsURLWithParams(endPoint, bucketName, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads string) string {
  1496  	queryValue := url.Values{}
  1497  	queryValue.Set("uploads", "")
  1498  	queryValue.Set("prefix", prefix)
  1499  	queryValue.Set("delimiter", delimiter)
  1500  	queryValue.Set("key-marker", keyMarker)
  1501  	queryValue.Set("upload-id-marker", uploadIDMarker)
  1502  	queryValue.Set("max-uploads", maxUploads)
  1503  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1504  }
  1505  
  1506  // return URL for a listing parts on a given upload id.
  1507  func getListMultipartURLWithParams(endPoint, bucketName, objectName, uploadID, maxParts, partNumberMarker, encoding string) string {
  1508  	queryValues := url.Values{}
  1509  	queryValues.Set("uploadId", uploadID)
  1510  	queryValues.Set("max-parts", maxParts)
  1511  	if partNumberMarker != "" {
  1512  		queryValues.Set("part-number-marker", partNumberMarker)
  1513  	}
  1514  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValues)
  1515  }
  1516  
  1517  // return URL for completing multipart upload.
  1518  // complete multipart upload request is sent after all parts are uploaded.
  1519  func getCompleteMultipartUploadURL(endPoint, bucketName, objectName, uploadID string) string {
  1520  	queryValue := url.Values{}
  1521  	queryValue.Set("uploadId", uploadID)
  1522  	return makeTestTargetURL(endPoint, bucketName, objectName, queryValue)
  1523  }
  1524  
  1525  // return URL for listen bucket notification.
  1526  func getListenNotificationURL(endPoint, bucketName string, prefixes, suffixes, events []string) string {
  1527  	queryValue := url.Values{}
  1528  
  1529  	queryValue["prefix"] = prefixes
  1530  	queryValue["suffix"] = suffixes
  1531  	queryValue["events"] = events
  1532  	return makeTestTargetURL(endPoint, bucketName, "", queryValue)
  1533  }
  1534  
  1535  // returns temp root directory. `
  1536  func getTestRoot() (string, error) {
  1537  	return ioutil.TempDir(globalTestTmpDir, "api-")
  1538  }
  1539  
  1540  // getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX
  1541  func getRandomDisks(N int) ([]string, error) {
  1542  	var erasureDisks []string
  1543  	for i := 0; i < N; i++ {
  1544  		path, err := ioutil.TempDir(globalTestTmpDir, "minio-")
  1545  		if err != nil {
  1546  			// Remove directories created so far.
  1547  			removeRoots(erasureDisks)
  1548  			return nil, err
  1549  		}
  1550  		erasureDisks = append(erasureDisks, path)
  1551  	}
  1552  	return erasureDisks, nil
  1553  }
  1554  
  1555  // Initialize object layer with the supplied disks, objectLayer is nil upon any error.
  1556  func newTestObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools) (newObject ObjectLayer, err error) {
  1557  	// For FS only, directly use the disk.
  1558  	if endpointServerPools.NEndpoints() == 1 {
  1559  		// Initialize new FS object layer.
  1560  		return NewFSObjectLayer(endpointServerPools[0].Endpoints[0].Path)
  1561  	}
  1562  
  1563  	z, err := newErasureServerPools(ctx, endpointServerPools)
  1564  	if err != nil {
  1565  		return nil, err
  1566  	}
  1567  
  1568  	newAllSubsystems()
  1569  
  1570  	initAllSubsystems(ctx, z)
  1571  
  1572  	return z, nil
  1573  }
  1574  
  1575  // initObjectLayer - Instantiates object layer and returns it.
  1576  func initObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, []StorageAPI, error) {
  1577  	objLayer, err := newTestObjectLayer(ctx, endpointServerPools)
  1578  	if err != nil {
  1579  		return nil, nil, err
  1580  	}
  1581  
  1582  	var formattedDisks []StorageAPI
  1583  	// Should use the object layer tests for validating cache.
  1584  	if z, ok := objLayer.(*erasureServerPools); ok {
  1585  		formattedDisks = z.serverPools[0].GetDisks(0)()
  1586  	}
  1587  
  1588  	// Success.
  1589  	return objLayer, formattedDisks, nil
  1590  }
  1591  
  1592  // removeRoots - Cleans up initialized directories during tests.
  1593  func removeRoots(roots []string) {
  1594  	for _, root := range roots {
  1595  		os.RemoveAll(root)
  1596  	}
  1597  }
  1598  
  1599  //removeDiskN - removes N disks from supplied disk slice.
  1600  func removeDiskN(disks []string, n int) {
  1601  	if n > len(disks) {
  1602  		n = len(disks)
  1603  	}
  1604  	for _, disk := range disks[:n] {
  1605  		os.RemoveAll(disk)
  1606  	}
  1607  }
  1608  
  1609  // creates a bucket for the tests and returns the bucket name.
  1610  // initializes the specified API endpoints for the tests.
  1611  // initialies the root and returns its path.
  1612  // return credentials.
  1613  func initAPIHandlerTest(obj ObjectLayer, endpoints []string) (string, http.Handler, error) {
  1614  	newAllSubsystems()
  1615  
  1616  	initAllSubsystems(context.Background(), obj)
  1617  
  1618  	// get random bucket name.
  1619  	bucketName := getRandomBucketName()
  1620  
  1621  	// Create bucket.
  1622  	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
  1623  	if err != nil {
  1624  		// failed to create newbucket, return err.
  1625  		return "", nil, err
  1626  	}
  1627  	// Register the API end points with Erasure object layer.
  1628  	// Registering only the GetObject handler.
  1629  	apiRouter := initTestAPIEndPoints(obj, endpoints)
  1630  	f := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1631  		r.RequestURI = r.URL.RequestURI()
  1632  		apiRouter.ServeHTTP(w, r)
  1633  	})
  1634  	return bucketName, f, nil
  1635  }
  1636  
  1637  // prepare test backend.
  1638  // create FS/Erasure/ErasureSet backend.
  1639  // return object layer, backend disks.
  1640  func prepareTestBackend(ctx context.Context, instanceType string) (ObjectLayer, []string, error) {
  1641  	switch instanceType {
  1642  	// Total number of disks for Erasure sets backend is set to 32.
  1643  	case ErasureSetsTestStr:
  1644  		return prepareErasureSets32(ctx)
  1645  	// Total number of disks for Erasure backend is set to 16.
  1646  	case ErasureTestStr:
  1647  		return prepareErasure16(ctx)
  1648  	default:
  1649  		// return FS backend by default.
  1650  		obj, disk, err := prepareFS()
  1651  		if err != nil {
  1652  			return nil, nil, err
  1653  		}
  1654  		return obj, []string{disk}, nil
  1655  	}
  1656  }
  1657  
  1658  // ExecObjectLayerAPIAnonTest - Helper function to validate object Layer API handler
  1659  // response for anonymous/unsigned and unknown signature type HTTP request.
  1660  
  1661  // Here is the brief description of some of the arguments to the function below.
  1662  //   apiRouter - http.Handler with the relevant API endPoint (API endPoint under test) registered.
  1663  //   anonReq   - unsigned *http.Request to invoke the handler's response for anonymous requests.
  1664  //   policyFunc    - function to return bucketPolicy statement which would permit the anonymous request to be served.
  1665  // The test works in 2 steps, here is the description of the steps.
  1666  //   STEP 1: Call the handler with the unsigned HTTP request (anonReq), assert for the `ErrAccessDenied` error response.
  1667  func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketName, objectName, instanceType string, apiRouter http.Handler,
  1668  	anonReq *http.Request, bucketPolicy *policy.Policy) {
  1669  
  1670  	anonTestStr := "Anonymous HTTP request test"
  1671  	unknownSignTestStr := "Unknown HTTP signature test"
  1672  
  1673  	// simple function which returns a message which gives the context of the test
  1674  	// and then followed by the the actual error message.
  1675  	failTestStr := func(testType, failMsg string) string {
  1676  		return fmt.Sprintf("MinIO %s: %s fail for \"%s\": \n<Error> %s", instanceType, testType, testName, failMsg)
  1677  	}
  1678  
  1679  	// httptest Recorder to capture all the response by the http handler.
  1680  	rec := httptest.NewRecorder()
  1681  	// reading the body to preserve it so that it can be used again for second attempt of sending unsigned HTTP request.
  1682  	// If the body is read in the handler the same request cannot be made use of.
  1683  	buf, err := ioutil.ReadAll(anonReq.Body)
  1684  	if err != nil {
  1685  		t.Fatal(failTestStr(anonTestStr, err.Error()))
  1686  	}
  1687  
  1688  	// creating 2 read closer (to set as request body) from the body content.
  1689  	readerOne := ioutil.NopCloser(bytes.NewBuffer(buf))
  1690  	readerTwo := ioutil.NopCloser(bytes.NewBuffer(buf))
  1691  
  1692  	anonReq.Body = readerOne
  1693  
  1694  	// call the HTTP handler.
  1695  	apiRouter.ServeHTTP(rec, anonReq)
  1696  
  1697  	// expected error response when the unsigned HTTP request is not permitted.
  1698  	accessDenied := GetAPIError(ErrAccessDenied).HTTPStatusCode
  1699  	if rec.Code != accessDenied {
  1700  		t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Object API Nil Test expected to fail with %d, but failed with %d", accessDenied, rec.Code)))
  1701  	}
  1702  
  1703  	// HEAD HTTTP request doesn't contain response body.
  1704  	if anonReq.Method != http.MethodHead {
  1705  		// read the response body.
  1706  		var actualContent []byte
  1707  		actualContent, err = ioutil.ReadAll(rec.Body)
  1708  		if err != nil {
  1709  			t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Failed parsing response body: <ERROR> %v", err)))
  1710  		}
  1711  
  1712  		actualError := &APIErrorResponse{}
  1713  		if err = xml.Unmarshal(actualContent, actualError); err != nil {
  1714  			t.Fatal(failTestStr(anonTestStr, "error response failed to parse error XML"))
  1715  		}
  1716  
  1717  		if actualError.BucketName != bucketName {
  1718  			t.Fatal(failTestStr(anonTestStr, "error response bucket name differs from expected value"))
  1719  		}
  1720  
  1721  		if actualError.Key != objectName {
  1722  			t.Fatal(failTestStr(anonTestStr, "error response object name differs from expected value"))
  1723  		}
  1724  	}
  1725  
  1726  	// test for unknown auth case.
  1727  	anonReq.Body = readerTwo
  1728  	// Setting the `Authorization` header to a random value so that the signature falls into unknown auth case.
  1729  	anonReq.Header.Set("Authorization", "nothingElse")
  1730  	// initialize new response recorder.
  1731  	rec = httptest.NewRecorder()
  1732  	// call the handler using the HTTP Request.
  1733  	apiRouter.ServeHTTP(rec, anonReq)
  1734  	// verify the response body for `ErrAccessDenied` message =.
  1735  	if anonReq.Method != http.MethodHead {
  1736  		// read the response body.
  1737  		actualContent, err := ioutil.ReadAll(rec.Body)
  1738  		if err != nil {
  1739  			t.Fatal(failTestStr(unknownSignTestStr, fmt.Sprintf("Failed parsing response body: <ERROR> %v", err)))
  1740  		}
  1741  
  1742  		actualError := &APIErrorResponse{}
  1743  		if err = xml.Unmarshal(actualContent, actualError); err != nil {
  1744  			t.Fatal(failTestStr(unknownSignTestStr, "error response failed to parse error XML"))
  1745  		}
  1746  
  1747  		if actualError.BucketName != bucketName {
  1748  			t.Fatal(failTestStr(unknownSignTestStr, "error response bucket name differs from expected value"))
  1749  		}
  1750  
  1751  		if actualError.Key != objectName {
  1752  			t.Fatal(failTestStr(unknownSignTestStr, "error response object name differs from expected value"))
  1753  		}
  1754  	}
  1755  
  1756  	// expected error response when the unsigned HTTP request is not permitted.
  1757  	unsupportedSignature := GetAPIError(ErrSignatureVersionNotSupported).HTTPStatusCode
  1758  	if rec.Code != unsupportedSignature {
  1759  		t.Fatal(failTestStr(unknownSignTestStr, fmt.Sprintf("Object API Unknow auth test for \"%s\", expected to fail with %d, but failed with %d", testName, unsupportedSignature, rec.Code)))
  1760  	}
  1761  
  1762  }
  1763  
  1764  // ExecObjectLayerAPINilTest - Sets the object layer to `nil`, and calls rhe registered object layer API endpoint,
  1765  // and assert the error response. The purpose is to validate the API handlers response when the object layer is uninitialized.
  1766  // Usage hint: Should be used at the end of the API end points tests (ex: check the last few lines of `testAPIListObjectPartsHandler`),
  1767  // need a sample HTTP request to be sent as argument so that the relevant handler is called, the handler registration is expected
  1768  // to be done since its called from within the API handler tests, the reference to the registered HTTP handler has to be sent
  1769  // as an argument.
  1770  func ExecObjectLayerAPINilTest(t TestErrHandler, bucketName, objectName, instanceType string, apiRouter http.Handler, req *http.Request) {
  1771  	// httptest Recorder to capture all the response by the http handler.
  1772  	rec := httptest.NewRecorder()
  1773  
  1774  	// The  API handler gets the referece to the object layer via the global object Layer,
  1775  	// setting it to `nil` in order test for handlers response for uninitialized object layer.
  1776  	globalObjLayerMutex.Lock()
  1777  	globalObjectAPI = nil
  1778  	globalObjLayerMutex.Unlock()
  1779  
  1780  	// call the HTTP handler.
  1781  	apiRouter.ServeHTTP(rec, req)
  1782  
  1783  	// expected error response when the API handler is called before the object layer is initialized,
  1784  	// or when objectLayer is `nil`.
  1785  	serverNotInitializedErr := GetAPIError(ErrServerNotInitialized).HTTPStatusCode
  1786  	if rec.Code != serverNotInitializedErr {
  1787  		t.Errorf("Object API Nil Test expected to fail with %d, but failed with %d", serverNotInitializedErr, rec.Code)
  1788  	}
  1789  
  1790  	// HEAD HTTP Request doesn't contain body in its response,
  1791  	// for other type of HTTP requests compare the response body content with the expected one.
  1792  	if req.Method != http.MethodHead {
  1793  		// read the response body.
  1794  		actualContent, err := ioutil.ReadAll(rec.Body)
  1795  		if err != nil {
  1796  			t.Fatalf("MinIO %s: Failed parsing response body: <ERROR> %v", instanceType, err)
  1797  		}
  1798  
  1799  		actualError := &APIErrorResponse{}
  1800  		if err = xml.Unmarshal(actualContent, actualError); err != nil {
  1801  			t.Errorf("MinIO %s: error response failed to parse error XML", instanceType)
  1802  		}
  1803  
  1804  		if actualError.BucketName != bucketName {
  1805  			t.Errorf("MinIO %s: error response bucket name differs from expected value", instanceType)
  1806  		}
  1807  
  1808  		if actualError.Key != objectName {
  1809  			t.Errorf("MinIO %s: error response object name differs from expected value", instanceType)
  1810  		}
  1811  	}
  1812  }
  1813  
  1814  // ExecObjectLayerAPITest - executes object layer API tests.
  1815  // Creates single node and Erasure ObjectLayer instance, registers the specified API end points and runs test for both the layers.
  1816  func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
  1817  	ctx, cancel := context.WithCancel(context.Background())
  1818  	defer cancel()
  1819  
  1820  	// reset globals.
  1821  	// this is to make sure that the tests are not affected by modified value.
  1822  	resetTestGlobals()
  1823  
  1824  	objLayer, fsDir, err := prepareFS()
  1825  	if err != nil {
  1826  		t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
  1827  	}
  1828  
  1829  	bucketFS, fsAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
  1830  	if err != nil {
  1831  		t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
  1832  	}
  1833  
  1834  	// initialize the server and obtain the credentials and root.
  1835  	// credentials are necessary to sign the HTTP request.
  1836  	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
  1837  		t.Fatalf("Unable to initialize server config. %s", err)
  1838  	}
  1839  
  1840  	credentials := globalActiveCred
  1841  
  1842  	// Executing the object layer tests for single node setup.
  1843  	objAPITest(objLayer, FSTestStr, bucketFS, fsAPIRouter, credentials, t)
  1844  
  1845  	objLayer, erasureDisks, err := prepareErasure16(ctx)
  1846  	if err != nil {
  1847  		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
  1848  	}
  1849  	defer objLayer.Shutdown(ctx)
  1850  
  1851  	bucketErasure, erAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
  1852  	if err != nil {
  1853  		t.Fatalf("Initialzation of API handler tests failed: <ERROR> %s", err)
  1854  	}
  1855  	// Executing the object layer tests for Erasure.
  1856  	objAPITest(objLayer, ErasureTestStr, bucketErasure, erAPIRouter, credentials, t)
  1857  
  1858  	// clean up the temporary test backend.
  1859  	removeRoots(append(erasureDisks, fsDir))
  1860  }
  1861  
  1862  // ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed.
  1863  // This can be used to test functionality when reading and writing data.
  1864  func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) {
  1865  	execExtended(t, func(t *testing.T) {
  1866  		ExecObjectLayerAPITest(t, objAPITest, endpoints)
  1867  	})
  1868  }
  1869  
  1870  // function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests.
  1871  type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string,
  1872  	apiRouter http.Handler, credentials auth.Credentials, t *testing.T)
  1873  
  1874  // Regular object test type.
  1875  type objTestType func(obj ObjectLayer, instanceType string, t TestErrHandler)
  1876  
  1877  // Special test type for test with directories
  1878  type objTestTypeWithDirs func(obj ObjectLayer, instanceType string, dirs []string, t TestErrHandler)
  1879  
  1880  // Special object test type for disk not found situations.
  1881  type objTestDiskNotFoundType func(obj ObjectLayer, instanceType string, dirs []string, t *testing.T)
  1882  
  1883  // ExecObjectLayerTest - executes object layer tests.
  1884  // Creates single node and Erasure ObjectLayer instance and runs test for both the layers.
  1885  func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
  1886  	ctx, cancel := context.WithCancel(context.Background())
  1887  	defer cancel()
  1888  
  1889  	if localMetacacheMgr != nil {
  1890  		localMetacacheMgr.deleteAll()
  1891  	}
  1892  	defer SetObjectLayer(newObjectLayerFn())
  1893  
  1894  	objLayer, fsDir, err := prepareFS()
  1895  	if err != nil {
  1896  		t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
  1897  	}
  1898  	SetObjectLayer(objLayer)
  1899  
  1900  	newAllSubsystems()
  1901  
  1902  	// initialize the server and obtain the credentials and root.
  1903  	// credentials are necessary to sign the HTTP request.
  1904  	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
  1905  		t.Fatal("Unexpected error", err)
  1906  	}
  1907  
  1908  	initAllSubsystems(ctx, objLayer)
  1909  
  1910  	// Executing the object layer tests for single node setup.
  1911  	objTest(objLayer, FSTestStr, t)
  1912  
  1913  	if localMetacacheMgr != nil {
  1914  		localMetacacheMgr.deleteAll()
  1915  	}
  1916  	defer SetObjectLayer(newObjectLayerFn())
  1917  
  1918  	newAllSubsystems()
  1919  	objLayer, fsDirs, err := prepareErasureSets32(ctx)
  1920  	if err != nil {
  1921  		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
  1922  	}
  1923  	SetObjectLayer(objLayer)
  1924  
  1925  	defer objLayer.Shutdown(context.Background())
  1926  
  1927  	initAllSubsystems(ctx, objLayer)
  1928  
  1929  	defer removeRoots(append(fsDirs, fsDir))
  1930  	// Executing the object layer tests for Erasure.
  1931  	objTest(objLayer, ErasureTestStr, t)
  1932  
  1933  	if localMetacacheMgr != nil {
  1934  		localMetacacheMgr.deleteAll()
  1935  	}
  1936  }
  1937  
  1938  // ExecObjectLayerTestWithDirs - executes object layer tests.
  1939  // Creates single node and Erasure ObjectLayer instance and runs test for both the layers.
  1940  func ExecObjectLayerTestWithDirs(t TestErrHandler, objTest objTestTypeWithDirs) {
  1941  	ctx, cancel := context.WithCancel(context.Background())
  1942  	defer cancel()
  1943  
  1944  	objLayer, fsDirs, err := prepareErasure16(ctx)
  1945  	if err != nil {
  1946  		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
  1947  	}
  1948  	defer objLayer.Shutdown(ctx)
  1949  
  1950  	// initialize the server and obtain the credentials and root.
  1951  	// credentials are necessary to sign the HTTP request.
  1952  	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
  1953  		t.Fatal("Unexpected error", err)
  1954  	}
  1955  
  1956  	// Executing the object layer tests for Erasure.
  1957  	objTest(objLayer, ErasureTestStr, fsDirs, t)
  1958  	defer removeRoots(fsDirs)
  1959  }
  1960  
  1961  // ExecObjectLayerDiskAlteredTest - executes object layer tests while altering
  1962  // disks in between tests. Creates Erasure ObjectLayer instance and runs test for Erasure layer.
  1963  func ExecObjectLayerDiskAlteredTest(t *testing.T, objTest objTestDiskNotFoundType) {
  1964  	ctx, cancel := context.WithCancel(context.Background())
  1965  	defer cancel()
  1966  
  1967  	objLayer, fsDirs, err := prepareErasure16(ctx)
  1968  	if err != nil {
  1969  		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
  1970  	}
  1971  	defer objLayer.Shutdown(ctx)
  1972  
  1973  	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
  1974  		t.Fatal("Failed to create config directory", err)
  1975  	}
  1976  
  1977  	// Executing the object layer tests for Erasure.
  1978  	objTest(objLayer, ErasureTestStr, fsDirs, t)
  1979  	defer removeRoots(fsDirs)
  1980  }
  1981  
  1982  // Special object test type for stale files situations.
  1983  type objTestStaleFilesType func(obj ObjectLayer, instanceType string, dirs []string, t *testing.T)
  1984  
  1985  // ExecObjectLayerStaleFilesTest - executes object layer tests those leaves stale
  1986  // files/directories under .minio/tmp.  Creates Erasure ObjectLayer instance and runs test for Erasure layer.
  1987  func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) {
  1988  	ctx, cancel := context.WithCancel(context.Background())
  1989  	defer cancel()
  1990  
  1991  	nDisks := 16
  1992  	erasureDisks, err := getRandomDisks(nDisks)
  1993  	if err != nil {
  1994  		t.Fatalf("Initialization of disks for Erasure setup: %s", err)
  1995  	}
  1996  	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(erasureDisks...))
  1997  	if err != nil {
  1998  		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
  1999  	}
  2000  	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
  2001  		t.Fatal("Failed to create config directory", err)
  2002  	}
  2003  
  2004  	// Executing the object layer tests for Erasure.
  2005  	objTest(objLayer, ErasureTestStr, erasureDisks, t)
  2006  	defer removeRoots(erasureDisks)
  2007  }
  2008  
  2009  func registerBucketLevelFunc(bucket *mux.Router, api ObjectAPIHandlers, apiFunctions ...string) {
  2010  	for _, apiFunction := range apiFunctions {
  2011  		switch apiFunction {
  2012  		case "PostPolicy":
  2013  			// Register PostPolicy handler.
  2014  			bucket.Methods(http.MethodPost).HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
  2015  		case "HeadObject":
  2016  			// Register HeadObject handler.
  2017  			bucket.Methods("Head").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
  2018  		case "GetObject":
  2019  			// Register GetObject handler.
  2020  			bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
  2021  		case "PutObject":
  2022  			// Register PutObject handler.
  2023  			bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
  2024  		case "DeleteObject":
  2025  			// Register Delete Object handler.
  2026  			bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
  2027  		case "CopyObject":
  2028  			// Register Copy Object  handler.
  2029  			bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
  2030  		case "PutBucketPolicy":
  2031  			// Register PutBucket Policy handler.
  2032  			bucket.Methods(http.MethodPut).HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
  2033  		case "DeleteBucketPolicy":
  2034  			// Register Delete bucket HTTP policy handler.
  2035  			bucket.Methods(http.MethodDelete).HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
  2036  		case "GetBucketPolicy":
  2037  			// Register Get Bucket policy HTTP Handler.
  2038  			bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
  2039  		case "GetBucketLifecycle":
  2040  			bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketLifecycleHandler).Queries("lifecycle", "")
  2041  		case "PutBucketLifecycle":
  2042  			bucket.Methods(http.MethodPut).HandlerFunc(api.PutBucketLifecycleHandler).Queries("lifecycle", "")
  2043  		case "DeleteBucketLifecycle":
  2044  			bucket.Methods(http.MethodDelete).HandlerFunc(api.DeleteBucketLifecycleHandler).Queries("lifecycle", "")
  2045  		case "GetBucketLocation":
  2046  			// Register GetBucketLocation handler.
  2047  			bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
  2048  		case "HeadBucket":
  2049  			// Register HeadBucket handler.
  2050  			bucket.Methods(http.MethodHead).HandlerFunc(api.HeadBucketHandler)
  2051  		case "DeleteMultipleObjects":
  2052  			// Register DeleteMultipleObjects handler.
  2053  			bucket.Methods(http.MethodPost).HandlerFunc(api.DeleteMultipleObjectsHandler).Queries("delete", "")
  2054  		case "NewMultipart":
  2055  			// Register New Multipart upload handler.
  2056  			bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
  2057  		case "CopyObjectPart":
  2058  			// Register CopyObjectPart handler.
  2059  			bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
  2060  		case "PutObjectPart":
  2061  			// Register PutObjectPart handler.
  2062  			bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
  2063  		case "ListObjectParts":
  2064  			// Register ListObjectParts handler.
  2065  			bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
  2066  		case "ListMultipartUploads":
  2067  			// Register ListMultipartUploads handler.
  2068  			bucket.Methods(http.MethodGet).HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
  2069  		case "CompleteMultipart":
  2070  			// Register Complete Multipart Upload handler.
  2071  			bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
  2072  		case "AbortMultipart":
  2073  			// Register AbortMultipart Handler.
  2074  			bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
  2075  		case "GetBucketNotification":
  2076  			// Register GetBucketNotification Handler.
  2077  			bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
  2078  		case "PutBucketNotification":
  2079  			// Register PutBucketNotification Handler.
  2080  			bucket.Methods(http.MethodPut).HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
  2081  		case "ListenNotification":
  2082  			// Register ListenNotification Handler.
  2083  			bucket.Methods(http.MethodGet).HandlerFunc(api.ListenNotificationHandler).Queries("events", "{events:.*}")
  2084  		}
  2085  	}
  2086  }
  2087  
  2088  // registerAPIFunctions helper function to add API functions identified by name to the routers.
  2089  func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFunctions ...string) {
  2090  	if len(apiFunctions) == 0 {
  2091  		// Register all api endpoints by default.
  2092  		registerAPIRouter(muxRouter)
  2093  		return
  2094  	}
  2095  	// API Router.
  2096  	apiRouter := muxRouter.PathPrefix(SlashSeparator).Subrouter()
  2097  	// Bucket router.
  2098  	bucketRouter := apiRouter.PathPrefix("/{bucket}").Subrouter()
  2099  
  2100  	// All object storage operations are registered as HTTP handlers on `ObjectAPIHandlers`.
  2101  	// When the handlers get a HTTP request they use the underlying ObjectLayer to perform operations.
  2102  	globalObjLayerMutex.Lock()
  2103  	globalObjectAPI = objLayer
  2104  	globalObjLayerMutex.Unlock()
  2105  
  2106  	// When cache is enabled, Put and Get operations are passed
  2107  	// to underlying cache layer to manage object layer operation and disk caching
  2108  	// operation
  2109  	api := ObjectAPIHandlers{
  2110  		ObjectAPI: func() ObjectLayer {
  2111  			return globalObjectAPI
  2112  		},
  2113  		CacheAPI: func() CacheObjectLayer {
  2114  			return globalCacheObjectAPI
  2115  		},
  2116  	}
  2117  
  2118  	// Register ListBuckets	handler.
  2119  	apiRouter.Methods(http.MethodGet).HandlerFunc(api.ListBucketsHandler)
  2120  	// Register all bucket level handlers.
  2121  	registerBucketLevelFunc(bucketRouter, api, apiFunctions...)
  2122  }
  2123  
  2124  // Takes in Erasure object layer, and the list of API end points to be tested/required, registers the API end points and returns the HTTP handler.
  2125  // Need isolated registration of API end points while writing unit tests for end points.
  2126  // All the API end points are registered only for the default case.
  2127  func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Handler {
  2128  	// initialize a new mux router.
  2129  	// goriilla/mux is the library used to register all the routes and handle them.
  2130  	muxRouter := mux.NewRouter().SkipClean(true)
  2131  	if len(apiFunctions) > 0 {
  2132  		// Iterate the list of API functions requested for and register them in mux HTTP handler.
  2133  		registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
  2134  		return muxRouter
  2135  	}
  2136  	registerAPIRouter(muxRouter)
  2137  	return muxRouter
  2138  }
  2139  
  2140  // Initialize Web RPC Handlers for testing
  2141  func initTestWebRPCEndPoint(objLayer ObjectLayer) http.Handler {
  2142  	globalObjLayerMutex.Lock()
  2143  	globalObjectAPI = objLayer
  2144  	globalObjLayerMutex.Unlock()
  2145  
  2146  	// Initialize router.
  2147  	muxRouter := mux.NewRouter().SkipClean(true)
  2148  	registerWebRouter(muxRouter)
  2149  	return muxRouter
  2150  }
  2151  
  2152  // generateTLSCertKey creates valid key/cert with registered DNS or IP address
  2153  // depending on the passed parameter. That way, we can use tls config without
  2154  // passing InsecureSkipVerify flag.  This code is a simplified version of
  2155  // https://golang.org/src/crypto/tls/generate_cert.go
  2156  func generateTLSCertKey(host string) ([]byte, []byte, error) {
  2157  	validFor := 365 * 24 * time.Hour
  2158  	rsaBits := 2048
  2159  
  2160  	if len(host) == 0 {
  2161  		return nil, nil, fmt.Errorf("Missing host parameter")
  2162  	}
  2163  
  2164  	publicKey := func(priv interface{}) interface{} {
  2165  		switch k := priv.(type) {
  2166  		case *rsa.PrivateKey:
  2167  			return &k.PublicKey
  2168  		case *ecdsa.PrivateKey:
  2169  			return &k.PublicKey
  2170  		default:
  2171  			return nil
  2172  		}
  2173  	}
  2174  
  2175  	pemBlockForKey := func(priv interface{}) *pem.Block {
  2176  		switch k := priv.(type) {
  2177  		case *rsa.PrivateKey:
  2178  			return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
  2179  		case *ecdsa.PrivateKey:
  2180  			b, err := x509.MarshalECPrivateKey(k)
  2181  			if err != nil {
  2182  				fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err)
  2183  				os.Exit(2)
  2184  			}
  2185  			return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
  2186  		default:
  2187  			return nil
  2188  		}
  2189  	}
  2190  
  2191  	var priv interface{}
  2192  	var err error
  2193  	priv, err = rsa.GenerateKey(crand.Reader, rsaBits)
  2194  	if err != nil {
  2195  		return nil, nil, fmt.Errorf("failed to generate private key: %w", err)
  2196  	}
  2197  
  2198  	notBefore := time.Now()
  2199  	notAfter := notBefore.Add(validFor)
  2200  
  2201  	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
  2202  	serialNumber, err := crand.Int(crand.Reader, serialNumberLimit)
  2203  	if err != nil {
  2204  		return nil, nil, fmt.Errorf("failed to generate serial number: %w", err)
  2205  	}
  2206  
  2207  	template := x509.Certificate{
  2208  		SerialNumber: serialNumber,
  2209  		Subject: pkix.Name{
  2210  			Organization: []string{"Acme Co"},
  2211  		},
  2212  		NotBefore: notBefore,
  2213  		NotAfter:  notAfter,
  2214  
  2215  		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
  2216  		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
  2217  		BasicConstraintsValid: true,
  2218  	}
  2219  
  2220  	hosts := strings.Split(host, ",")
  2221  	for _, h := range hosts {
  2222  		if ip := net.ParseIP(h); ip != nil {
  2223  			template.IPAddresses = append(template.IPAddresses, ip)
  2224  		} else {
  2225  			template.DNSNames = append(template.DNSNames, h)
  2226  		}
  2227  	}
  2228  
  2229  	template.IsCA = true
  2230  	template.KeyUsage |= x509.KeyUsageCertSign
  2231  
  2232  	derBytes, err := x509.CreateCertificate(crand.Reader, &template, &template, publicKey(priv), priv)
  2233  	if err != nil {
  2234  		return nil, nil, fmt.Errorf("Failed to create certificate: %w", err)
  2235  	}
  2236  
  2237  	certOut := bytes.NewBuffer([]byte{})
  2238  	pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
  2239  
  2240  	keyOut := bytes.NewBuffer([]byte{})
  2241  	pem.Encode(keyOut, pemBlockForKey(priv))
  2242  
  2243  	return certOut.Bytes(), keyOut.Bytes(), nil
  2244  }
  2245  
  2246  func mustGetPoolEndpoints(args ...string) EndpointServerPools {
  2247  	endpoints := mustGetNewEndpoints(args...)
  2248  	drivesPerSet := len(args)
  2249  	setCount := 1
  2250  	if len(args) >= 16 {
  2251  		drivesPerSet = 16
  2252  		setCount = len(args) / 16
  2253  	}
  2254  	return []PoolEndpoints{{
  2255  		SetCount:     setCount,
  2256  		DrivesPerSet: drivesPerSet,
  2257  		Endpoints:    endpoints,
  2258  	}}
  2259  }
  2260  
  2261  func mustGetNewEndpoints(args ...string) (endpoints Endpoints) {
  2262  	endpoints, err := NewEndpoints(args...)
  2263  	logger.FatalIf(err, "unable to create new endpoint list")
  2264  	return endpoints
  2265  }
  2266  
  2267  func getEndpointsLocalAddr(endpointServerPools EndpointServerPools) string {
  2268  	for _, endpoints := range endpointServerPools {
  2269  		for _, endpoint := range endpoints.Endpoints {
  2270  			if endpoint.IsLocal && endpoint.Type() == URLEndpointType {
  2271  				return endpoint.Host
  2272  			}
  2273  		}
  2274  	}
  2275  
  2276  	return net.JoinHostPort(globalMinioHost, globalMinioPort)
  2277  }
  2278  
  2279  // fetches a random number between range min-max.
  2280  func getRandomRange(min, max int, seed int64) int {
  2281  	// special value -1 means no explicit seeding.
  2282  	if seed != -1 {
  2283  		rand.Seed(seed)
  2284  	}
  2285  	return rand.Intn(max-min) + min
  2286  }
  2287  
  2288  // Randomizes the order of bytes in the byte array
  2289  // using Knuth Fisher-Yates shuffle algorithm.
  2290  func randomizeBytes(s []byte, seed int64) []byte {
  2291  	// special value -1 means no explicit seeding.
  2292  	if seed != -1 {
  2293  		rand.Seed(seed)
  2294  	}
  2295  	n := len(s)
  2296  	var j int
  2297  	for i := 0; i < n-1; i++ {
  2298  		j = i + rand.Intn(n-i)
  2299  		s[i], s[j] = s[j], s[i]
  2300  	}
  2301  	return s
  2302  }
  2303  
  2304  func TestToErrIsNil(t *testing.T) {
  2305  	if toObjectErr(nil) != nil {
  2306  		t.Errorf("Test expected to return nil, failed instead got a non-nil value %s", toObjectErr(nil))
  2307  	}
  2308  	if toStorageErr(nil) != nil {
  2309  		t.Errorf("Test expected to return nil, failed instead got a non-nil value %s", toStorageErr(nil))
  2310  	}
  2311  	ctx := context.Background()
  2312  	if ToAPIError(ctx, nil) != noError {
  2313  		t.Errorf("Test expected error code to be ErrNone, failed instead provided %s", ToAPIError(ctx, nil).Code)
  2314  	}
  2315  }
  2316  
  2317  // Uploads an object using DummyDataGen directly via the http
  2318  // handler. Each part in a multipart object is a new DummyDataGen
  2319  // instance (so the part sizes are needed to reconstruct the whole
  2320  // object). When `len(partSizes) == 1`, asMultipart is used to upload
  2321  // the object as multipart with 1 part or as a regular single object.
  2322  //
  2323  // All upload failures are considered test errors - this function is
  2324  // intended as a helper for other tests.
  2325  func uploadTestObject(t *testing.T, apiRouter http.Handler, creds auth.Credentials, bucketName, objectName string,
  2326  	partSizes []int64, metadata map[string]string, asMultipart bool) {
  2327  
  2328  	if len(partSizes) == 0 {
  2329  		t.Fatalf("Cannot upload an object without part sizes")
  2330  	}
  2331  	if len(partSizes) > 1 {
  2332  		asMultipart = true
  2333  	}
  2334  
  2335  	checkRespErr := func(rec *httptest.ResponseRecorder, exp int) {
  2336  		if rec.Code != exp {
  2337  			b, err := ioutil.ReadAll(rec.Body)
  2338  			t.Fatalf("Expected: %v, Got: %v, Body: %s, err: %v", exp, rec.Code, string(b), err)
  2339  		}
  2340  	}
  2341  
  2342  	if !asMultipart {
  2343  		srcData := NewDummyDataGen(partSizes[0], 0)
  2344  		req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, objectName),
  2345  			partSizes[0], srcData, creds.AccessKey, creds.SecretKey, metadata)
  2346  		if err != nil {
  2347  			t.Fatalf("Unexpected err: %#v", err)
  2348  		}
  2349  		rec := httptest.NewRecorder()
  2350  		apiRouter.ServeHTTP(rec, req)
  2351  		checkRespErr(rec, http.StatusOK)
  2352  	} else {
  2353  		// Multipart upload - each part is a new DummyDataGen
  2354  		// (so the part lengths are required to verify the
  2355  		// object when reading).
  2356  
  2357  		// Initiate mp upload
  2358  		reqI, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName),
  2359  			0, nil, creds.AccessKey, creds.SecretKey, metadata)
  2360  		if err != nil {
  2361  			t.Fatalf("Unexpected err: %#v", err)
  2362  		}
  2363  		rec := httptest.NewRecorder()
  2364  		apiRouter.ServeHTTP(rec, reqI)
  2365  		checkRespErr(rec, http.StatusOK)
  2366  		decoder := xml.NewDecoder(rec.Body)
  2367  		multipartResponse := &InitiateMultipartUploadResponse{}
  2368  		err = decoder.Decode(multipartResponse)
  2369  		if err != nil {
  2370  			t.Fatalf("Error decoding the recorded response Body")
  2371  		}
  2372  		upID := multipartResponse.UploadID
  2373  
  2374  		// Upload each part
  2375  		var cp []CompletePart
  2376  		cumulativeSum := int64(0)
  2377  		for i, partLen := range partSizes {
  2378  			partID := i + 1
  2379  			partSrc := NewDummyDataGen(partLen, cumulativeSum)
  2380  			cumulativeSum += partLen
  2381  			req, errP := newTestSignedRequestV4(http.MethodPut,
  2382  				getPutObjectPartURL("", bucketName, objectName, upID, fmt.Sprintf("%d", partID)),
  2383  				partLen, partSrc, creds.AccessKey, creds.SecretKey, metadata)
  2384  			if errP != nil {
  2385  				t.Fatalf("Unexpected err: %#v", errP)
  2386  			}
  2387  			rec = httptest.NewRecorder()
  2388  			apiRouter.ServeHTTP(rec, req)
  2389  			checkRespErr(rec, http.StatusOK)
  2390  			header := rec.Header()
  2391  			if v, ok := header["ETag"]; ok {
  2392  				etag := v[0]
  2393  				if etag == "" {
  2394  					t.Fatalf("Unexpected empty etag")
  2395  				}
  2396  				cp = append(cp, CompletePart{partID, etag[1 : len(etag)-1]})
  2397  			} else {
  2398  				t.Fatalf("Missing etag header")
  2399  			}
  2400  		}
  2401  
  2402  		// Call CompleteMultipart API
  2403  		compMpBody, err := xml.Marshal(CompleteMultipartUpload{Parts: cp})
  2404  		if err != nil {
  2405  			t.Fatalf("Unexpected err: %#v", err)
  2406  		}
  2407  		reqC, errP := newTestSignedRequestV4(http.MethodPost,
  2408  			getCompleteMultipartUploadURL("", bucketName, objectName, upID),
  2409  			int64(len(compMpBody)), bytes.NewReader(compMpBody),
  2410  			creds.AccessKey, creds.SecretKey, metadata)
  2411  		if errP != nil {
  2412  			t.Fatalf("Unexpected err: %#v", errP)
  2413  		}
  2414  		rec = httptest.NewRecorder()
  2415  		apiRouter.ServeHTTP(rec, reqC)
  2416  		checkRespErr(rec, http.StatusOK)
  2417  	}
  2418  }