github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/cmd/warm-backend-minio.go (about)

     1  // Copyright (c) 2015-2022 MinIO, Inc.
     2  //
     3  // This file is part of MinIO Object Storage stack
     4  //
     5  // This program is free software: you can redistribute it and/or modify
     6  // it under the terms of the GNU Affero General Public License as published by
     7  // the Free Software Foundation, either version 3 of the License, or
     8  // (at your option) any later version.
     9  //
    10  // This program is distributed in the hope that it will be useful
    11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13  // GNU Affero General Public License for more details.
    14  //
    15  // You should have received a copy of the GNU Affero General Public License
    16  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17  
    18  package cmd
    19  
    20  import (
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"math"
    26  	"net/url"
    27  	"strings"
    28  	"time"
    29  
    30  	"github.com/minio/madmin-go/v3"
    31  	minio "github.com/minio/minio-go/v7"
    32  	"github.com/minio/minio-go/v7/pkg/credentials"
    33  )
    34  
    35  type warmBackendMinIO struct {
    36  	warmBackendS3
    37  }
    38  
    39  var _ WarmBackend = (*warmBackendMinIO)(nil)
    40  
    41  const (
    42  	maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5
    43  	maxPartsCount             = 10000
    44  	maxPartSize               = 1024 * 1024 * 1024 * 5
    45  	minPartSize               = 1024 * 1024 * 64 // chosen by us to be optimal for HDDs
    46  )
    47  
    48  // optimalPartInfo - calculate the optimal part info for a given
    49  // object size.
    50  //
    51  // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible
    52  // object storage it will have the following parameters as constants.
    53  //
    54  //	maxPartsCount - 10000
    55  //	maxMultipartPutObjectSize - 5TiB
    56  func optimalPartSize(objectSize int64) (partSize int64, err error) {
    57  	// object size is '-1' set it to 5TiB.
    58  	if objectSize == -1 {
    59  		objectSize = maxMultipartPutObjectSize
    60  	}
    61  
    62  	// object size is larger than supported maximum.
    63  	if objectSize > maxMultipartPutObjectSize {
    64  		err = errors.New("entity too large")
    65  		return
    66  	}
    67  
    68  	configuredPartSize := minPartSize
    69  	// Use floats for part size for all calculations to avoid
    70  	// overflows during float64 to int64 conversions.
    71  	partSizeFlt := float64(objectSize / maxPartsCount)
    72  	partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize)
    73  
    74  	// Part size.
    75  	partSize = int64(partSizeFlt)
    76  	if partSize == 0 {
    77  		return minPartSize, nil
    78  	}
    79  	return partSize, nil
    80  }
    81  
    82  func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) {
    83  	partSize, err := optimalPartSize(length)
    84  	if err != nil {
    85  		return remoteVersionID(""), err
    86  	}
    87  	res, err := m.client.PutObject(ctx, m.Bucket, m.getDest(object), r, length, minio.PutObjectOptions{
    88  		StorageClass:         m.StorageClass,
    89  		PartSize:             uint64(partSize),
    90  		DisableContentSha256: true,
    91  	})
    92  	return remoteVersionID(res.VersionID), m.ToObjectError(err, object)
    93  }
    94  
    95  func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO, error) {
    96  	// Validation of credentials
    97  	if conf.AccessKey == "" || conf.SecretKey == "" {
    98  		return nil, errors.New("both access and secret keys are required")
    99  	}
   100  
   101  	if conf.Bucket == "" {
   102  		return nil, errors.New("no bucket name was provided")
   103  	}
   104  
   105  	u, err := url.Parse(conf.Endpoint)
   106  	if err != nil {
   107  		return nil, err
   108  	}
   109  
   110  	creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "")
   111  
   112  	getRemoteTierTargetInstanceTransportOnce.Do(func() {
   113  		getRemoteTierTargetInstanceTransport = NewHTTPTransportWithTimeout(10 * time.Minute)
   114  	})
   115  	opts := &minio.Options{
   116  		Creds:           creds,
   117  		Secure:          u.Scheme == "https",
   118  		Transport:       getRemoteTierTargetInstanceTransport,
   119  		TrailingHeaders: true,
   120  	}
   121  	client, err := minio.New(u.Host, opts)
   122  	if err != nil {
   123  		return nil, err
   124  	}
   125  	client.SetAppInfo(fmt.Sprintf("minio-tier-%s", tier), ReleaseTag)
   126  
   127  	core := &minio.Core{Client: client}
   128  	return &warmBackendMinIO{
   129  		warmBackendS3{
   130  			client: client,
   131  			core:   core,
   132  			Bucket: conf.Bucket,
   133  			Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator),
   134  		},
   135  	}, nil
   136  }