github.com/minio/mc@v0.0.0-20240503112107-b471de8d1882/cmd/subnet-file-uploader.go (about)

     1  // Copyright (c) 2015-2024 MinIO, Inc.
     2  //
     3  // This file is part of MinIO Object Storage stack
     4  //
     5  // This program is free software: you can redistribute it and/or modify
     6  // it under the terms of the GNU Affero General Public License as published by
     7  // the Free Software Foundation, either version 3 of the License, or
     8  // (at your option) any later version.
     9  //
    10  // This program is distributed in the hope that it will be useful
    11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13  // GNU Affero General Public License for more details.
    14  //
    15  // You should have received a copy of the GNU Affero General Public License
    16  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17  
    18  package cmd
    19  
    20  import (
    21  	"io"
    22  	"mime/multipart"
    23  	"net/http"
    24  	"net/url"
    25  	"os"
    26  	"path/filepath"
    27  	"strings"
    28  
    29  	"github.com/klauspost/compress/zstd"
    30  )
    31  
    32  // SubnetFileUploader - struct to upload files to SUBNET
    33  type SubnetFileUploader struct {
    34  	alias             string        // used for saving api-key and license from response
    35  	filename          string        // filename passed in the SUBNET request
    36  	FilePath          string        // file to upload
    37  	ReqURL            string        // SUBNET upload URL
    38  	Params            url.Values    // query params to be sent in the request
    39  	Headers           SubnetHeaders // headers to be sent in the request
    40  	AutoCompress      bool          // whether to compress (zst) the file before uploading
    41  	DeleteAfterUpload bool          // whether to delete the file after successful upload
    42  }
    43  
    44  // UploadFileToSubnet - uploads the file to SUBNET
    45  func (i *SubnetFileUploader) UploadFileToSubnet() (string, error) {
    46  	req, e := i.subnetUploadReq()
    47  	if e != nil {
    48  		return "", e
    49  	}
    50  
    51  	resp, e := subnetReqDo(req, i.Headers)
    52  	if e != nil {
    53  		return "", e
    54  	}
    55  
    56  	if i.DeleteAfterUpload {
    57  		os.Remove(i.FilePath)
    58  	}
    59  
    60  	// ensure that both api-key and license from
    61  	// SUBNET response are saved in the config
    62  	if len(i.alias) > 0 {
    63  		extractAndSaveSubnetCreds(i.alias, resp)
    64  	}
    65  
    66  	return resp, nil
    67  }
    68  
    69  func (i *SubnetFileUploader) updateParams() {
    70  	if i.Params == nil {
    71  		i.Params = url.Values{}
    72  	}
    73  
    74  	if i.filename == "" {
    75  		i.filename = filepath.Base(i.FilePath)
    76  	}
    77  
    78  	i.AutoCompress = i.AutoCompress && !strings.HasSuffix(strings.ToLower(i.FilePath), ".zst")
    79  	if i.AutoCompress {
    80  		i.filename += ".zst"
    81  		i.Params.Add("auto-compression", "zstd")
    82  	}
    83  
    84  	i.Params.Add("filename", i.filename)
    85  	i.ReqURL += "?" + i.Params.Encode()
    86  }
    87  
    88  func (i *SubnetFileUploader) subnetUploadReq() (*http.Request, error) {
    89  	i.updateParams()
    90  
    91  	r, w := io.Pipe()
    92  	mwriter := multipart.NewWriter(w)
    93  	contentType := mwriter.FormDataContentType()
    94  
    95  	go func() {
    96  		var (
    97  			part io.Writer
    98  			e    error
    99  		)
   100  		defer func() {
   101  			mwriter.Close()
   102  			w.CloseWithError(e)
   103  		}()
   104  
   105  		part, e = mwriter.CreateFormFile("file", i.filename)
   106  		if e != nil {
   107  			return
   108  		}
   109  
   110  		file, e := os.Open(i.FilePath)
   111  		if e != nil {
   112  			return
   113  		}
   114  		defer file.Close()
   115  
   116  		if i.AutoCompress {
   117  			z, _ := zstd.NewWriter(part, zstd.WithEncoderConcurrency(2))
   118  			defer z.Close()
   119  			_, e = z.ReadFrom(file)
   120  		} else {
   121  			_, e = io.Copy(part, file)
   122  		}
   123  	}()
   124  
   125  	req, e := http.NewRequest(http.MethodPost, i.ReqURL, r)
   126  	if e != nil {
   127  		return nil, e
   128  	}
   129  	req.Header.Add("Content-Type", contentType)
   130  
   131  	return req, nil
   132  }