storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/bucket-listobjects-handlers.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"net/http"
    22  	"strconv"
    23  	"strings"
    24  
    25  	"github.com/gorilla/mux"
    26  
    27  	"storj.io/minio/cmd/logger"
    28  	"storj.io/minio/pkg/bucket/policy"
    29  	"storj.io/minio/pkg/sync/errgroup"
    30  )
    31  
    32  func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
    33  	g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
    34  	_, cancel := g.WithCancelOnError(ctx)
    35  	defer cancel()
    36  	for index := range objects {
    37  		index := index
    38  		g.Go(func() error {
    39  			size, err := objects[index].GetActualSize()
    40  			if err == nil {
    41  				objects[index].Size = size
    42  			}
    43  			objects[index].ETag = objects[index].GetActualETag(nil)
    44  			return nil
    45  		}, index)
    46  	}
    47  	g.WaitErr()
    48  }
    49  
    50  // Validate all the ListObjects query arguments, returns an APIErrorCode
    51  // if one of the args do not meet the required conditions.
    52  // Special conditions required by MinIO server are as below
    53  // - delimiter if set should be equal to '/', otherwise the request is rejected.
    54  // - marker if set should have a common prefix with 'prefix' param, otherwise
    55  //   the request is rejected.
    56  func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
    57  	// Max keys cannot be negative.
    58  	if maxKeys < 0 {
    59  		return ErrInvalidMaxKeys
    60  	}
    61  
    62  	if encodingType != "" {
    63  		// Only url encoding type is supported
    64  		if strings.ToLower(encodingType) != "url" {
    65  			return ErrInvalidEncodingMethod
    66  		}
    67  	}
    68  
    69  	return ErrNone
    70  }
    71  
    72  // ListObjectVersions - GET Bucket Object versions
    73  // You can use the versions subresource to list metadata about all
    74  // of the versions of objects in a bucket.
    75  func (api ObjectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
    76  	ctx := NewContext(r, w, "ListObjectVersions")
    77  
    78  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
    79  
    80  	vars := mux.Vars(r)
    81  	bucket := vars["bucket"]
    82  
    83  	objectAPI := api.ObjectAPI()
    84  	if objectAPI == nil {
    85  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
    86  		return
    87  	}
    88  
    89  	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone {
    90  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
    91  		return
    92  	}
    93  
    94  	urlValues := r.URL.Query()
    95  
    96  	// Extract all the listBucketVersions query params to their native values.
    97  	prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode := getListBucketObjectVersionsArgs(urlValues)
    98  	if errCode != ErrNone {
    99  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
   100  		return
   101  	}
   102  
   103  	// Validate the query params before beginning to serve the request.
   104  	if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxkeys); s3Error != ErrNone {
   105  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   106  		return
   107  	}
   108  
   109  	listObjectVersions := objectAPI.ListObjectVersions
   110  
   111  	// Inititate a list object versions operation based on the input params.
   112  	// On success would return back ListObjectsInfo object to be
   113  	// marshaled into S3 compatible XML header.
   114  	listObjectVersionsInfo, err := listObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, delimiter, maxkeys)
   115  	if err != nil {
   116  		WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
   117  		return
   118  	}
   119  
   120  	concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects)
   121  
   122  	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
   123  
   124  	// Write success response.
   125  	WriteSuccessResponseXML(w, EncodeResponse(response))
   126  }
   127  
   128  // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
   129  // --------------------------
   130  // This implementation of the GET operation returns some or all (up to 10000)
   131  // of the objects in a bucket. You can use the request parameters as selection
   132  // criteria to return a subset of the objects in a bucket.
   133  //
   134  // NOTE: It is recommended that this API to be used for application development.
   135  // MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools.
   136  func (api ObjectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
   137  	ctx := NewContext(r, w, "ListObjectsV2M")
   138  
   139  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   140  
   141  	vars := mux.Vars(r)
   142  	bucket := vars["bucket"]
   143  
   144  	objectAPI := api.ObjectAPI()
   145  	if objectAPI == nil {
   146  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
   147  		return
   148  	}
   149  
   150  	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
   151  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   152  		return
   153  	}
   154  
   155  	urlValues := r.URL.Query()
   156  
   157  	// Extract all the listObjectsV2 query params to their native values.
   158  	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
   159  	if errCode != ErrNone {
   160  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
   161  		return
   162  	}
   163  
   164  	// Validate the query params before beginning to serve the request.
   165  	// fetch-owner is not validated since it is a boolean
   166  	if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
   167  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   168  		return
   169  	}
   170  
   171  	listObjectsV2 := objectAPI.ListObjectsV2
   172  
   173  	// Inititate a list objects operation based on the input params.
   174  	// On success would return back ListObjectsInfo object to be
   175  	// marshaled into S3 compatible XML header.
   176  	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
   177  	if err != nil {
   178  		WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
   179  		return
   180  	}
   181  
   182  	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
   183  
   184  	// The next continuation token has id@node_index format to optimize paginated listing
   185  	nextContinuationToken := listObjectsV2Info.NextContinuationToken
   186  
   187  	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
   188  		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
   189  		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
   190  
   191  	// Write success response.
   192  	WriteSuccessResponseXML(w, EncodeResponse(response))
   193  }
   194  
   195  // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
   196  // --------------------------
   197  // This implementation of the GET operation returns some or all (up to 10000)
   198  // of the objects in a bucket. You can use the request parameters as selection
   199  // criteria to return a subset of the objects in a bucket.
   200  //
   201  // NOTE: It is recommended that this API to be used for application development.
   202  // MinIO continues to support ListObjectsV1 for supporting legacy tools.
   203  func (api ObjectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
   204  	ctx := NewContext(r, w, "ListObjectsV2")
   205  
   206  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   207  
   208  	vars := mux.Vars(r)
   209  	bucket := vars["bucket"]
   210  
   211  	objectAPI := api.ObjectAPI()
   212  	if objectAPI == nil {
   213  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
   214  		return
   215  	}
   216  
   217  	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
   218  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   219  		return
   220  	}
   221  
   222  	urlValues := r.URL.Query()
   223  
   224  	// Extract all the listObjectsV2 query params to their native values.
   225  	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
   226  	if errCode != ErrNone {
   227  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
   228  		return
   229  	}
   230  
   231  	// Validate the query params before beginning to serve the request.
   232  	// fetch-owner is not validated since it is a boolean
   233  	if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
   234  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   235  		return
   236  	}
   237  
   238  	listObjectsV2 := objectAPI.ListObjectsV2
   239  
   240  	// Inititate a list objects operation based on the input params.
   241  	// On success would return back ListObjectsInfo object to be
   242  	// marshaled into S3 compatible XML header.
   243  	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
   244  	if err != nil {
   245  		WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
   246  		return
   247  	}
   248  
   249  	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
   250  
   251  	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
   252  		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
   253  		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
   254  
   255  	// Write success response.
   256  	WriteSuccessResponseXML(w, EncodeResponse(response))
   257  }
   258  
   259  func parseRequestToken(token string) (subToken string, nodeIndex int) {
   260  	if token == "" {
   261  		return token, -1
   262  	}
   263  	i := strings.Index(token, "@")
   264  	if i < 0 {
   265  		return token, -1
   266  	}
   267  	nodeIndex, err := strconv.Atoi(token[i+1:])
   268  	if err != nil {
   269  		return token, -1
   270  	}
   271  	subToken = token[:i]
   272  	return subToken, nodeIndex
   273  }
   274  
   275  func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) {
   276  	subToken, nodeIndex := parseRequestToken(token)
   277  	if nodeIndex > 0 {
   278  		return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex)
   279  	}
   280  	return subToken, false
   281  }
   282  
   283  func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int) (success bool) {
   284  	if len(globalProxyEndpoints) == 0 {
   285  		return false
   286  	}
   287  	if index < 0 || index >= len(globalProxyEndpoints) {
   288  		return false
   289  	}
   290  	ep := globalProxyEndpoints[index]
   291  	if ep.IsLocal {
   292  		return false
   293  	}
   294  	return proxyRequest(ctx, w, r, ep)
   295  }
   296  
   297  // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
   298  // --------------------------
   299  // This implementation of the GET operation returns some or all (up to 10000)
   300  // of the objects in a bucket. You can use the request parameters as selection
   301  // criteria to return a subset of the objects in a bucket.
   302  //
   303  func (api ObjectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
   304  	ctx := NewContext(r, w, "ListObjectsV1")
   305  
   306  	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
   307  
   308  	vars := mux.Vars(r)
   309  	bucket := vars["bucket"]
   310  
   311  	objectAPI := api.ObjectAPI()
   312  	if objectAPI == nil {
   313  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
   314  		return
   315  	}
   316  
   317  	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
   318  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   319  		return
   320  	}
   321  
   322  	// Extract all the litsObjectsV1 query params to their native values.
   323  	prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query())
   324  	if s3Error != ErrNone {
   325  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   326  		return
   327  	}
   328  
   329  	// Validate all the query params before beginning to serve the request.
   330  	if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
   331  		WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
   332  		return
   333  	}
   334  
   335  	listObjects := objectAPI.ListObjects
   336  
   337  	// Inititate a list objects operation based on the input params.
   338  	// On success would return back ListObjectsInfo object to be
   339  	// marshaled into S3 compatible XML header.
   340  	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
   341  	if err != nil {
   342  		WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
   343  		return
   344  	}
   345  
   346  	concurrentDecryptETag(ctx, listObjectsInfo.Objects)
   347  
   348  	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
   349  
   350  	// Write success response.
   351  	WriteSuccessResponseXML(w, EncodeResponse(response))
   352  }