storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/bucket-listobjects-handlers.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "net/http" 22 "strconv" 23 "strings" 24 25 "github.com/gorilla/mux" 26 27 "storj.io/minio/cmd/logger" 28 "storj.io/minio/pkg/bucket/policy" 29 "storj.io/minio/pkg/sync/errgroup" 30 ) 31 32 func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) { 33 g := errgroup.WithNErrs(len(objects)).WithConcurrency(500) 34 _, cancel := g.WithCancelOnError(ctx) 35 defer cancel() 36 for index := range objects { 37 index := index 38 g.Go(func() error { 39 size, err := objects[index].GetActualSize() 40 if err == nil { 41 objects[index].Size = size 42 } 43 objects[index].ETag = objects[index].GetActualETag(nil) 44 return nil 45 }, index) 46 } 47 g.WaitErr() 48 } 49 50 // Validate all the ListObjects query arguments, returns an APIErrorCode 51 // if one of the args do not meet the required conditions. 52 // Special conditions required by MinIO server are as below 53 // - delimiter if set should be equal to '/', otherwise the request is rejected. 54 // - marker if set should have a common prefix with 'prefix' param, otherwise 55 // the request is rejected. 56 func validateListObjectsArgs(marker, delimiter, encodingType string, maxKeys int) APIErrorCode { 57 // Max keys cannot be negative. 58 if maxKeys < 0 { 59 return ErrInvalidMaxKeys 60 } 61 62 if encodingType != "" { 63 // Only url encoding type is supported 64 if strings.ToLower(encodingType) != "url" { 65 return ErrInvalidEncodingMethod 66 } 67 } 68 69 return ErrNone 70 } 71 72 // ListObjectVersions - GET Bucket Object versions 73 // You can use the versions subresource to list metadata about all 74 // of the versions of objects in a bucket. 75 func (api ObjectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { 76 ctx := NewContext(r, w, "ListObjectVersions") 77 78 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 79 80 vars := mux.Vars(r) 81 bucket := vars["bucket"] 82 83 objectAPI := api.ObjectAPI() 84 if objectAPI == nil { 85 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) 86 return 87 } 88 89 if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone { 90 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 91 return 92 } 93 94 urlValues := r.URL.Query() 95 96 // Extract all the listBucketVersions query params to their native values. 97 prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode := getListBucketObjectVersionsArgs(urlValues) 98 if errCode != ErrNone { 99 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) 100 return 101 } 102 103 // Validate the query params before beginning to serve the request. 104 if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxkeys); s3Error != ErrNone { 105 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 106 return 107 } 108 109 listObjectVersions := objectAPI.ListObjectVersions 110 111 // Inititate a list object versions operation based on the input params. 112 // On success would return back ListObjectsInfo object to be 113 // marshaled into S3 compatible XML header. 114 listObjectVersionsInfo, err := listObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, delimiter, maxkeys) 115 if err != nil { 116 WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) 117 return 118 } 119 120 concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects) 121 122 response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo) 123 124 // Write success response. 125 WriteSuccessResponseXML(w, EncodeResponse(response)) 126 } 127 128 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata. 129 // -------------------------- 130 // This implementation of the GET operation returns some or all (up to 10000) 131 // of the objects in a bucket. You can use the request parameters as selection 132 // criteria to return a subset of the objects in a bucket. 133 // 134 // NOTE: It is recommended that this API to be used for application development. 135 // MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools. 136 func (api ObjectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) { 137 ctx := NewContext(r, w, "ListObjectsV2M") 138 139 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 140 141 vars := mux.Vars(r) 142 bucket := vars["bucket"] 143 144 objectAPI := api.ObjectAPI() 145 if objectAPI == nil { 146 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) 147 return 148 } 149 150 if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { 151 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 152 return 153 } 154 155 urlValues := r.URL.Query() 156 157 // Extract all the listObjectsV2 query params to their native values. 158 prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues) 159 if errCode != ErrNone { 160 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) 161 return 162 } 163 164 // Validate the query params before beginning to serve the request. 165 // fetch-owner is not validated since it is a boolean 166 if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone { 167 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 168 return 169 } 170 171 listObjectsV2 := objectAPI.ListObjectsV2 172 173 // Inititate a list objects operation based on the input params. 174 // On success would return back ListObjectsInfo object to be 175 // marshaled into S3 compatible XML header. 176 listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter) 177 if err != nil { 178 WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) 179 return 180 } 181 182 concurrentDecryptETag(ctx, listObjectsV2Info.Objects) 183 184 // The next continuation token has id@node_index format to optimize paginated listing 185 nextContinuationToken := listObjectsV2Info.NextContinuationToken 186 187 response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter, 188 delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, 189 maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true) 190 191 // Write success response. 192 WriteSuccessResponseXML(w, EncodeResponse(response)) 193 } 194 195 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2. 196 // -------------------------- 197 // This implementation of the GET operation returns some or all (up to 10000) 198 // of the objects in a bucket. You can use the request parameters as selection 199 // criteria to return a subset of the objects in a bucket. 200 // 201 // NOTE: It is recommended that this API to be used for application development. 202 // MinIO continues to support ListObjectsV1 for supporting legacy tools. 203 func (api ObjectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { 204 ctx := NewContext(r, w, "ListObjectsV2") 205 206 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 207 208 vars := mux.Vars(r) 209 bucket := vars["bucket"] 210 211 objectAPI := api.ObjectAPI() 212 if objectAPI == nil { 213 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) 214 return 215 } 216 217 if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { 218 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 219 return 220 } 221 222 urlValues := r.URL.Query() 223 224 // Extract all the listObjectsV2 query params to their native values. 225 prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues) 226 if errCode != ErrNone { 227 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r)) 228 return 229 } 230 231 // Validate the query params before beginning to serve the request. 232 // fetch-owner is not validated since it is a boolean 233 if s3Error := validateListObjectsArgs(token, delimiter, encodingType, maxKeys); s3Error != ErrNone { 234 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 235 return 236 } 237 238 listObjectsV2 := objectAPI.ListObjectsV2 239 240 // Inititate a list objects operation based on the input params. 241 // On success would return back ListObjectsInfo object to be 242 // marshaled into S3 compatible XML header. 243 listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter) 244 if err != nil { 245 WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) 246 return 247 } 248 249 concurrentDecryptETag(ctx, listObjectsV2Info.Objects) 250 251 response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter, 252 delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, 253 maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false) 254 255 // Write success response. 256 WriteSuccessResponseXML(w, EncodeResponse(response)) 257 } 258 259 func parseRequestToken(token string) (subToken string, nodeIndex int) { 260 if token == "" { 261 return token, -1 262 } 263 i := strings.Index(token, "@") 264 if i < 0 { 265 return token, -1 266 } 267 nodeIndex, err := strconv.Atoi(token[i+1:]) 268 if err != nil { 269 return token, -1 270 } 271 subToken = token[:i] 272 return subToken, nodeIndex 273 } 274 275 func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string) (string, bool) { 276 subToken, nodeIndex := parseRequestToken(token) 277 if nodeIndex > 0 { 278 return subToken, proxyRequestByNodeIndex(ctx, w, r, nodeIndex) 279 } 280 return subToken, false 281 } 282 283 func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int) (success bool) { 284 if len(globalProxyEndpoints) == 0 { 285 return false 286 } 287 if index < 0 || index >= len(globalProxyEndpoints) { 288 return false 289 } 290 ep := globalProxyEndpoints[index] 291 if ep.IsLocal { 292 return false 293 } 294 return proxyRequest(ctx, w, r, ep) 295 } 296 297 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1. 298 // -------------------------- 299 // This implementation of the GET operation returns some or all (up to 10000) 300 // of the objects in a bucket. You can use the request parameters as selection 301 // criteria to return a subset of the objects in a bucket. 302 // 303 func (api ObjectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { 304 ctx := NewContext(r, w, "ListObjectsV1") 305 306 defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) 307 308 vars := mux.Vars(r) 309 bucket := vars["bucket"] 310 311 objectAPI := api.ObjectAPI() 312 if objectAPI == nil { 313 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r)) 314 return 315 } 316 317 if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { 318 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 319 return 320 } 321 322 // Extract all the litsObjectsV1 query params to their native values. 323 prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query()) 324 if s3Error != ErrNone { 325 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 326 return 327 } 328 329 // Validate all the query params before beginning to serve the request. 330 if s3Error := validateListObjectsArgs(marker, delimiter, encodingType, maxKeys); s3Error != ErrNone { 331 WriteErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r)) 332 return 333 } 334 335 listObjects := objectAPI.ListObjects 336 337 // Inititate a list objects operation based on the input params. 338 // On success would return back ListObjectsInfo object to be 339 // marshaled into S3 compatible XML header. 340 listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys) 341 if err != nil { 342 WriteErrorResponse(ctx, w, ToAPIError(ctx, err), r.URL, guessIsBrowserReq(r)) 343 return 344 } 345 346 concurrentDecryptETag(ctx, listObjectsInfo.Objects) 347 348 response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo) 349 350 // Write success response. 351 WriteSuccessResponseXML(w, EncodeResponse(response)) 352 }