storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/handler-api.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"net/http"
    21  	"sync"
    22  	"time"
    23  
    24  	"storj.io/minio/cmd/config/api"
    25  	"storj.io/minio/cmd/logger"
    26  	"storj.io/minio/pkg/sys"
    27  )
    28  
    29  type apiConfig struct {
    30  	mu sync.RWMutex
    31  
    32  	requestsDeadline time.Duration
    33  	requestsPool     chan struct{}
    34  	clusterDeadline  time.Duration
    35  	listQuorum       int
    36  	extendListLife   time.Duration
    37  	corsAllowOrigins []string
    38  	// total drives per erasure set across pools.
    39  	totalDriveCount    int
    40  	replicationWorkers int
    41  }
    42  
    43  func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
    44  	t.mu.Lock()
    45  	defer t.mu.Unlock()
    46  
    47  	t.clusterDeadline = cfg.ClusterDeadline
    48  	t.corsAllowOrigins = cfg.CorsAllowOrigin
    49  	for _, setDriveCount := range setDriveCounts {
    50  		t.totalDriveCount += setDriveCount
    51  	}
    52  
    53  	var apiRequestsMaxPerNode int
    54  	if cfg.RequestsMax <= 0 {
    55  		stats, err := sys.GetStats()
    56  		if err != nil {
    57  			logger.LogIf(GlobalContext, err)
    58  			// Default to 8 GiB, not critical.
    59  			stats.TotalRAM = 8 << 30
    60  		}
    61  		// max requests per node is calculated as
    62  		// total_ram / ram_per_request
    63  		// ram_per_request is (2MiB+128KiB) * driveCount \
    64  		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
    65  		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(t.totalDriveCount*(blockSizeLarge+blockSizeSmall)+int(blockSizeV1*2+blockSizeV2*2)))
    66  	} else {
    67  		apiRequestsMaxPerNode = cfg.RequestsMax
    68  		if len(globalEndpoints.Hostnames()) > 0 {
    69  			apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames())
    70  		}
    71  	}
    72  	if cap(t.requestsPool) < apiRequestsMaxPerNode {
    73  		// Only replace if needed.
    74  		// Existing requests will use the previous limit,
    75  		// but new requests will use the new limit.
    76  		// There will be a short overlap window,
    77  		// but this shouldn't last long.
    78  		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
    79  	}
    80  	t.requestsDeadline = cfg.RequestsDeadline
    81  	t.listQuorum = cfg.GetListQuorum()
    82  	t.extendListLife = cfg.ExtendListLife
    83  	if globalReplicationPool != nil &&
    84  		cfg.ReplicationWorkers != t.replicationWorkers {
    85  		globalReplicationPool.Resize(cfg.ReplicationWorkers)
    86  	}
    87  	t.replicationWorkers = cfg.ReplicationWorkers
    88  }
    89  
    90  func (t *apiConfig) getListQuorum() int {
    91  	t.mu.RLock()
    92  	defer t.mu.RUnlock()
    93  
    94  	return t.listQuorum
    95  }
    96  
    97  func (t *apiConfig) getExtendListLife() time.Duration {
    98  	t.mu.RLock()
    99  	defer t.mu.RUnlock()
   100  
   101  	return t.extendListLife
   102  }
   103  
   104  func (t *apiConfig) getCorsAllowOrigins() []string {
   105  	t.mu.RLock()
   106  	defer t.mu.RUnlock()
   107  
   108  	corsAllowOrigins := make([]string, len(t.corsAllowOrigins))
   109  	copy(corsAllowOrigins, t.corsAllowOrigins)
   110  	return corsAllowOrigins
   111  }
   112  
   113  func (t *apiConfig) getClusterDeadline() time.Duration {
   114  	t.mu.RLock()
   115  	defer t.mu.RUnlock()
   116  
   117  	if t.clusterDeadline == 0 {
   118  		return 10 * time.Second
   119  	}
   120  
   121  	return t.clusterDeadline
   122  }
   123  
   124  func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
   125  	t.mu.RLock()
   126  	defer t.mu.RUnlock()
   127  
   128  	if t.requestsPool == nil {
   129  		return nil, time.Duration(0)
   130  	}
   131  
   132  	return t.requestsPool, t.requestsDeadline
   133  }
   134  
   135  // MaxClients throttles the S3 API calls
   136  func MaxClients(f http.HandlerFunc) http.HandlerFunc {
   137  	return func(w http.ResponseWriter, r *http.Request) {
   138  		pool, deadline := globalAPIConfig.getRequestsPool()
   139  		if pool == nil {
   140  			f.ServeHTTP(w, r)
   141  			return
   142  		}
   143  
   144  		globalHTTPStats.addRequestsInQueue(1)
   145  
   146  		deadlineTimer := time.NewTimer(deadline)
   147  		defer deadlineTimer.Stop()
   148  
   149  		select {
   150  		case pool <- struct{}{}:
   151  			defer func() { <-pool }()
   152  			globalHTTPStats.addRequestsInQueue(-1)
   153  			f.ServeHTTP(w, r)
   154  		case <-deadlineTimer.C:
   155  			// Send a http timeout message
   156  			WriteErrorResponse(r.Context(), w,
   157  				errorCodes.ToAPIErr(ErrOperationMaxedOut),
   158  				r.URL, guessIsBrowserReq(r))
   159  			globalHTTPStats.addRequestsInQueue(-1)
   160  			return
   161  		case <-r.Context().Done():
   162  			globalHTTPStats.addRequestsInQueue(-1)
   163  			return
   164  		}
   165  	}
   166  }
   167  
   168  func (t *apiConfig) getReplicationWorkers() int {
   169  	t.mu.RLock()
   170  	defer t.mu.RUnlock()
   171  
   172  	return t.replicationWorkers
   173  }