storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/handler-api.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "net/http" 21 "sync" 22 "time" 23 24 "storj.io/minio/cmd/config/api" 25 "storj.io/minio/cmd/logger" 26 "storj.io/minio/pkg/sys" 27 ) 28 29 type apiConfig struct { 30 mu sync.RWMutex 31 32 requestsDeadline time.Duration 33 requestsPool chan struct{} 34 clusterDeadline time.Duration 35 listQuorum int 36 extendListLife time.Duration 37 corsAllowOrigins []string 38 // total drives per erasure set across pools. 39 totalDriveCount int 40 replicationWorkers int 41 } 42 43 func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) { 44 t.mu.Lock() 45 defer t.mu.Unlock() 46 47 t.clusterDeadline = cfg.ClusterDeadline 48 t.corsAllowOrigins = cfg.CorsAllowOrigin 49 for _, setDriveCount := range setDriveCounts { 50 t.totalDriveCount += setDriveCount 51 } 52 53 var apiRequestsMaxPerNode int 54 if cfg.RequestsMax <= 0 { 55 stats, err := sys.GetStats() 56 if err != nil { 57 logger.LogIf(GlobalContext, err) 58 // Default to 8 GiB, not critical. 59 stats.TotalRAM = 8 << 30 60 } 61 // max requests per node is calculated as 62 // total_ram / ram_per_request 63 // ram_per_request is (2MiB+128KiB) * driveCount \ 64 // + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2) 65 apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(t.totalDriveCount*(blockSizeLarge+blockSizeSmall)+int(blockSizeV1*2+blockSizeV2*2))) 66 } else { 67 apiRequestsMaxPerNode = cfg.RequestsMax 68 if len(globalEndpoints.Hostnames()) > 0 { 69 apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames()) 70 } 71 } 72 if cap(t.requestsPool) < apiRequestsMaxPerNode { 73 // Only replace if needed. 74 // Existing requests will use the previous limit, 75 // but new requests will use the new limit. 76 // There will be a short overlap window, 77 // but this shouldn't last long. 78 t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode) 79 } 80 t.requestsDeadline = cfg.RequestsDeadline 81 t.listQuorum = cfg.GetListQuorum() 82 t.extendListLife = cfg.ExtendListLife 83 if globalReplicationPool != nil && 84 cfg.ReplicationWorkers != t.replicationWorkers { 85 globalReplicationPool.Resize(cfg.ReplicationWorkers) 86 } 87 t.replicationWorkers = cfg.ReplicationWorkers 88 } 89 90 func (t *apiConfig) getListQuorum() int { 91 t.mu.RLock() 92 defer t.mu.RUnlock() 93 94 return t.listQuorum 95 } 96 97 func (t *apiConfig) getExtendListLife() time.Duration { 98 t.mu.RLock() 99 defer t.mu.RUnlock() 100 101 return t.extendListLife 102 } 103 104 func (t *apiConfig) getCorsAllowOrigins() []string { 105 t.mu.RLock() 106 defer t.mu.RUnlock() 107 108 corsAllowOrigins := make([]string, len(t.corsAllowOrigins)) 109 copy(corsAllowOrigins, t.corsAllowOrigins) 110 return corsAllowOrigins 111 } 112 113 func (t *apiConfig) getClusterDeadline() time.Duration { 114 t.mu.RLock() 115 defer t.mu.RUnlock() 116 117 if t.clusterDeadline == 0 { 118 return 10 * time.Second 119 } 120 121 return t.clusterDeadline 122 } 123 124 func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) { 125 t.mu.RLock() 126 defer t.mu.RUnlock() 127 128 if t.requestsPool == nil { 129 return nil, time.Duration(0) 130 } 131 132 return t.requestsPool, t.requestsDeadline 133 } 134 135 // MaxClients throttles the S3 API calls 136 func MaxClients(f http.HandlerFunc) http.HandlerFunc { 137 return func(w http.ResponseWriter, r *http.Request) { 138 pool, deadline := globalAPIConfig.getRequestsPool() 139 if pool == nil { 140 f.ServeHTTP(w, r) 141 return 142 } 143 144 globalHTTPStats.addRequestsInQueue(1) 145 146 deadlineTimer := time.NewTimer(deadline) 147 defer deadlineTimer.Stop() 148 149 select { 150 case pool <- struct{}{}: 151 defer func() { <-pool }() 152 globalHTTPStats.addRequestsInQueue(-1) 153 f.ServeHTTP(w, r) 154 case <-deadlineTimer.C: 155 // Send a http timeout message 156 WriteErrorResponse(r.Context(), w, 157 errorCodes.ToAPIErr(ErrOperationMaxedOut), 158 r.URL, guessIsBrowserReq(r)) 159 globalHTTPStats.addRequestsInQueue(-1) 160 return 161 case <-r.Context().Done(): 162 globalHTTPStats.addRequestsInQueue(-1) 163 return 164 } 165 } 166 } 167 168 func (t *apiConfig) getReplicationWorkers() int { 169 t.mu.RLock() 170 defer t.mu.RUnlock() 171 172 return t.replicationWorkers 173 }