storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/background-heal-ops.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2019 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "context" 21 "time" 22 23 "storj.io/minio/cmd/logger" 24 "storj.io/minio/pkg/madmin" 25 ) 26 27 // healTask represents what to heal along with options 28 // path: '/' => Heal disk formats along with metadata 29 // path: 'bucket/' or '/bucket/' => Heal bucket 30 // path: 'bucket/object' => Heal object 31 type healTask struct { 32 bucket string 33 object string 34 versionID string 35 opts madmin.HealOpts 36 // Healing response will be sent here 37 responseCh chan healResult 38 } 39 40 // healResult represents a healing result with a possible error 41 type healResult struct { 42 result madmin.HealResultItem 43 err error 44 } 45 46 // healRoutine receives heal tasks, to heal buckets, objects and format.json 47 type healRoutine struct { 48 tasks chan healTask 49 doneCh chan struct{} 50 } 51 52 // Add a new task in the tasks queue 53 func (h *healRoutine) queueHealTask(task healTask) { 54 h.tasks <- task 55 } 56 57 func waitForLowHTTPReq(maxIO int, maxWait time.Duration) { 58 // No need to wait run at full speed. 59 if maxIO <= 0 { 60 return 61 } 62 63 // At max 10 attempts to wait with 100 millisecond interval before proceeding 64 waitTick := 100 * time.Millisecond 65 66 // Bucket notification and http trace are not costly, it is okay to ignore them 67 // while counting the number of concurrent connections 68 maxIOFn := func() int { 69 return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers()) 70 } 71 72 tmpMaxWait := maxWait 73 if httpServer := newHTTPServerFn(); httpServer != nil { 74 // Any requests in progress, delay the heal. 75 for httpServer.GetRequestCount() >= maxIOFn() { 76 if tmpMaxWait > 0 { 77 if tmpMaxWait < waitTick { 78 time.Sleep(tmpMaxWait) 79 } else { 80 time.Sleep(waitTick) 81 } 82 tmpMaxWait = tmpMaxWait - waitTick 83 } 84 if tmpMaxWait <= 0 { 85 if intDataUpdateTracker.debug { 86 logger.Info("waitForLowHTTPReq: waited max %s, resuming", maxWait) 87 } 88 break 89 } 90 } 91 } 92 } 93 94 // Wait for heal requests and process them 95 func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) { 96 for { 97 select { 98 case task, ok := <-h.tasks: 99 if !ok { 100 return 101 } 102 103 var res madmin.HealResultItem 104 var err error 105 switch task.bucket { 106 case nopHeal: 107 continue 108 case SlashSeparator: 109 res, err = healDiskFormat(ctx, objAPI, task.opts) 110 default: 111 if task.object == "" { 112 res, err = objAPI.HealBucket(ctx, task.bucket, task.opts) 113 } else { 114 res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts) 115 } 116 } 117 task.responseCh <- healResult{result: res, err: err} 118 119 case <-h.doneCh: 120 return 121 case <-ctx.Done(): 122 return 123 } 124 } 125 } 126 127 func newHealRoutine() *healRoutine { 128 return &healRoutine{ 129 tasks: make(chan healTask), 130 doneCh: make(chan struct{}), 131 } 132 133 } 134 135 // healDiskFormat - heals format.json, return value indicates if a 136 // failure error occurred. 137 func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) { 138 res, err := objAPI.HealFormat(ctx, opts.DryRun) 139 140 // return any error, ignore error returned when disks have 141 // already healed. 142 if err != nil && err != errNoHealRequired { 143 return madmin.HealResultItem{}, err 144 } 145 146 return res, nil 147 }