storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/background-heal-ops.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"time"
    22  
    23  	"storj.io/minio/cmd/logger"
    24  	"storj.io/minio/pkg/madmin"
    25  )
    26  
    27  // healTask represents what to heal along with options
    28  //   path: '/' =>  Heal disk formats along with metadata
    29  //   path: 'bucket/' or '/bucket/' => Heal bucket
    30  //   path: 'bucket/object' => Heal object
    31  type healTask struct {
    32  	bucket    string
    33  	object    string
    34  	versionID string
    35  	opts      madmin.HealOpts
    36  	// Healing response will be sent here
    37  	responseCh chan healResult
    38  }
    39  
    40  // healResult represents a healing result with a possible error
    41  type healResult struct {
    42  	result madmin.HealResultItem
    43  	err    error
    44  }
    45  
    46  // healRoutine receives heal tasks, to heal buckets, objects and format.json
    47  type healRoutine struct {
    48  	tasks  chan healTask
    49  	doneCh chan struct{}
    50  }
    51  
    52  // Add a new task in the tasks queue
    53  func (h *healRoutine) queueHealTask(task healTask) {
    54  	h.tasks <- task
    55  }
    56  
    57  func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
    58  	// No need to wait run at full speed.
    59  	if maxIO <= 0 {
    60  		return
    61  	}
    62  
    63  	// At max 10 attempts to wait with 100 millisecond interval before proceeding
    64  	waitTick := 100 * time.Millisecond
    65  
    66  	// Bucket notification and http trace are not costly, it is okay to ignore them
    67  	// while counting the number of concurrent connections
    68  	maxIOFn := func() int {
    69  		return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalTrace.NumSubscribers())
    70  	}
    71  
    72  	tmpMaxWait := maxWait
    73  	if httpServer := newHTTPServerFn(); httpServer != nil {
    74  		// Any requests in progress, delay the heal.
    75  		for httpServer.GetRequestCount() >= maxIOFn() {
    76  			if tmpMaxWait > 0 {
    77  				if tmpMaxWait < waitTick {
    78  					time.Sleep(tmpMaxWait)
    79  				} else {
    80  					time.Sleep(waitTick)
    81  				}
    82  				tmpMaxWait = tmpMaxWait - waitTick
    83  			}
    84  			if tmpMaxWait <= 0 {
    85  				if intDataUpdateTracker.debug {
    86  					logger.Info("waitForLowHTTPReq: waited max %s, resuming", maxWait)
    87  				}
    88  				break
    89  			}
    90  		}
    91  	}
    92  }
    93  
    94  // Wait for heal requests and process them
    95  func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
    96  	for {
    97  		select {
    98  		case task, ok := <-h.tasks:
    99  			if !ok {
   100  				return
   101  			}
   102  
   103  			var res madmin.HealResultItem
   104  			var err error
   105  			switch task.bucket {
   106  			case nopHeal:
   107  				continue
   108  			case SlashSeparator:
   109  				res, err = healDiskFormat(ctx, objAPI, task.opts)
   110  			default:
   111  				if task.object == "" {
   112  					res, err = objAPI.HealBucket(ctx, task.bucket, task.opts)
   113  				} else {
   114  					res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts)
   115  				}
   116  			}
   117  			task.responseCh <- healResult{result: res, err: err}
   118  
   119  		case <-h.doneCh:
   120  			return
   121  		case <-ctx.Done():
   122  			return
   123  		}
   124  	}
   125  }
   126  
   127  func newHealRoutine() *healRoutine {
   128  	return &healRoutine{
   129  		tasks:  make(chan healTask),
   130  		doneCh: make(chan struct{}),
   131  	}
   132  
   133  }
   134  
   135  // healDiskFormat - heals format.json, return value indicates if a
   136  // failure error occurred.
   137  func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {
   138  	res, err := objAPI.HealFormat(ctx, opts.DryRun)
   139  
   140  	// return any error, ignore error returned when disks have
   141  	// already healed.
   142  	if err != nil && err != errNoHealRequired {
   143  		return madmin.HealResultItem{}, err
   144  	}
   145  
   146  	return res, nil
   147  }