storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/erasure-encode.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"context"
    21  	"io"
    22  
    23  	"sync"
    24  
    25  	"storj.io/minio/cmd/logger"
    26  )
    27  
    28  // Writes in parallel to writers
    29  type parallelWriter struct {
    30  	writers     []io.Writer
    31  	writeQuorum int
    32  	errs        []error
    33  }
    34  
    35  // Write writes data to writers in parallel.
    36  func (p *parallelWriter) Write(ctx context.Context, blocks [][]byte) error {
    37  	var wg sync.WaitGroup
    38  
    39  	for i := range p.writers {
    40  		if p.writers[i] == nil {
    41  			p.errs[i] = errDiskNotFound
    42  			continue
    43  		}
    44  
    45  		wg.Add(1)
    46  		go func(i int) {
    47  			defer wg.Done()
    48  			_, p.errs[i] = p.writers[i].Write(blocks[i])
    49  			if p.errs[i] != nil {
    50  				p.writers[i] = nil
    51  			}
    52  		}(i)
    53  	}
    54  	wg.Wait()
    55  
    56  	// If nilCount >= p.writeQuorum, we return nil. This is because HealFile() uses
    57  	// CreateFile with p.writeQuorum=1 to accommodate healing of single disk.
    58  	// i.e if we do no return here in such a case, reduceWriteQuorumErrs() would
    59  	// return a quorum error to HealFile().
    60  	nilCount := 0
    61  	for _, err := range p.errs {
    62  		if err == nil {
    63  			nilCount++
    64  		}
    65  	}
    66  	if nilCount >= p.writeQuorum {
    67  		return nil
    68  	}
    69  	return reduceWriteQuorumErrs(ctx, p.errs, objectOpIgnoredErrs, p.writeQuorum)
    70  }
    71  
    72  // Encode reads from the reader, erasure-encodes the data and writes to the writers.
    73  func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer, buf []byte, quorum int) (total int64, err error) {
    74  	writer := &parallelWriter{
    75  		writers:     writers,
    76  		writeQuorum: quorum,
    77  		errs:        make([]error, len(writers)),
    78  	}
    79  
    80  	for {
    81  		var blocks [][]byte
    82  		n, err := io.ReadFull(src, buf)
    83  		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
    84  			logger.LogIf(ctx, err)
    85  			return 0, err
    86  		}
    87  		eof := err == io.EOF || err == io.ErrUnexpectedEOF
    88  		if n == 0 && total != 0 {
    89  			// Reached EOF, nothing more to be done.
    90  			break
    91  		}
    92  		// We take care of the situation where if n == 0 and total == 0 by creating empty data and parity files.
    93  		blocks, err = e.EncodeData(ctx, buf[:n])
    94  		if err != nil {
    95  			logger.LogIf(ctx, err)
    96  			return 0, err
    97  		}
    98  
    99  		if err = writer.Write(ctx, blocks); err != nil {
   100  			logger.LogIf(ctx, err)
   101  			return 0, err
   102  		}
   103  		total += int64(n)
   104  		if eof {
   105  			break
   106  		}
   107  	}
   108  	return total, nil
   109  }