github.com/cozy/cozy-stack@v0.0.0-20240603063001-31110fa4cae1/model/vfs/vfsswift/swift.go (about)

     1  package vfsswift
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"time"
     7  
     8  	"github.com/cozy/cozy-stack/pkg/utils"
     9  	multierror "github.com/hashicorp/go-multierror"
    10  	"github.com/ncw/swift/v2"
    11  )
    12  
    13  // maxNbFilesToDelete is the maximal number of files that we will try to delete
    14  // in a single call to swift.
    15  const maxNbFilesToDelete = 8000
    16  
    17  // maxSimultaneousCalls is the maximal number of simultaneous calls to Swift to
    18  // delete files in the same container.
    19  const maxSimultaneousCalls = 8
    20  
    21  var errFailFast = errors.New("fail fast")
    22  
    23  // DeleteContainer removes all the files inside the given container, and then
    24  // deletes it.
    25  func DeleteContainer(ctx context.Context, c *swift.Connection, container string) error {
    26  	_, _, err := c.Container(ctx, container)
    27  	if errors.Is(err, swift.ContainerNotFound) {
    28  		return nil
    29  	}
    30  	if err != nil {
    31  		return err
    32  	}
    33  	objectNames, err := c.ObjectNamesAll(ctx, container, nil)
    34  	if err != nil {
    35  		return err
    36  	}
    37  	if len(objectNames) > 0 {
    38  		if err = deleteContainerFiles(ctx, c, container, objectNames); err != nil {
    39  			return err
    40  		}
    41  	}
    42  
    43  	// XXX Swift has told us that all the files have been deleted on the bulk
    44  	// delete, but it only means that they have been deleted on one object
    45  	// server (at least). And, when we try to delete the container, Swift can
    46  	// send an error as some container servers will still have objects
    47  	// registered for this container. We will try several times to delete the
    48  	// container to work-around this limitation.
    49  	return utils.RetryWithExpBackoff(5, 2*time.Second, func() error {
    50  		err = c.ContainerDelete(ctx, container)
    51  		if errors.Is(err, swift.ContainerNotFound) {
    52  			return nil
    53  		}
    54  		return err
    55  	})
    56  }
    57  
    58  func deleteContainerFiles(ctx context.Context, c *swift.Connection, container string, objectNames []string) error {
    59  	nb := 1 + (len(objectNames)-1)/maxNbFilesToDelete
    60  	ch := make(chan error)
    61  
    62  	// Use a system of tokens to limit the number of simultaneous calls to
    63  	// Swift: only a goroutine that has a token can make a call.
    64  	tokens := make(chan int, maxSimultaneousCalls)
    65  	for k := 0; k < maxSimultaneousCalls; k++ {
    66  		tokens <- k
    67  	}
    68  
    69  	for i := 0; i < nb; i++ {
    70  		begin := i * maxNbFilesToDelete
    71  		end := (i + 1) * maxNbFilesToDelete
    72  		if end > len(objectNames) {
    73  			end = len(objectNames)
    74  		}
    75  		objectToDelete := objectNames[begin:end]
    76  		go func() {
    77  			k := <-tokens
    78  			_, err := c.BulkDelete(ctx, container, objectToDelete)
    79  			ch <- err
    80  			tokens <- k
    81  		}()
    82  	}
    83  
    84  	var errm error
    85  	for i := 0; i < nb; i++ {
    86  		if err := <-ch; err != nil {
    87  			errm = multierror.Append(errm, err)
    88  		}
    89  	}
    90  	// Get back the tokens to ensure that each goroutine can finish.
    91  	for k := 0; k < maxSimultaneousCalls; k++ {
    92  		<-tokens
    93  	}
    94  	return errm
    95  }