github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/block/gs/compose.go (about)

     1  package gs
     2  
     3  import (
     4  	"fmt"
     5  )
     6  
     7  const MaxPartsInCompose = 32
     8  
     9  type ComposeFunc func(target string, parts []string) error
    10  
    11  func ComposeAll(target string, parts []string, composeFunc ComposeFunc) error {
    12  	for layer := 1; len(parts) > MaxPartsInCompose; layer++ {
    13  		var nextParts []string
    14  		for i := 0; i < len(parts); i += MaxPartsInCompose {
    15  			chunkSize := len(parts) - i
    16  			if chunkSize > MaxPartsInCompose {
    17  				chunkSize = MaxPartsInCompose
    18  			}
    19  			chunk := parts[i : i+chunkSize]
    20  			if chunkSize == 1 || (chunkSize < MaxPartsInCompose && len(nextParts)+chunkSize <= MaxPartsInCompose) {
    21  				nextParts = append(nextParts, chunk...)
    22  			} else {
    23  				targetName := fmt.Sprintf("%s_%d", chunk[0], layer)
    24  				if err := composeFunc(targetName, chunk); err != nil {
    25  					return err
    26  				}
    27  				nextParts = append(nextParts, targetName)
    28  			}
    29  		}
    30  		parts = nextParts
    31  	}
    32  	return composeFunc(target, parts)
    33  }