github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/docs/debugging/hash-set/main.go (about)

     1  // Copyright (c) 2015-2021 MinIO, Inc.
     2  //
     3  // This file is part of MinIO Object Storage stack
     4  //
     5  // This program is free software: you can redistribute it and/or modify
     6  // it under the terms of the GNU Affero General Public License as published by
     7  // the Free Software Foundation, either version 3 of the License, or
     8  // (at your option) any later version.
     9  //
    10  // This program is distributed in the hope that it will be useful
    11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13  // GNU Affero General Public License for more details.
    14  //
    15  // You should have received a copy of the GNU Affero General Public License
    16  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17  
    18  package main
    19  
    20  import (
    21  	"bufio"
    22  	"bytes"
    23  	"encoding/binary"
    24  	"flag"
    25  	"fmt"
    26  	"hash/crc32"
    27  	"log"
    28  	"os"
    29  	"strings"
    30  
    31  	"github.com/dchest/siphash"
    32  	"github.com/google/uuid"
    33  )
    34  
    35  // hashes the key returning an integer based on the input algorithm.
    36  // This function currently supports
    37  // - SIPMOD
    38  func sipHashMod(key string, cardinality int, id [16]byte) int {
    39  	if cardinality <= 0 {
    40  		return -1
    41  	}
    42  	// use the faster version as per siphash docs
    43  	// https://github.com/dchest/siphash#usage
    44  	k0, k1 := binary.LittleEndian.Uint64(id[0:8]), binary.LittleEndian.Uint64(id[8:16])
    45  	sum64 := siphash.Hash(k0, k1, []byte(key))
    46  	return int(sum64 % uint64(cardinality))
    47  }
    48  
    49  // hashOrder - hashes input key to return consistent
    50  // hashed integer slice. Returned integer order is salted
    51  // with an input key. This results in consistent order.
    52  // NOTE: collisions are fine, we are not looking for uniqueness
    53  // in the slices returned.
    54  func hashOrder(key string, cardinality int) []int {
    55  	if cardinality <= 0 {
    56  		// Returns an empty int slice for cardinality < 0.
    57  		return nil
    58  	}
    59  
    60  	nums := make([]int, cardinality)
    61  	keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable)
    62  
    63  	start := int(keyCrc % uint32(cardinality))
    64  	for i := 1; i <= cardinality; i++ {
    65  		nums[i-1] = 1 + ((start + i) % cardinality)
    66  	}
    67  	return nums
    68  }
    69  
    70  var (
    71  	file, object, deploymentID, prefix string
    72  	setCount, shards                   int
    73  	verbose                            bool
    74  )
    75  
    76  func main() {
    77  	flag.StringVar(&file, "file", "", "Read all objects from file, newline separated")
    78  	flag.StringVar(&prefix, "prefix", "", "Add prefix to all objects")
    79  	flag.StringVar(&object, "object", "", "Select an object")
    80  	flag.StringVar(&deploymentID, "deployment-id", "", "MinIO deployment ID, obtained from 'format.json'")
    81  	flag.IntVar(&setCount, "set-count", 0, "Total set count")
    82  	flag.IntVar(&shards, "shards", 0, "Total shards count")
    83  	flag.BoolVar(&verbose, "v", false, "Display all objects")
    84  
    85  	flag.Parse()
    86  
    87  	if deploymentID == "" {
    88  		log.Fatalln("deployment ID is mandatory")
    89  	}
    90  
    91  	if setCount == 0 {
    92  		log.Fatalln("set count cannot be zero")
    93  	}
    94  
    95  	id := uuid.MustParse(deploymentID)
    96  
    97  	if file != "" {
    98  		distrib := make([][]string, setCount)
    99  		b, err := os.ReadFile(file)
   100  		if err != nil {
   101  			log.Fatalln(err)
   102  		}
   103  		b = bytes.ReplaceAll(b, []byte("\r"), []byte{})
   104  		sc := bufio.NewScanner(bytes.NewBuffer(b))
   105  		for sc.Scan() {
   106  			object = strings.TrimSpace(sc.Text())
   107  			set := sipHashMod(prefix+object, setCount, id)
   108  			distrib[set] = append(distrib[set], prefix+object)
   109  		}
   110  		for set, files := range distrib {
   111  			fmt.Println("Set:", set+1, "Objects:", len(files))
   112  			if !verbose {
   113  				continue
   114  			}
   115  			for _, s := range files {
   116  				fmt.Printf("\t%s\n", s)
   117  			}
   118  		}
   119  		os.Exit(0)
   120  	}
   121  
   122  	if object == "" {
   123  		log.Fatalln("object name is mandatory")
   124  	}
   125  
   126  	if shards != 0 {
   127  		fmt.Println("Erasure distribution for the object", hashOrder(prefix+object, shards))
   128  	}
   129  	fmt.Println("Erasure setNumber for the object", sipHashMod(prefix+object, setCount, id)+1)
   130  }