github.com/minio/console@v1.4.1/pkg/utils/parity.go (about) 1 // This file is part of MinIO Console Server 2 // Copyright (c) 2021 MinIO, Inc. 3 // 4 // This program is free software: you can redistribute it and/or modify 5 // it under the terms of the GNU Affero General Public License as published by 6 // the Free Software Foundation, either version 3 of the License, or 7 // (at your option) any later version. 8 // 9 // This program is distributed in the hope that it will be useful, 10 // but WITHOUT ANY WARRANTY; without even the implied warranty of 11 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 // GNU Affero General Public License for more details. 13 // 14 // You should have received a copy of the GNU Affero General Public License 15 // along with this program. If not, see <http://www.gnu.org/licenses/>. 16 17 package utils 18 19 import ( 20 "errors" 21 "fmt" 22 "sort" 23 24 "github.com/minio/pkg/v3/ellipses" 25 ) 26 27 // This file implements and supports ellipses pattern for 28 // `minio server` command line arguments. 29 30 // Supported set sizes this is used to find the optimal 31 // single set size. 32 var setSizes = []uint64{4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} 33 34 // getDivisibleSize - returns a greatest common divisor of 35 // all the ellipses sizes. 36 func getDivisibleSize(totalSizes []uint64) (result uint64) { 37 gcd := func(x, y uint64) uint64 { 38 for y != 0 { 39 x, y = y, x%y 40 } 41 return x 42 } 43 result = totalSizes[0] 44 for i := 1; i < len(totalSizes); i++ { 45 result = gcd(result, totalSizes[i]) 46 } 47 return result 48 } 49 50 // isValidSetSize - checks whether given count is a valid set size for erasure coding. 51 var isValidSetSize = func(count uint64) bool { 52 return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1]) 53 } 54 55 // possibleSetCountsWithSymmetry returns symmetrical setCounts based on the 56 // input argument patterns, the symmetry calculation is to ensure that 57 // we also use uniform number of drives common across all ellipses patterns. 58 func possibleSetCountsWithSymmetry(setCounts []uint64, argPatterns []ellipses.ArgPattern) []uint64 { 59 newSetCounts := make(map[uint64]struct{}) 60 for _, ss := range setCounts { 61 var symmetry bool 62 for _, argPattern := range argPatterns { 63 for _, p := range argPattern { 64 if uint64(len(p.Seq)) > ss { 65 symmetry = uint64(len(p.Seq))%ss == 0 66 } else { 67 symmetry = ss%uint64(len(p.Seq)) == 0 68 } 69 } 70 } 71 // With no arg patterns, it is expected that user knows 72 // the right symmetry, so either ellipses patterns are 73 // provided (recommended) or no ellipses patterns. 74 if _, ok := newSetCounts[ss]; !ok && (symmetry || argPatterns == nil) { 75 newSetCounts[ss] = struct{}{} 76 } 77 } 78 79 setCounts = []uint64{} 80 for setCount := range newSetCounts { 81 setCounts = append(setCounts, setCount) 82 } 83 84 // Not necessarily needed but it ensures to the readers 85 // eyes that we prefer a sorted setCount slice for the 86 // subsequent function to figure out the right common 87 // divisor, it avoids loops. 88 sort.Slice(setCounts, func(i, j int) bool { 89 return setCounts[i] < setCounts[j] 90 }) 91 92 return setCounts 93 } 94 95 func commonSetDriveCount(divisibleSize uint64, setCounts []uint64) (setSize uint64) { 96 // prefers setCounts to be sorted for optimal behavior. 97 if divisibleSize < setCounts[len(setCounts)-1] { 98 return divisibleSize 99 } 100 101 // Figure out largest value of total_drives_in_erasure_set which results 102 // in least number of total_drives/total_drives_erasure_set ratio. 103 prevD := divisibleSize / setCounts[0] 104 for _, cnt := range setCounts { 105 if divisibleSize%cnt == 0 { 106 d := divisibleSize / cnt 107 if d <= prevD { 108 prevD = d 109 setSize = cnt 110 } 111 } 112 } 113 return setSize 114 } 115 116 // getSetIndexes returns list of indexes which provides the set size 117 // on each index, this function also determines the final set size 118 // The final set size has the affinity towards choosing smaller 119 // indexes (total sets) 120 func getSetIndexes(args []string, totalSizes []uint64, argPatterns []ellipses.ArgPattern) (setIndexes [][]uint64, err error) { 121 if len(totalSizes) == 0 || len(args) == 0 { 122 return nil, errors.New("invalid argument") 123 } 124 125 setIndexes = make([][]uint64, len(totalSizes)) 126 for _, totalSize := range totalSizes { 127 // Check if totalSize has minimum range upto setSize 128 if totalSize < setSizes[0] { 129 return nil, fmt.Errorf("incorrect number of endpoints provided %s", args) 130 } 131 } 132 133 commonSize := getDivisibleSize(totalSizes) 134 possibleSetCounts := func(setSize uint64) (ss []uint64) { 135 for _, s := range setSizes { 136 if setSize%s == 0 { 137 ss = append(ss, s) 138 } 139 } 140 return ss 141 } 142 143 setCounts := possibleSetCounts(commonSize) 144 if len(setCounts) == 0 { 145 err = fmt.Errorf("incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes) 146 return nil, err 147 } 148 149 // Returns possible set counts with symmetry. 150 setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns) 151 if len(setCounts) == 0 { 152 err = fmt.Errorf("no symmetric distribution detected with input endpoints provided %s, disks %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes) 153 return nil, err 154 } 155 156 // Final set size with all the symmetry accounted for. 157 setSize := commonSetDriveCount(commonSize, setCounts) 158 159 // Check whether setSize is with the supported range. 160 if !isValidSetSize(setSize) { 161 err = fmt.Errorf("incorrect number of endpoints provided %s, number of disks %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes) 162 return nil, err 163 } 164 165 for i := range totalSizes { 166 for j := uint64(0); j < totalSizes[i]/setSize; j++ { 167 setIndexes[i] = append(setIndexes[i], setSize) 168 } 169 } 170 171 return setIndexes, nil 172 } 173 174 // Return the total size for each argument patterns. 175 func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 { 176 var totalSizes []uint64 177 for _, argPattern := range argPatterns { 178 var totalSize uint64 = 1 179 for _, p := range argPattern { 180 totalSize *= uint64(len(p.Seq)) 181 } 182 totalSizes = append(totalSizes, totalSize) 183 } 184 return totalSizes 185 } 186 187 // PossibleParityValues returns possible parities for input args, 188 // parties are calculated in uniform manner for one pool or 189 // multiple pools, ensuring that parities returned are common 190 // and applicable across all pools. 191 func PossibleParityValues(args ...string) ([]string, error) { 192 setIndexes, err := parseEndpointSet(args...) 193 if err != nil { 194 return nil, err 195 } 196 maximumParity := setIndexes[0][0] / 2 197 var parities []string 198 for maximumParity >= 2 { 199 parities = append(parities, fmt.Sprintf("EC:%d", maximumParity)) 200 maximumParity-- 201 } 202 return parities, nil 203 } 204 205 // Parses all arguments and returns an endpointSet which is a collection 206 // of endpoints following the ellipses pattern, this is what is used 207 // by the object layer for initializing itself. 208 func parseEndpointSet(args ...string) (setIndexes [][]uint64, err error) { 209 argPatterns := make([]ellipses.ArgPattern, len(args)) 210 for i, arg := range args { 211 patterns, err := ellipses.FindEllipsesPatterns(arg) 212 if err != nil { 213 return nil, err 214 } 215 argPatterns[i] = patterns 216 } 217 218 return getSetIndexes(args, getTotalSizes(argPatterns), argPatterns) 219 }