github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/pkg/ruler/rulestore/bucketclient/bucket_client.go (about) 1 package bucketclient 2 3 import ( 4 "bytes" 5 "context" 6 "encoding/base64" 7 "fmt" 8 "io/ioutil" 9 "strings" 10 11 "github.com/go-kit/log" 12 "github.com/go-kit/log/level" 13 "github.com/gogo/protobuf/proto" 14 "github.com/pkg/errors" 15 "github.com/thanos-io/thanos/pkg/objstore" 16 "golang.org/x/sync/errgroup" 17 18 "github.com/cortexproject/cortex/pkg/ruler/rulespb" 19 "github.com/cortexproject/cortex/pkg/ruler/rulestore" 20 "github.com/cortexproject/cortex/pkg/storage/bucket" 21 ) 22 23 const ( 24 // The bucket prefix under which all tenants rule groups are stored. 25 rulesPrefix = "rules" 26 27 loadConcurrency = 10 28 ) 29 30 var ( 31 errInvalidRuleGroupKey = errors.New("invalid rule group object key") 32 errEmptyUser = errors.New("empty user") 33 errEmptyNamespace = errors.New("empty namespace") 34 errEmptyGroupName = errors.New("empty group name") 35 ) 36 37 // BucketRuleStore is used to support the RuleStore interface against an object storage backend. It is implemented 38 // using the Thanos objstore.Bucket interface 39 type BucketRuleStore struct { 40 bucket objstore.Bucket 41 cfgProvider bucket.TenantConfigProvider 42 logger log.Logger 43 } 44 45 func NewBucketRuleStore(bkt objstore.Bucket, cfgProvider bucket.TenantConfigProvider, logger log.Logger) *BucketRuleStore { 46 return &BucketRuleStore{ 47 bucket: bucket.NewPrefixedBucketClient(bkt, rulesPrefix), 48 cfgProvider: cfgProvider, 49 logger: logger, 50 } 51 } 52 53 // getRuleGroup loads and return a rules group. If existing rule group is supplied, it is Reset and reused. If nil, new RuleGroupDesc is allocated. 54 func (b *BucketRuleStore) getRuleGroup(ctx context.Context, userID, namespace, groupName string, rg *rulespb.RuleGroupDesc) (*rulespb.RuleGroupDesc, error) { 55 userBucket := bucket.NewUserBucketClient(userID, b.bucket, b.cfgProvider) 56 objectKey := getRuleGroupObjectKey(namespace, groupName) 57 58 reader, err := userBucket.Get(ctx, objectKey) 59 if userBucket.IsObjNotFoundErr(err) { 60 level.Debug(b.logger).Log("msg", "rule group does not exist", "user", userID, "key", objectKey) 61 return nil, rulestore.ErrGroupNotFound 62 } 63 64 if err != nil { 65 return nil, errors.Wrapf(err, "failed to get rule group %s", objectKey) 66 } 67 defer func() { _ = reader.Close() }() 68 69 buf, err := ioutil.ReadAll(reader) 70 if err != nil { 71 return nil, errors.Wrapf(err, "failed to read rule group %s", objectKey) 72 } 73 74 if rg == nil { 75 rg = &rulespb.RuleGroupDesc{} 76 } else { 77 rg.Reset() 78 } 79 80 err = proto.Unmarshal(buf, rg) 81 if err != nil { 82 return nil, errors.Wrapf(err, "failed to unmarshal rule group %s", objectKey) 83 } 84 85 return rg, nil 86 } 87 88 // ListAllUsers implements rules.RuleStore. 89 func (b *BucketRuleStore) ListAllUsers(ctx context.Context) ([]string, error) { 90 var users []string 91 err := b.bucket.Iter(ctx, "", func(user string) error { 92 users = append(users, strings.TrimSuffix(user, objstore.DirDelim)) 93 return nil 94 }) 95 if err != nil { 96 return nil, fmt.Errorf("unable to list users in rule store bucket: %w", err) 97 } 98 99 return users, nil 100 } 101 102 // ListAllRuleGroups implements rules.RuleStore. 103 func (b *BucketRuleStore) ListAllRuleGroups(ctx context.Context) (map[string]rulespb.RuleGroupList, error) { 104 out := map[string]rulespb.RuleGroupList{} 105 106 // List rule groups for all tenants. 107 err := b.bucket.Iter(ctx, "", func(key string) error { 108 userID, namespace, group, err := parseRuleGroupObjectKeyWithUser(key) 109 if err != nil { 110 level.Warn(b.logger).Log("msg", "invalid rule group object key found while listing rule groups", "key", key, "err", err) 111 112 // Do not fail just because of a spurious item in the bucket. 113 return nil 114 } 115 116 out[userID] = append(out[userID], &rulespb.RuleGroupDesc{ 117 User: userID, 118 Namespace: namespace, 119 Name: group, 120 }) 121 return nil 122 }, objstore.WithRecursiveIter) 123 124 if err != nil { 125 return nil, err 126 } 127 128 return out, nil 129 } 130 131 // ListRuleGroupsForUserAndNamespace implements rules.RuleStore. 132 func (b *BucketRuleStore) ListRuleGroupsForUserAndNamespace(ctx context.Context, userID string, namespace string) (rulespb.RuleGroupList, error) { 133 userBucket := bucket.NewUserBucketClient(userID, b.bucket, b.cfgProvider) 134 135 groupList := rulespb.RuleGroupList{} 136 137 // The prefix to list objects depends on whether the namespace has been 138 // specified in the request. 139 prefix := "" 140 if namespace != "" { 141 prefix = getNamespacePrefix(namespace) 142 } 143 144 err := userBucket.Iter(ctx, prefix, func(key string) error { 145 namespace, group, err := parseRuleGroupObjectKey(key) 146 if err != nil { 147 level.Warn(b.logger).Log("msg", "invalid rule group object key found while listing rule groups", "user", userID, "key", key, "err", err) 148 149 // Do not fail just because of a spurious item in the bucket. 150 return nil 151 } 152 153 groupList = append(groupList, &rulespb.RuleGroupDesc{ 154 User: userID, 155 Namespace: namespace, 156 Name: group, 157 }) 158 return nil 159 }, objstore.WithRecursiveIter) 160 if err != nil { 161 return nil, err 162 } 163 164 return groupList, nil 165 } 166 167 // LoadRuleGroups implements rules.RuleStore. 168 func (b *BucketRuleStore) LoadRuleGroups(ctx context.Context, groupsToLoad map[string]rulespb.RuleGroupList) error { 169 ch := make(chan *rulespb.RuleGroupDesc) 170 171 // Given we store one file per rule group. With this, we create a pool of workers that will 172 // download all rule groups in parallel. We limit the number of workers to avoid a 173 // particular user having too many rule groups rate limiting us with the object storage. 174 g, gCtx := errgroup.WithContext(ctx) 175 for i := 0; i < loadConcurrency; i++ { 176 g.Go(func() error { 177 for gr := range ch { 178 user, namespace, group := gr.GetUser(), gr.GetNamespace(), gr.GetName() 179 if user == "" || namespace == "" || group == "" { 180 return fmt.Errorf("invalid rule group: user=%q, namespace=%q, group=%q", user, namespace, group) 181 } 182 183 gr, err := b.getRuleGroup(gCtx, user, namespace, group, gr) // reuse group pointer from the map. 184 if err != nil { 185 return errors.Wrapf(err, "get rule group user=%q, namespace=%q, name=%q", user, namespace, group) 186 } 187 188 if user != gr.User || namespace != gr.Namespace || group != gr.Name { 189 return fmt.Errorf("mismatch between requested rule group and loaded rule group, requested: user=%q, namespace=%q, group=%q, loaded: user=%q, namespace=%q, group=%q", user, namespace, group, gr.User, gr.Namespace, gr.Name) 190 } 191 } 192 193 return nil 194 }) 195 } 196 197 outer: 198 for _, gs := range groupsToLoad { 199 for _, g := range gs { 200 if g == nil { 201 continue 202 } 203 select { 204 case <-gCtx.Done(): 205 break outer 206 case ch <- g: 207 // ok 208 } 209 } 210 } 211 close(ch) 212 213 return g.Wait() 214 } 215 216 // GetRuleGroup implements rules.RuleStore. 217 func (b *BucketRuleStore) GetRuleGroup(ctx context.Context, userID string, namespace string, group string) (*rulespb.RuleGroupDesc, error) { 218 return b.getRuleGroup(ctx, userID, namespace, group, nil) 219 } 220 221 // SetRuleGroup implements rules.RuleStore. 222 func (b *BucketRuleStore) SetRuleGroup(ctx context.Context, userID string, namespace string, group *rulespb.RuleGroupDesc) error { 223 userBucket := bucket.NewUserBucketClient(userID, b.bucket, b.cfgProvider) 224 data, err := proto.Marshal(group) 225 if err != nil { 226 return err 227 } 228 229 return userBucket.Upload(ctx, getRuleGroupObjectKey(namespace, group.Name), bytes.NewBuffer(data)) 230 } 231 232 // DeleteRuleGroup implements rules.RuleStore. 233 func (b *BucketRuleStore) DeleteRuleGroup(ctx context.Context, userID string, namespace string, group string) error { 234 userBucket := bucket.NewUserBucketClient(userID, b.bucket, b.cfgProvider) 235 err := userBucket.Delete(ctx, getRuleGroupObjectKey(namespace, group)) 236 if b.bucket.IsObjNotFoundErr(err) { 237 return rulestore.ErrGroupNotFound 238 } 239 return err 240 } 241 242 // DeleteNamespace implements rules.RuleStore. 243 func (b *BucketRuleStore) DeleteNamespace(ctx context.Context, userID string, namespace string) error { 244 ruleGroupList, err := b.ListRuleGroupsForUserAndNamespace(ctx, userID, namespace) 245 if err != nil { 246 return err 247 } 248 249 if len(ruleGroupList) == 0 { 250 return rulestore.ErrGroupNamespaceNotFound 251 } 252 253 userBucket := bucket.NewUserBucketClient(userID, b.bucket, b.cfgProvider) 254 for _, rg := range ruleGroupList { 255 if err := ctx.Err(); err != nil { 256 return err 257 } 258 objectKey := getRuleGroupObjectKey(rg.Namespace, rg.Name) 259 level.Debug(b.logger).Log("msg", "deleting rule group", "user", userID, "namespace", namespace, "key", objectKey) 260 err = userBucket.Delete(ctx, objectKey) 261 if err != nil { 262 level.Error(b.logger).Log("msg", "unable to delete rule group from namespace", "user", userID, "namespace", namespace, "key", objectKey, "err", err) 263 return err 264 } 265 } 266 267 return nil 268 } 269 270 func getNamespacePrefix(namespace string) string { 271 return base64.URLEncoding.EncodeToString([]byte(namespace)) + objstore.DirDelim 272 } 273 274 func getRuleGroupObjectKey(namespace, group string) string { 275 return getNamespacePrefix(namespace) + base64.URLEncoding.EncodeToString([]byte(group)) 276 } 277 278 // parseRuleGroupObjectKeyWithUser parses a bucket object key in the format "<user>/<namespace>/<rules group>". 279 func parseRuleGroupObjectKeyWithUser(key string) (user, namespace, group string, err error) { 280 parts := strings.SplitN(key, objstore.DirDelim, 2) 281 if len(parts) != 2 { 282 return "", "", "", errInvalidRuleGroupKey 283 } 284 285 user = parts[0] 286 if user == "" { 287 return "", "", "", errEmptyUser 288 } 289 namespace, group, err = parseRuleGroupObjectKey(parts[1]) 290 return 291 } 292 293 // parseRuleGroupObjectKey parses a bucket object key in the format "<namespace>/<rules group>". 294 func parseRuleGroupObjectKey(key string) (namespace, group string, _ error) { 295 parts := strings.Split(key, objstore.DirDelim) 296 if len(parts) != 2 { 297 return "", "", errInvalidRuleGroupKey 298 } 299 300 decodedNamespace, err := base64.URLEncoding.DecodeString(parts[0]) 301 if err != nil { 302 return "", "", err 303 } 304 305 if len(decodedNamespace) == 0 { 306 return "", "", errEmptyNamespace 307 } 308 309 decodedGroup, err := base64.URLEncoding.DecodeString(parts[1]) 310 if err != nil { 311 return "", "", err 312 } 313 314 if len(decodedGroup) == 0 { 315 return "", "", errEmptyGroupName 316 } 317 318 return string(decodedNamespace), string(decodedGroup), nil 319 }