github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/internal/store/store.go (about) 1 // Copyright (c) 2015-2023 MinIO, Inc. 2 // 3 // This file is part of MinIO Object Storage stack 4 // 5 // This program is free software: you can redistribute it and/or modify 6 // it under the terms of the GNU Affero General Public License as published by 7 // the Free Software Foundation, either version 3 of the License, or 8 // (at your option) any later version. 9 // 10 // This program is distributed in the hope that it will be useful 11 // but WITHOUT ANY WARRANTY; without even the implied warranty of 12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 // GNU Affero General Public License for more details. 14 // 15 // You should have received a copy of the GNU Affero General Public License 16 // along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 package store 19 20 import ( 21 "context" 22 "errors" 23 "fmt" 24 "strings" 25 "time" 26 27 xioutil "github.com/minio/minio/internal/ioutil" 28 ) 29 30 const ( 31 retryInterval = 3 * time.Second 32 ) 33 34 type logger = func(ctx context.Context, err error, id string, errKind ...interface{}) 35 36 // ErrNotConnected - indicates that the target connection is not active. 37 var ErrNotConnected = errors.New("not connected to target server/service") 38 39 // Target - store target interface 40 type Target interface { 41 Name() string 42 SendFromStore(key Key) error 43 } 44 45 // Store - Used to persist items. 46 type Store[I any] interface { 47 Put(item I) error 48 PutMultiple(item []I) error 49 Get(key string) (I, error) 50 GetRaw(key string) ([]byte, error) 51 Len() int 52 List() ([]string, error) 53 Del(key string) error 54 DelList(key []string) error 55 Open() error 56 Delete() error 57 Extension() string 58 } 59 60 // Key denotes the key present in the store. 61 type Key struct { 62 Name string 63 IsLast bool 64 } 65 66 // replayItems - Reads the items from the store and replays. 67 func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan Key { 68 keyCh := make(chan Key) 69 70 go func() { 71 defer xioutil.SafeClose(keyCh) 72 73 retryTicker := time.NewTicker(retryInterval) 74 defer retryTicker.Stop() 75 76 for { 77 names, err := store.List() 78 if err != nil { 79 log(context.Background(), fmt.Errorf("store.List() failed with: %w", err), id) 80 } else { 81 keyCount := len(names) 82 for i, name := range names { 83 select { 84 case keyCh <- Key{strings.TrimSuffix(name, store.Extension()), keyCount == i+1}: 85 // Get next key. 86 case <-doneCh: 87 return 88 } 89 } 90 } 91 92 select { 93 case <-retryTicker.C: 94 case <-doneCh: 95 return 96 } 97 } 98 }() 99 100 return keyCh 101 } 102 103 // sendItems - Reads items from the store and re-plays. 104 func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger logger) { 105 retryTicker := time.NewTicker(retryInterval) 106 defer retryTicker.Stop() 107 108 send := func(key Key) bool { 109 for { 110 err := target.SendFromStore(key) 111 if err == nil { 112 break 113 } 114 115 logger( 116 context.Background(), 117 fmt.Errorf("unable to send webhook log entry to '%s' err '%w'", target.Name(), err), 118 target.Name(), 119 ) 120 121 select { 122 // Retrying after 3secs back-off 123 case <-retryTicker.C: 124 case <-doneCh: 125 return false 126 } 127 } 128 return true 129 } 130 131 for { 132 select { 133 case key, ok := <-keyCh: 134 if !ok { 135 return 136 } 137 138 if !send(key) { 139 return 140 } 141 case <-doneCh: 142 return 143 } 144 } 145 } 146 147 // StreamItems reads the keys from the store and replays the corresponding item to the target. 148 func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, logger logger) { 149 go func() { 150 keyCh := replayItems(store, doneCh, logger, target.Name()) 151 sendItems(target, keyCh, doneCh, logger) 152 }() 153 }