github.com/minio/minio@v0.0.0-20240328213742-3f72439b8a27/internal/store/store.go (about)

     1  // Copyright (c) 2015-2023 MinIO, Inc.
     2  //
     3  // This file is part of MinIO Object Storage stack
     4  //
     5  // This program is free software: you can redistribute it and/or modify
     6  // it under the terms of the GNU Affero General Public License as published by
     7  // the Free Software Foundation, either version 3 of the License, or
     8  // (at your option) any later version.
     9  //
    10  // This program is distributed in the hope that it will be useful
    11  // but WITHOUT ANY WARRANTY; without even the implied warranty of
    12  // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13  // GNU Affero General Public License for more details.
    14  //
    15  // You should have received a copy of the GNU Affero General Public License
    16  // along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17  
    18  package store
    19  
    20  import (
    21  	"context"
    22  	"errors"
    23  	"fmt"
    24  	"strings"
    25  	"time"
    26  
    27  	xioutil "github.com/minio/minio/internal/ioutil"
    28  )
    29  
    30  const (
    31  	retryInterval = 3 * time.Second
    32  )
    33  
    34  type logger = func(ctx context.Context, err error, id string, errKind ...interface{})
    35  
    36  // ErrNotConnected - indicates that the target connection is not active.
    37  var ErrNotConnected = errors.New("not connected to target server/service")
    38  
    39  // Target - store target interface
    40  type Target interface {
    41  	Name() string
    42  	SendFromStore(key Key) error
    43  }
    44  
    45  // Store - Used to persist items.
    46  type Store[I any] interface {
    47  	Put(item I) error
    48  	PutMultiple(item []I) error
    49  	Get(key string) (I, error)
    50  	GetRaw(key string) ([]byte, error)
    51  	Len() int
    52  	List() ([]string, error)
    53  	Del(key string) error
    54  	DelList(key []string) error
    55  	Open() error
    56  	Delete() error
    57  	Extension() string
    58  }
    59  
    60  // Key denotes the key present in the store.
    61  type Key struct {
    62  	Name   string
    63  	IsLast bool
    64  }
    65  
    66  // replayItems - Reads the items from the store and replays.
    67  func replayItems[I any](store Store[I], doneCh <-chan struct{}, log logger, id string) <-chan Key {
    68  	keyCh := make(chan Key)
    69  
    70  	go func() {
    71  		defer xioutil.SafeClose(keyCh)
    72  
    73  		retryTicker := time.NewTicker(retryInterval)
    74  		defer retryTicker.Stop()
    75  
    76  		for {
    77  			names, err := store.List()
    78  			if err != nil {
    79  				log(context.Background(), fmt.Errorf("store.List() failed with: %w", err), id)
    80  			} else {
    81  				keyCount := len(names)
    82  				for i, name := range names {
    83  					select {
    84  					case keyCh <- Key{strings.TrimSuffix(name, store.Extension()), keyCount == i+1}:
    85  					// Get next key.
    86  					case <-doneCh:
    87  						return
    88  					}
    89  				}
    90  			}
    91  
    92  			select {
    93  			case <-retryTicker.C:
    94  			case <-doneCh:
    95  				return
    96  			}
    97  		}
    98  	}()
    99  
   100  	return keyCh
   101  }
   102  
   103  // sendItems - Reads items from the store and re-plays.
   104  func sendItems(target Target, keyCh <-chan Key, doneCh <-chan struct{}, logger logger) {
   105  	retryTicker := time.NewTicker(retryInterval)
   106  	defer retryTicker.Stop()
   107  
   108  	send := func(key Key) bool {
   109  		for {
   110  			err := target.SendFromStore(key)
   111  			if err == nil {
   112  				break
   113  			}
   114  
   115  			logger(
   116  				context.Background(),
   117  				fmt.Errorf("unable to send webhook log entry to '%s' err '%w'", target.Name(), err),
   118  				target.Name(),
   119  			)
   120  
   121  			select {
   122  			// Retrying after 3secs back-off
   123  			case <-retryTicker.C:
   124  			case <-doneCh:
   125  				return false
   126  			}
   127  		}
   128  		return true
   129  	}
   130  
   131  	for {
   132  		select {
   133  		case key, ok := <-keyCh:
   134  			if !ok {
   135  				return
   136  			}
   137  
   138  			if !send(key) {
   139  				return
   140  			}
   141  		case <-doneCh:
   142  			return
   143  		}
   144  	}
   145  }
   146  
   147  // StreamItems reads the keys from the store and replays the corresponding item to the target.
   148  func StreamItems[I any](store Store[I], target Target, doneCh <-chan struct{}, logger logger) {
   149  	go func() {
   150  		keyCh := replayItems(store, doneCh, logger, target.Name())
   151  		sendItems(target, keyCh, doneCh, logger)
   152  	}()
   153  }