code.gitea.io/gitea@v1.22.3/modules/queue/base_redis.go (about)

     1  // Copyright 2023 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  package queue
     5  
     6  import (
     7  	"context"
     8  	"sync"
     9  	"time"
    10  
    11  	"code.gitea.io/gitea/modules/graceful"
    12  	"code.gitea.io/gitea/modules/log"
    13  	"code.gitea.io/gitea/modules/nosql"
    14  
    15  	"github.com/redis/go-redis/v9"
    16  )
    17  
    18  type baseRedis struct {
    19  	client   redis.UniversalClient
    20  	isUnique bool
    21  	cfg      *BaseConfig
    22  
    23  	mu sync.Mutex // the old implementation is not thread-safe, the queue operation and set operation should be protected together
    24  }
    25  
    26  var _ baseQueue = (*baseRedis)(nil)
    27  
    28  func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) {
    29  	client := nosql.GetManager().GetRedisClient(cfg.ConnStr)
    30  
    31  	var err error
    32  	for i := 0; i < 10; i++ {
    33  		err = client.Ping(graceful.GetManager().ShutdownContext()).Err()
    34  		if err == nil {
    35  			break
    36  		}
    37  		log.Warn("Redis is not ready, waiting for 1 second to retry: %v", err)
    38  		time.Sleep(time.Second)
    39  	}
    40  	if err != nil {
    41  		return nil, err
    42  	}
    43  
    44  	return &baseRedis{cfg: cfg, client: client, isUnique: unique}, nil
    45  }
    46  
    47  func newBaseRedisSimple(cfg *BaseConfig) (baseQueue, error) {
    48  	return newBaseRedisGeneric(cfg, false)
    49  }
    50  
    51  func newBaseRedisUnique(cfg *BaseConfig) (baseQueue, error) {
    52  	return newBaseRedisGeneric(cfg, true)
    53  }
    54  
    55  func (q *baseRedis) PushItem(ctx context.Context, data []byte) error {
    56  	return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) {
    57  		q.mu.Lock()
    58  		defer q.mu.Unlock()
    59  
    60  		cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result()
    61  		if err != nil {
    62  			return false, err
    63  		}
    64  		if int(cnt) >= q.cfg.Length {
    65  			return true, nil
    66  		}
    67  
    68  		if q.isUnique {
    69  			added, err := q.client.SAdd(ctx, q.cfg.SetFullName, data).Result()
    70  			if err != nil {
    71  				return false, err
    72  			}
    73  			if added == 0 {
    74  				return false, ErrAlreadyInQueue
    75  			}
    76  		}
    77  		return false, q.client.RPush(ctx, q.cfg.QueueFullName, data).Err()
    78  	})
    79  }
    80  
    81  func (q *baseRedis) PopItem(ctx context.Context) ([]byte, error) {
    82  	return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) {
    83  		q.mu.Lock()
    84  		defer q.mu.Unlock()
    85  
    86  		data, err = q.client.LPop(ctx, q.cfg.QueueFullName).Bytes()
    87  		if err == redis.Nil {
    88  			return true, nil, nil
    89  		}
    90  		if err != nil {
    91  			return true, nil, nil
    92  		}
    93  		if q.isUnique {
    94  			// the data has been popped, even if there is any error we can't do anything
    95  			_ = q.client.SRem(ctx, q.cfg.SetFullName, data).Err()
    96  		}
    97  		return false, data, err
    98  	})
    99  }
   100  
   101  func (q *baseRedis) HasItem(ctx context.Context, data []byte) (bool, error) {
   102  	q.mu.Lock()
   103  	defer q.mu.Unlock()
   104  	if !q.isUnique {
   105  		return false, nil
   106  	}
   107  	return q.client.SIsMember(ctx, q.cfg.SetFullName, data).Result()
   108  }
   109  
   110  func (q *baseRedis) Len(ctx context.Context) (int, error) {
   111  	q.mu.Lock()
   112  	defer q.mu.Unlock()
   113  	cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result()
   114  	return int(cnt), err
   115  }
   116  
   117  func (q *baseRedis) Close() error {
   118  	q.mu.Lock()
   119  	defer q.mu.Unlock()
   120  	return q.client.Close()
   121  }
   122  
   123  func (q *baseRedis) RemoveAll(ctx context.Context) error {
   124  	q.mu.Lock()
   125  	defer q.mu.Unlock()
   126  
   127  	c1 := q.client.Del(ctx, q.cfg.QueueFullName)
   128  	// the "set" must be cleared after the "list" because there is no transaction.
   129  	// it's better to have duplicate items than losing items.
   130  	c2 := q.client.Del(ctx, q.cfg.SetFullName)
   131  	if c1.Err() != nil {
   132  		return c1.Err()
   133  	}
   134  	if c2.Err() != nil {
   135  		return c2.Err()
   136  	}
   137  	return nil // actually, checking errors doesn't make sense here because the state could be out-of-sync
   138  }