github.com/treeverse/lakefs@v1.24.1-0.20240520134607-95648127bfb0/pkg/kv/kvparams/database.go (about)

     1  package kvparams
     2  
     3  import (
     4  	"fmt"
     5  	"net/http"
     6  	"time"
     7  
     8  	"github.com/mitchellh/go-homedir"
     9  	"github.com/treeverse/lakefs/pkg/config"
    10  )
    11  
    12  type Config struct {
    13  	Type     string
    14  	Postgres *Postgres
    15  	DynamoDB *DynamoDB
    16  	Local    *Local
    17  	CosmosDB *CosmosDB
    18  }
    19  
    20  type Local struct {
    21  	// Path - Local directory path to store the DB files
    22  	Path string
    23  	// SyncWrites - Sync ensures data written to disk on each writing instead of mem cache
    24  	SyncWrites bool
    25  	// PrefetchSize - Number of elements to prefetch while iterating
    26  	PrefetchSize int
    27  	// EnableLogging - Enable store and badger (trace only) logging
    28  	EnableLogging bool
    29  }
    30  
    31  type Postgres struct {
    32  	ConnectionString      string
    33  	MaxOpenConnections    int32
    34  	MaxIdleConnections    int32
    35  	ConnectionMaxLifetime time.Duration
    36  	ScanPageSize          int
    37  	Metrics               bool
    38  }
    39  
    40  type DynamoDB struct {
    41  	// The name of the DynamoDB table to be used as KV
    42  	TableName string
    43  
    44  	// Maximal number of items per page during scan operation
    45  	ScanLimit int64
    46  
    47  	// Specifies the maximum number attempts to make on a request.
    48  	MaxAttempts int
    49  
    50  	// The endpoint URL of the DynamoDB endpoint
    51  	// Can be used to redirect to DynamoDB on AWS, local docker etc.
    52  	Endpoint string
    53  
    54  	// AWS connection details - region, profile and credentials
    55  	// This will override any such details that are already exist in the system
    56  	// While in general, AWS region and credentials are configured in the system for AWS usage,
    57  	// these can be used to specify fake values, that cna be used to connect to local DynamoDB,
    58  	// in case there are no credentials configured in the system
    59  	// This is a client requirement as described in section 4 in
    60  	// https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html
    61  	AwsRegion             string
    62  	AwsProfile            string
    63  	AwsAccessKeyID        string
    64  	AwsSecretAccessKey    string
    65  	HealthCheckInterval   time.Duration
    66  	MaxConnectionsPerHost int
    67  }
    68  
    69  type CosmosDB struct {
    70  	Key        string
    71  	Endpoint   string
    72  	Database   string
    73  	Container  string
    74  	Throughput int32
    75  	Autoscale  bool
    76  	// These values should only be set to false for testing purposes using the CosmosDB emulator
    77  	Client            *http.Client
    78  	StrongConsistency bool
    79  }
    80  
    81  func NewConfig(cfg *config.Config) (Config, error) {
    82  	p := Config{
    83  		Type: cfg.Database.Type,
    84  	}
    85  	if cfg.Database.Local != nil {
    86  		localPath, err := homedir.Expand(cfg.Database.Local.Path)
    87  		if err != nil {
    88  			return Config{}, fmt.Errorf("parse database local path '%s': %w", cfg.Database.Local.Path, err)
    89  		}
    90  		p.Local = &Local{
    91  			Path:         localPath,
    92  			PrefetchSize: cfg.Database.Local.PrefetchSize,
    93  		}
    94  	}
    95  
    96  	if cfg.Database.Postgres != nil {
    97  		p.Postgres = &Postgres{
    98  			ConnectionString:      cfg.Database.Postgres.ConnectionString.SecureValue(),
    99  			MaxIdleConnections:    cfg.Database.Postgres.MaxIdleConnections,
   100  			MaxOpenConnections:    cfg.Database.Postgres.MaxOpenConnections,
   101  			ConnectionMaxLifetime: cfg.Database.Postgres.ConnectionMaxLifetime,
   102  		}
   103  	}
   104  
   105  	if cfg.Database.DynamoDB != nil {
   106  		p.DynamoDB = &DynamoDB{
   107  			TableName:             cfg.Database.DynamoDB.TableName,
   108  			ScanLimit:             cfg.Database.DynamoDB.ScanLimit,
   109  			Endpoint:              cfg.Database.DynamoDB.Endpoint,
   110  			AwsRegion:             cfg.Database.DynamoDB.AwsRegion,
   111  			AwsProfile:            cfg.Database.DynamoDB.AwsProfile,
   112  			AwsAccessKeyID:        cfg.Database.DynamoDB.AwsAccessKeyID.SecureValue(),
   113  			AwsSecretAccessKey:    cfg.Database.DynamoDB.AwsSecretAccessKey.SecureValue(),
   114  			HealthCheckInterval:   cfg.Database.DynamoDB.HealthCheckInterval,
   115  			MaxAttempts:           cfg.Database.DynamoDB.MaxAttempts,
   116  			MaxConnectionsPerHost: cfg.Database.DynamoDB.MaxConnections,
   117  		}
   118  	}
   119  
   120  	if cfg.Database.CosmosDB != nil {
   121  		if cfg.Database.CosmosDB.Autoscale && cfg.Database.CosmosDB.Throughput == 0 {
   122  			return Config{}, fmt.Errorf("enabling autoscale requires setting the throughput param: %w", config.ErrBadConfiguration)
   123  		}
   124  		p.CosmosDB = &CosmosDB{
   125  			Key:               cfg.Database.CosmosDB.Key.SecureValue(),
   126  			Endpoint:          cfg.Database.CosmosDB.Endpoint,
   127  			Database:          cfg.Database.CosmosDB.Database,
   128  			Container:         cfg.Database.CosmosDB.Container,
   129  			Throughput:        cfg.Database.CosmosDB.Throughput,
   130  			Autoscale:         cfg.Database.CosmosDB.Autoscale,
   131  			StrongConsistency: true,
   132  		}
   133  	}
   134  
   135  	return p, nil
   136  }