code.gitea.io/gitea@v1.21.7/models/dbfs/dbfs.go (about)

     1  // Copyright 2022 The Gitea Authors. All rights reserved.
     2  // SPDX-License-Identifier: MIT
     3  
     4  package dbfs
     5  
     6  import (
     7  	"context"
     8  	"io/fs"
     9  	"os"
    10  	"path"
    11  	"time"
    12  
    13  	"code.gitea.io/gitea/models/db"
    14  )
    15  
    16  /*
    17  The reasons behind the DBFS (database-filesystem) package:
    18  When a Gitea action is running, the Gitea action server should collect and store all the logs.
    19  
    20  The requirements are:
    21  * The running logs must be stored across the cluster if the Gitea servers are deployed as a cluster.
    22  * The logs will be archived to Object Storage (S3/MinIO, etc.) after a period of time.
    23  * The Gitea action UI should be able to render the running logs and the archived logs.
    24  
    25  Some possible solutions for the running logs:
    26  * [Not ideal] Using local temp file: it can not be shared across the cluster.
    27  * [Not ideal] Using shared file in the filesystem of git repository: although at the moment, the Gitea cluster's
    28  	git repositories must be stored in a shared filesystem, in the future, Gitea may need a dedicated Git Service Server
    29  	to decouple the shared filesystem. Then the action logs will become a blocker.
    30  * [Not ideal] Record the logs in a database table line by line: it has a couple of problems:
    31  	- It's difficult to make multiple increasing sequence (log line number) for different databases.
    32  	- The database table will have a lot of rows and be affected by the big-table performance problem.
    33  	- It's difficult to load logs by using the same interface as other storages.
    34    - It's difficult to calculate the size of the logs.
    35  
    36  The DBFS solution:
    37  * It can be used in a cluster.
    38  * It can share the same interface (Read/Write/Seek) as other storages.
    39  * It's very friendly to database because it only needs to store much fewer rows than the log-line solution.
    40  * In the future, when Gitea action needs to limit the log size (other CI/CD services also do so), it's easier to calculate the log file size.
    41  * Even sometimes the UI needs to render the tailing lines, the tailing lines can be found be counting the "\n" from the end of the file by seek.
    42    The seeking and finding is not the fastest way, but it's still acceptable and won't affect the performance too much.
    43  */
    44  
    45  type dbfsMeta struct {
    46  	ID              int64  `xorm:"pk autoincr"`
    47  	FullPath        string `xorm:"VARCHAR(500) UNIQUE NOT NULL"`
    48  	BlockSize       int64  `xorm:"BIGINT NOT NULL"`
    49  	FileSize        int64  `xorm:"BIGINT NOT NULL"`
    50  	CreateTimestamp int64  `xorm:"BIGINT NOT NULL"`
    51  	ModifyTimestamp int64  `xorm:"BIGINT NOT NULL"`
    52  }
    53  
    54  type dbfsData struct {
    55  	ID         int64  `xorm:"pk autoincr"`
    56  	Revision   int64  `xorm:"BIGINT NOT NULL"`
    57  	MetaID     int64  `xorm:"BIGINT index(meta_offset) NOT NULL"`
    58  	BlobOffset int64  `xorm:"BIGINT index(meta_offset) NOT NULL"`
    59  	BlobSize   int64  `xorm:"BIGINT NOT NULL"`
    60  	BlobData   []byte `xorm:"BLOB NOT NULL"`
    61  }
    62  
    63  func init() {
    64  	db.RegisterModel(new(dbfsMeta))
    65  	db.RegisterModel(new(dbfsData))
    66  }
    67  
    68  func OpenFile(ctx context.Context, name string, flag int) (File, error) {
    69  	f, err := newDbFile(ctx, name)
    70  	if err != nil {
    71  		return nil, err
    72  	}
    73  	err = f.open(flag)
    74  	if err != nil {
    75  		_ = f.Close()
    76  		return nil, err
    77  	}
    78  	return f, nil
    79  }
    80  
    81  func Open(ctx context.Context, name string) (File, error) {
    82  	return OpenFile(ctx, name, os.O_RDONLY)
    83  }
    84  
    85  func Create(ctx context.Context, name string) (File, error) {
    86  	return OpenFile(ctx, name, os.O_RDWR|os.O_CREATE|os.O_TRUNC)
    87  }
    88  
    89  func Rename(ctx context.Context, oldPath, newPath string) error {
    90  	f, err := newDbFile(ctx, oldPath)
    91  	if err != nil {
    92  		return err
    93  	}
    94  	defer f.Close()
    95  	return f.renameTo(newPath)
    96  }
    97  
    98  func Remove(ctx context.Context, name string) error {
    99  	f, err := newDbFile(ctx, name)
   100  	if err != nil {
   101  		return err
   102  	}
   103  	defer f.Close()
   104  	return f.delete()
   105  }
   106  
   107  var _ fs.FileInfo = (*dbfsMeta)(nil)
   108  
   109  func (m *dbfsMeta) Name() string {
   110  	return path.Base(m.FullPath)
   111  }
   112  
   113  func (m *dbfsMeta) Size() int64 {
   114  	return m.FileSize
   115  }
   116  
   117  func (m *dbfsMeta) Mode() fs.FileMode {
   118  	return os.ModePerm
   119  }
   120  
   121  func (m *dbfsMeta) ModTime() time.Time {
   122  	return fileTimestampToTime(m.ModifyTimestamp)
   123  }
   124  
   125  func (m *dbfsMeta) IsDir() bool {
   126  	return false
   127  }
   128  
   129  func (m *dbfsMeta) Sys() any {
   130  	return nil
   131  }