github.com/ethersphere/bee/v2@v2.2.0/pkg/storer/migration/refCntSize.go (about)

     1  // Copyright 2023 The Swarm Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package migration
     6  
     7  import (
     8  	"context"
     9  	"encoding/binary"
    10  	"errors"
    11  	"os"
    12  
    13  	"github.com/ethersphere/bee/v2/pkg/log"
    14  	"github.com/ethersphere/bee/v2/pkg/sharky"
    15  	"github.com/ethersphere/bee/v2/pkg/storage"
    16  	"github.com/ethersphere/bee/v2/pkg/storage/storageutil"
    17  	"github.com/ethersphere/bee/v2/pkg/storer/internal/chunkstore"
    18  	"github.com/ethersphere/bee/v2/pkg/swarm"
    19  )
    20  
    21  const oldRretrievalIndexItemSize = swarm.HashSize + 8 + sharky.LocationSize + 1
    22  
    23  var _ storage.Item = (*OldRetrievalIndexItem)(nil)
    24  
    25  var (
    26  	// errMarshalInvalidRetrievalIndexAddress is returned if the RetrievalIndexItem address is zero during marshaling.
    27  	errMarshalInvalidRetrievalIndexAddress = errors.New("marshal RetrievalIndexItem: address is zero")
    28  	// errMarshalInvalidRetrievalIndexLocation is returned if the RetrievalIndexItem location is invalid during marshaling.
    29  	errMarshalInvalidRetrievalIndexLocation = errors.New("marshal RetrievalIndexItem: location is invalid")
    30  	// errUnmarshalInvalidRetrievalIndexSize is returned during unmarshaling if the passed buffer is not the expected size.
    31  	errUnmarshalInvalidRetrievalIndexSize = errors.New("unmarshal RetrievalIndexItem: invalid size")
    32  	// errUnmarshalInvalidRetrievalIndexLocationBytes is returned during unmarshaling if the location buffer is invalid.
    33  	errUnmarshalInvalidRetrievalIndexLocationBytes = errors.New("unmarshal RetrievalIndexItem: invalid location bytes")
    34  )
    35  
    36  // OldRetrievalIndexItem is the index which gives us the sharky location from the swarm.Address.
    37  // The RefCnt stores the reference of each time a Put operation is issued on this Address.
    38  type OldRetrievalIndexItem struct {
    39  	Address   swarm.Address
    40  	Timestamp uint64
    41  	Location  sharky.Location
    42  	RefCnt    uint8
    43  }
    44  
    45  func (r *OldRetrievalIndexItem) ID() string { return r.Address.ByteString() }
    46  
    47  func (OldRetrievalIndexItem) Namespace() string { return "retrievalIdx" }
    48  
    49  // Stored in bytes as:
    50  // |--Address(32)--|--Timestamp(8)--|--Location(7)--|--RefCnt(1)--|
    51  func (r *OldRetrievalIndexItem) Marshal() ([]byte, error) {
    52  	if r.Address.IsZero() {
    53  		return nil, errMarshalInvalidRetrievalIndexAddress
    54  	}
    55  
    56  	locBuf, err := r.Location.MarshalBinary()
    57  	if err != nil {
    58  		return nil, errMarshalInvalidRetrievalIndexLocation
    59  	}
    60  
    61  	buf := make([]byte, oldRretrievalIndexItemSize)
    62  	copy(buf, r.Address.Bytes())
    63  	binary.LittleEndian.PutUint64(buf[swarm.HashSize:], r.Timestamp)
    64  	copy(buf[swarm.HashSize+8:], locBuf)
    65  	buf[oldRretrievalIndexItemSize-1] = r.RefCnt
    66  	return buf, nil
    67  }
    68  
    69  func (r *OldRetrievalIndexItem) Unmarshal(buf []byte) error {
    70  	if len(buf) != oldRretrievalIndexItemSize {
    71  		return errUnmarshalInvalidRetrievalIndexSize
    72  	}
    73  
    74  	loc := new(sharky.Location)
    75  	if err := loc.UnmarshalBinary(buf[swarm.HashSize+8:]); err != nil {
    76  		return errUnmarshalInvalidRetrievalIndexLocationBytes
    77  	}
    78  
    79  	ni := new(OldRetrievalIndexItem)
    80  	ni.Address = swarm.NewAddress(append(make([]byte, 0, swarm.HashSize), buf[:swarm.HashSize]...))
    81  	ni.Timestamp = binary.LittleEndian.Uint64(buf[swarm.HashSize:])
    82  	ni.Location = *loc
    83  	ni.RefCnt = buf[oldRretrievalIndexItemSize-1]
    84  	*r = *ni
    85  	return nil
    86  }
    87  
    88  func (r *OldRetrievalIndexItem) Clone() storage.Item {
    89  	if r == nil {
    90  		return nil
    91  	}
    92  	return &OldRetrievalIndexItem{
    93  		Address:   r.Address.Clone(),
    94  		Timestamp: r.Timestamp,
    95  		Location:  r.Location,
    96  		RefCnt:    r.RefCnt,
    97  	}
    98  }
    99  
   100  func (r OldRetrievalIndexItem) String() string {
   101  	return storageutil.JoinFields(r.Namespace(), r.ID())
   102  }
   103  
   104  func RefCountSizeInc(s storage.BatchStore) func() error {
   105  	return func() error {
   106  		logger := log.NewLogger("migration-RefCountSizeInc", log.WithSink(os.Stdout))
   107  
   108  		logger.Info("starting migration of replacing chunkstore items to increase refCnt capacity")
   109  
   110  		var itemsToDelete []*OldRetrievalIndexItem
   111  
   112  		err := s.Iterate(
   113  			storage.Query{
   114  				Factory: func() storage.Item { return &OldRetrievalIndexItem{} },
   115  			},
   116  			func(res storage.Result) (bool, error) {
   117  				item := res.Entry.(*OldRetrievalIndexItem)
   118  				itemsToDelete = append(itemsToDelete, item)
   119  				return false, nil
   120  			},
   121  		)
   122  		if err != nil {
   123  			return err
   124  		}
   125  
   126  		for i := 0; i < len(itemsToDelete); i += 10000 {
   127  			end := i + 10000
   128  			if end > len(itemsToDelete) {
   129  				end = len(itemsToDelete)
   130  			}
   131  
   132  			b := s.Batch(context.Background())
   133  			for _, item := range itemsToDelete[i:end] {
   134  
   135  				//create new
   136  				err = b.Put(&chunkstore.RetrievalIndexItem{
   137  					Address:   item.Address,
   138  					Timestamp: item.Timestamp,
   139  					Location:  item.Location,
   140  					RefCnt:    uint32(item.RefCnt),
   141  				})
   142  				if err != nil {
   143  					return err
   144  				}
   145  			}
   146  
   147  			err = b.Commit()
   148  			if err != nil {
   149  				return err
   150  			}
   151  		}
   152  
   153  		logger.Info("migration complete")
   154  
   155  		return nil
   156  	}
   157  }