github.com/pachyderm/pachyderm@v1.13.4/src/server/pkg/storage/chunk/client.go (about)

     1  package chunk
     2  
     3  import (
     4  	"context"
     5  	"io"
     6  	"io/ioutil"
     7  	"path"
     8  	"strings"
     9  	"time"
    10  
    11  	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
    12  	"github.com/pachyderm/pachyderm/src/server/pkg/obj"
    13  	"github.com/pachyderm/pachyderm/src/server/pkg/storage/track"
    14  )
    15  
    16  // Client allows manipulation of individual chunks, by maintaining consistency between
    17  // a tracker and an obj.Client.
    18  type Client struct {
    19  	objc    obj.Client
    20  	mdstore MetadataStore
    21  	tracker track.Tracker
    22  	renewer *track.Renewer
    23  	ttl     time.Duration
    24  }
    25  
    26  // NewClient returns a client which will write to objc, mdstore, and tracker.  Name is used
    27  // for the set of temporary objects
    28  func NewClient(objc obj.Client, mdstore MetadataStore, tr track.Tracker, name string) *Client {
    29  	var renewer *track.Renewer
    30  	if name != "" {
    31  		renewer = track.NewRenewer(tr, name, defaultChunkTTL)
    32  	}
    33  	c := &Client{
    34  		objc:    objc,
    35  		tracker: tr,
    36  		mdstore: mdstore,
    37  		renewer: renewer,
    38  		ttl:     defaultChunkTTL,
    39  	}
    40  	return c
    41  }
    42  
    43  // Create creates a chunk with data from r and Metadata md
    44  func (c *Client) Create(ctx context.Context, md Metadata, r io.Reader) (_ ID, retErr error) {
    45  	chunkData, err := ioutil.ReadAll(r)
    46  	if err != nil {
    47  		return nil, err
    48  	}
    49  	chunkID := Hash(chunkData)
    50  	var pointsTo []string
    51  	for _, cid := range md.PointsTo {
    52  		pointsTo = append(pointsTo, ObjectID(cid))
    53  	}
    54  	// TODO: retry on ErrTombstone
    55  	chunkOID := ObjectID(chunkID)
    56  	if err := c.tracker.CreateObject(ctx, chunkOID, pointsTo, c.ttl); err != nil {
    57  		if err != track.ErrObjectExists {
    58  			return nil, err
    59  		}
    60  	}
    61  	if err := c.renewer.Add(ctx, chunkOID); err != nil {
    62  		return nil, err
    63  	}
    64  	// at this point no one will be trying to delete the chunk, because there is an object pointing to it.
    65  	p := chunkPath(chunkID)
    66  	if c.objc.Exists(ctx, p) {
    67  		return chunkID, nil
    68  	}
    69  	if err := c.mdstore.Set(ctx, chunkID, md); err != nil && err != ErrMetadataExists {
    70  		return nil, err
    71  	}
    72  	objW, err := c.objc.Writer(ctx, p)
    73  	if err != nil {
    74  		return nil, err
    75  	}
    76  	defer func() {
    77  		if err := objW.Close(); retErr == nil {
    78  			retErr = err
    79  		}
    80  	}()
    81  	if _, err = objW.Write(chunkData); err != nil {
    82  		return nil, err
    83  	}
    84  	return chunkID, nil
    85  }
    86  
    87  // Get writes data for a chunk with ID chunkID to w.
    88  func (c *Client) Get(ctx context.Context, chunkID ID, w io.Writer) (retErr error) {
    89  	p := chunkPath(chunkID)
    90  	objR, err := c.objc.Reader(ctx, p, 0, 0)
    91  	if err != nil {
    92  		return err
    93  	}
    94  	defer func() {
    95  		if err := objR.Close(); retErr == nil {
    96  			retErr = err
    97  		}
    98  	}()
    99  	_, err = io.Copy(w, objR)
   100  	return err
   101  }
   102  
   103  // Close closes the client, stopping the background renewal of created objects
   104  func (c *Client) Close() error {
   105  	if c.renewer != nil {
   106  		return c.renewer.Close()
   107  	}
   108  	return nil
   109  }
   110  
   111  func chunkPath(chunkID ID) string {
   112  	if len(chunkID) == 0 {
   113  		panic("chunkID cannot be empty")
   114  	}
   115  	return path.Join(prefix, chunkID.HexString())
   116  }
   117  
   118  // ObjectID returns an object ID for use with a tracker
   119  func ObjectID(chunkID ID) string {
   120  	return prefix + "/" + chunkID.HexString()
   121  }
   122  
   123  var _ track.Deleter = &deleter{}
   124  
   125  type deleter struct {
   126  	mdstore MetadataStore
   127  	objc    obj.Client
   128  }
   129  
   130  func (d *deleter) Delete(ctx context.Context, id string) error {
   131  	if !strings.HasPrefix(id, prefix+"/") {
   132  		return errors.Errorf("cannot delete (%s)", id)
   133  	}
   134  	chunkID, err := IDFromHex(id[len(TrackerPrefix):])
   135  	if err != nil {
   136  		return err
   137  	}
   138  	if err := d.objc.Delete(ctx, chunkPath(chunkID)); err != nil {
   139  		return err
   140  	}
   141  	return d.mdstore.Delete(ctx, chunkID)
   142  }