github.com/2lambda123/git-lfs@v2.5.2+incompatible/tq/transfer.go (about) 1 // Package transfer collects together adapters for uploading and downloading LFS content 2 // NOTE: Subject to change, do not rely on this package from outside git-lfs source 3 package tq 4 5 import ( 6 "fmt" 7 "time" 8 9 "github.com/git-lfs/git-lfs/errors" 10 "github.com/git-lfs/git-lfs/lfsapi" 11 "github.com/git-lfs/git-lfs/tools" 12 ) 13 14 type Direction int 15 16 const ( 17 Upload = Direction(iota) 18 Download = Direction(iota) 19 Checkout = Direction(iota) 20 ) 21 22 // Verb returns a string containing the verb form of the receiving action. 23 func (d Direction) Verb() string { 24 switch d { 25 case Checkout: 26 return "Checking out" 27 case Download: 28 return "Downloading" 29 case Upload: 30 return "Uploading" 31 default: 32 return "<unknown>" 33 } 34 } 35 36 func (d Direction) String() string { 37 switch d { 38 case Checkout: 39 return "checkout" 40 case Download: 41 return "download" 42 case Upload: 43 return "upload" 44 default: 45 return "<unknown>" 46 } 47 } 48 49 type Transfer struct { 50 Name string `json:"name,omitempty"` 51 Oid string `json:"oid,omitempty"` 52 Size int64 `json:"size"` 53 Authenticated bool `json:"authenticated,omitempty"` 54 Actions ActionSet `json:"actions,omitempty"` 55 Links ActionSet `json:"_links,omitempty"` 56 Error *ObjectError `json:"error,omitempty"` 57 Path string `json:"path,omitempty"` 58 } 59 60 func (t *Transfer) Rel(name string) (*Action, error) { 61 a, err := t.Actions.Get(name) 62 if a != nil || err != nil { 63 return a, err 64 } 65 66 if t.Links != nil { 67 a, err := t.Links.Get(name) 68 if a != nil || err != nil { 69 return a, err 70 } 71 } 72 73 return nil, nil 74 } 75 76 type ObjectError struct { 77 Code int `json:"code"` 78 Message string `json:"message"` 79 } 80 81 func (e *ObjectError) Error() string { 82 return fmt.Sprintf("[%d] %s", e.Code, e.Message) 83 } 84 85 // newTransfer returns a copy of the given Transfer, with the name and path 86 // values set. 87 func newTransfer(tr *Transfer, name string, path string) *Transfer { 88 t := &Transfer{ 89 Name: name, 90 Path: path, 91 Oid: tr.Oid, 92 Size: tr.Size, 93 Authenticated: tr.Authenticated, 94 Actions: make(ActionSet), 95 } 96 97 if tr.Error != nil { 98 t.Error = &ObjectError{ 99 Code: tr.Error.Code, 100 Message: tr.Error.Message, 101 } 102 } 103 104 for rel, action := range tr.Actions { 105 t.Actions[rel] = &Action{ 106 Href: action.Href, 107 Header: action.Header, 108 ExpiresAt: action.ExpiresAt, 109 ExpiresIn: action.ExpiresIn, 110 createdAt: action.createdAt, 111 } 112 } 113 114 if tr.Links != nil { 115 t.Links = make(ActionSet) 116 117 for rel, link := range tr.Links { 118 t.Links[rel] = &Action{ 119 Href: link.Href, 120 Header: link.Header, 121 ExpiresAt: link.ExpiresAt, 122 ExpiresIn: link.ExpiresIn, 123 createdAt: link.createdAt, 124 } 125 } 126 } 127 128 return t 129 } 130 131 type Action struct { 132 Href string `json:"href"` 133 Header map[string]string `json:"header,omitempty"` 134 ExpiresAt time.Time `json:"expires_at,omitempty"` 135 ExpiresIn int `json:"expires_in,omitempty"` 136 137 createdAt time.Time 138 } 139 140 func (a *Action) IsExpiredWithin(d time.Duration) (time.Time, bool) { 141 return tools.IsExpiredAtOrIn(a.createdAt, d, a.ExpiresAt, time.Duration(a.ExpiresIn)*time.Second) 142 } 143 144 type ActionSet map[string]*Action 145 146 const ( 147 // objectExpirationToTransfer is the duration we expect to have passed 148 // from the time that the object's expires_at (or expires_in) property 149 // is checked to when the transfer is executed. 150 objectExpirationToTransfer = 5 * time.Second 151 ) 152 153 func (as ActionSet) Get(rel string) (*Action, error) { 154 a, ok := as[rel] 155 if !ok { 156 return nil, nil 157 } 158 159 if at, expired := a.IsExpiredWithin(objectExpirationToTransfer); expired { 160 return nil, errors.NewRetriableError(&ActionExpiredErr{Rel: rel, At: at}) 161 } 162 163 return a, nil 164 } 165 166 type ActionExpiredErr struct { 167 Rel string 168 At time.Time 169 } 170 171 func (e ActionExpiredErr) Error() string { 172 return fmt.Sprintf("tq: action %q expires at %s", 173 e.Rel, e.At.In(time.Local).Format(time.RFC822)) 174 } 175 176 func IsActionExpiredError(err error) bool { 177 if _, ok := err.(*ActionExpiredErr); ok { 178 return true 179 } 180 return false 181 } 182 183 // NewAdapterFunc creates new instances of Adapter. Code that wishes 184 // to provide new Adapter instances should pass an implementation of this 185 // function to RegisterNewTransferAdapterFunc() on a *Manifest. 186 // name and dir are to provide context if one func implements many instances 187 type NewAdapterFunc func(name string, dir Direction) Adapter 188 189 type ProgressCallback func(name string, totalSize, readSoFar int64, readSinceLast int) error 190 191 type AdapterConfig interface { 192 APIClient() *lfsapi.Client 193 ConcurrentTransfers() int 194 Remote() string 195 } 196 197 type adapterConfig struct { 198 apiClient *lfsapi.Client 199 concurrentTransfers int 200 remote string 201 } 202 203 func (c *adapterConfig) ConcurrentTransfers() int { 204 return c.concurrentTransfers 205 } 206 207 func (c *adapterConfig) APIClient() *lfsapi.Client { 208 return c.apiClient 209 } 210 211 func (c *adapterConfig) Remote() string { 212 return c.remote 213 } 214 215 // Adapter is implemented by types which can upload and/or download LFS 216 // file content to a remote store. Each Adapter accepts one or more requests 217 // which it may schedule and parallelise in whatever way it chooses, clients of 218 // this interface will receive notifications of progress and completion asynchronously. 219 // TransferAdapters support transfers in one direction; if an implementation 220 // provides support for upload and download, it should be instantiated twice, 221 // advertising support for each direction separately. 222 // Note that Adapter only implements the actual upload/download of content 223 // itself; organising the wider process including calling the API to get URLs, 224 // handling progress reporting and retries is the job of the core TransferQueue. 225 // This is so that the orchestration remains core & standard but Adapter 226 // can be changed to physically transfer to different hosts with less code. 227 type Adapter interface { 228 // Name returns the name of this adapter, which is the same for all instances 229 // of this type of adapter 230 Name() string 231 // Direction returns whether this instance is an upload or download instance 232 // Adapter instances can only be one or the other, although the same 233 // type may be instantiated for each direction 234 Direction() Direction 235 // Begin a new batch of uploads or downloads. Call this first, followed by 236 // one or more Add calls. maxConcurrency controls the number of transfers 237 // that may be done at once. The passed in callback will receive updates on 238 // progress. Either argument may be nil if not required by the client. 239 Begin(cfg AdapterConfig, cb ProgressCallback) error 240 // Add queues a download/upload, which will complete asynchronously and 241 // notify the callbacks given to Begin() 242 Add(transfers ...*Transfer) (results <-chan TransferResult) 243 // Indicate that all transfers have been scheduled and resources can be released 244 // once the queued items have completed. 245 // This call blocks until all items have been processed 246 End() 247 // ClearTempStorage clears any temporary files, such as unfinished downloads that 248 // would otherwise be resumed 249 ClearTempStorage() error 250 } 251 252 // Result of a transfer returned through CompletionChannel() 253 type TransferResult struct { 254 Transfer *Transfer 255 // This will be non-nil if there was an error transferring this item 256 Error error 257 }