github.com/2lambda123/git-lfs@v2.5.2+incompatible/tq/meter.go (about) 1 package tq 2 3 import ( 4 "fmt" 5 "math" 6 "os" 7 "path/filepath" 8 "sync" 9 "sync/atomic" 10 "time" 11 12 "github.com/git-lfs/git-lfs/tasklog" 13 "github.com/git-lfs/git-lfs/tools" 14 "github.com/git-lfs/git-lfs/tools/humanize" 15 ) 16 17 // Meter provides a progress bar type output for the TransferQueue. It 18 // is given an estimated file count and size up front and tracks the number of 19 // files and bytes transferred as well as the number of files and bytes that 20 // get skipped because the transfer is unnecessary. 21 type Meter struct { 22 finishedFiles int64 // int64s must come first for struct alignment 23 transferringFiles int64 24 estimatedBytes int64 25 lastBytes int64 26 currentBytes int64 27 sampleCount uint64 28 avgBytes float64 29 lastAvg time.Time 30 estimatedFiles int32 31 paused uint32 32 fileIndex map[string]int64 // Maps a file name to its transfer number 33 fileIndexMutex *sync.Mutex 34 updates chan *tasklog.Update 35 36 DryRun bool 37 Logger *tools.SyncWriter 38 Direction Direction 39 } 40 41 type env interface { 42 Get(key string) (val string, ok bool) 43 } 44 45 func (m *Meter) LoggerFromEnv(os env) *tools.SyncWriter { 46 name, _ := os.Get("GIT_LFS_PROGRESS") 47 if len(name) < 1 { 48 return nil 49 } 50 return m.LoggerToFile(name) 51 } 52 53 func (m *Meter) LoggerToFile(name string) *tools.SyncWriter { 54 printErr := func(err string) { 55 fmt.Fprintf(os.Stderr, "Error creating progress logger: %s\n", err) 56 } 57 58 if !filepath.IsAbs(name) { 59 printErr("GIT_LFS_PROGRESS must be an absolute path") 60 return nil 61 } 62 63 if err := os.MkdirAll(filepath.Dir(name), 0755); err != nil { 64 printErr(err.Error()) 65 return nil 66 } 67 68 file, err := os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) 69 if err != nil { 70 printErr(err.Error()) 71 return nil 72 } 73 74 return tools.NewSyncWriter(file) 75 } 76 77 // NewMeter creates a new Meter. 78 func NewMeter() *Meter { 79 m := &Meter{ 80 fileIndex: make(map[string]int64), 81 fileIndexMutex: &sync.Mutex{}, 82 updates: make(chan *tasklog.Update), 83 } 84 85 return m 86 } 87 88 // Start begins sending status updates to the optional log file, and stdout. 89 func (m *Meter) Start() { 90 if m == nil { 91 return 92 } 93 atomic.StoreUint32(&m.paused, 0) 94 } 95 96 // Pause stops sending status updates temporarily, until Start() is called again. 97 func (m *Meter) Pause() { 98 if m == nil { 99 return 100 } 101 atomic.StoreUint32(&m.paused, 1) 102 } 103 104 // Add tells the progress meter that a single file of the given size will 105 // possibly be transferred. If a file doesn't need to be transferred for some 106 // reason, be sure to call Skip(int64) with the same size. 107 func (m *Meter) Add(size int64) { 108 if m == nil { 109 return 110 } 111 112 defer m.update(false) 113 atomic.AddInt32(&m.estimatedFiles, 1) 114 atomic.AddInt64(&m.estimatedBytes, size) 115 } 116 117 // Skip tells the progress meter that a file of size `size` is being skipped 118 // because the transfer is unnecessary. 119 func (m *Meter) Skip(size int64) { 120 if m == nil { 121 return 122 } 123 124 defer m.update(false) 125 atomic.AddInt64(&m.finishedFiles, 1) 126 atomic.AddInt64(&m.currentBytes, size) 127 } 128 129 // StartTransfer tells the progress meter that a transferring file is being 130 // added to the TransferQueue. 131 func (m *Meter) StartTransfer(name string) { 132 if m == nil { 133 return 134 } 135 136 defer m.update(false) 137 idx := atomic.AddInt64(&m.transferringFiles, 1) 138 m.fileIndexMutex.Lock() 139 m.fileIndex[name] = idx 140 m.fileIndexMutex.Unlock() 141 } 142 143 // TransferBytes increments the number of bytes transferred 144 func (m *Meter) TransferBytes(direction, name string, read, total int64, current int) { 145 if m == nil { 146 return 147 } 148 149 defer m.update(false) 150 151 now := time.Now() 152 since := now.Sub(m.lastAvg) 153 atomic.AddInt64(&m.currentBytes, int64(current)) 154 atomic.AddInt64(&m.lastBytes, int64(current)) 155 156 if since > time.Second { 157 m.lastAvg = now 158 159 bps := float64(m.lastBytes) / since.Seconds() 160 161 m.avgBytes = (m.avgBytes*float64(m.sampleCount) + bps) / (float64(m.sampleCount) + 1.0) 162 163 atomic.StoreInt64(&m.lastBytes, 0) 164 atomic.AddUint64(&m.sampleCount, 1) 165 } 166 167 m.logBytes(direction, name, read, total) 168 } 169 170 // FinishTransfer increments the finished transfer count 171 func (m *Meter) FinishTransfer(name string) { 172 if m == nil { 173 return 174 } 175 176 defer m.update(false) 177 atomic.AddInt64(&m.finishedFiles, 1) 178 m.fileIndexMutex.Lock() 179 delete(m.fileIndex, name) 180 m.fileIndexMutex.Unlock() 181 } 182 183 // Flush sends the latest progress update, while leaving the meter active. 184 func (m *Meter) Flush() { 185 if m == nil { 186 return 187 } 188 189 m.update(true) 190 } 191 192 // Finish shuts down the Meter. 193 func (m *Meter) Finish() { 194 if m == nil { 195 return 196 } 197 198 m.update(false) 199 close(m.updates) 200 } 201 202 func (m *Meter) Updates() <-chan *tasklog.Update { 203 if m == nil { 204 return nil 205 } 206 return m.updates 207 } 208 209 func (m *Meter) Throttled() bool { 210 return true 211 } 212 213 func (m *Meter) update(force bool) { 214 if m.skipUpdate() { 215 return 216 } 217 218 m.updates <- &tasklog.Update{ 219 S: m.str(), 220 At: time.Now(), 221 Force: force, 222 } 223 } 224 225 func (m *Meter) skipUpdate() bool { 226 return m.DryRun || 227 m.estimatedFiles == 0 || 228 atomic.LoadUint32(&m.paused) == 1 229 } 230 231 func (m *Meter) str() string { 232 // (Uploading|Downloading) LFS objects: 100% (10/10) 100 MiB | 10 MiB/s 233 percentage := 100 * float64(m.finishedFiles) / float64(m.estimatedFiles) 234 235 return fmt.Sprintf("%s LFS objects: %3.f%% (%d/%d), %s | %s", 236 m.Direction.Verb(), 237 percentage, 238 m.finishedFiles, m.estimatedFiles, 239 humanize.FormatBytes(clamp(m.currentBytes)), 240 humanize.FormatByteRate(clampf(m.avgBytes), time.Second)) 241 } 242 243 // clamp clamps the given "x" within the acceptable domain of the uint64 integer 244 // type, so as to prevent over- and underflow. 245 func clamp(x int64) uint64 { 246 if x < 0 { 247 return 0 248 } 249 if x > math.MaxInt64 { 250 return math.MaxUint64 251 } 252 return uint64(x) 253 } 254 255 func clampf(x float64) uint64 { 256 if x < 0 { 257 return 0 258 } 259 if x > math.MaxUint64 { 260 return math.MaxUint64 261 } 262 return uint64(x) 263 } 264 265 func (m *Meter) logBytes(direction, name string, read, total int64) { 266 m.fileIndexMutex.Lock() 267 idx := m.fileIndex[name] 268 logger := m.Logger 269 m.fileIndexMutex.Unlock() 270 if logger == nil { 271 return 272 } 273 274 line := fmt.Sprintf("%s %d/%d %d/%d %s\n", direction, idx, m.estimatedFiles, read, total, name) 275 if err := m.Logger.Write([]byte(line)); err != nil { 276 m.fileIndexMutex.Lock() 277 m.Logger = nil 278 m.fileIndexMutex.Unlock() 279 } 280 }