github.com/dolthub/dolt/go@v0.40.5-0.20240520175717-68db7794bea6/libraries/doltcore/remotestorage/stats.go (about)

     1  // Copyright 2019 Dolthub, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package remotestorage
    16  
    17  import (
    18  	"fmt"
    19  	"io"
    20  	"os"
    21  	"sync"
    22  	"time"
    23  
    24  	"github.com/HdrHistogram/hdrhistogram-go"
    25  	"github.com/fatih/color"
    26  
    27  	"github.com/dolthub/dolt/go/libraries/doltcore/dconfig"
    28  )
    29  
    30  var StatsFactory func() StatsRecorder = NullStatsRecorderFactory
    31  var StatsFlusher func(StatsRecorder) = func(StatsRecorder) {}
    32  
    33  func NullStatsRecorderFactory() StatsRecorder {
    34  	return NullStatsRecorder{}
    35  }
    36  
    37  func HistogramStatsRecorderFactory() StatsRecorder {
    38  	return NewHistorgramStatsRecorder()
    39  }
    40  
    41  func StatsFlusherToColorError(r StatsRecorder) {
    42  	r.WriteSummaryTo(color.Error)
    43  }
    44  
    45  func init() {
    46  	if _, ok := os.LookupEnv(dconfig.EnvRemoteVersionDownloadStats); ok {
    47  		StatsFactory = HistogramStatsRecorderFactory
    48  		StatsFlusher = StatsFlusherToColorError
    49  	}
    50  }
    51  
    52  type StatsRecorder interface {
    53  	RecordTimeToFirstByte(retry int, size uint64, d time.Duration)
    54  	RecordDownloadAttemptStart(retry int, offset, size uint64)
    55  	RecordDownloadComplete(retry int, size uint64, d time.Duration)
    56  	WriteSummaryTo(io.Writer) error
    57  }
    58  
    59  var _ StatsRecorder = NullStatsRecorder{}
    60  
    61  type NullStatsRecorder struct {
    62  }
    63  
    64  func (NullStatsRecorder) RecordTimeToFirstByte(retry int, size uint64, d time.Duration) {
    65  }
    66  
    67  func (NullStatsRecorder) RecordDownloadAttemptStart(retry int, offset, size uint64) {
    68  }
    69  
    70  func (NullStatsRecorder) RecordDownloadComplete(retry int, size uint64, d time.Duration) {
    71  }
    72  
    73  func (NullStatsRecorder) WriteSummaryTo(io.Writer) error {
    74  	return nil
    75  }
    76  
    77  type HistogramStatsRecorder struct {
    78  	mu                 *sync.Mutex
    79  	sizes              *hdrhistogram.Histogram
    80  	downloadTimeMillis *hdrhistogram.Histogram
    81  	firstByteMillis    *hdrhistogram.Histogram
    82  	retryCount         int
    83  }
    84  
    85  func NewHistorgramStatsRecorder() *HistogramStatsRecorder {
    86  	return &HistogramStatsRecorder{
    87  		new(sync.Mutex),
    88  		hdrhistogram.New(128, 4294967296, 3), // 128 bytes - 4 GB
    89  		hdrhistogram.New(10, 3600000, 3),     // 10 ms - 1 hr
    90  		hdrhistogram.New(10, 300000, 3),      // 10 ms - 5 mins
    91  		0,
    92  	}
    93  }
    94  
    95  func (r *HistogramStatsRecorder) RecordTimeToFirstByte(retry int, size uint64, d time.Duration) {
    96  	r.mu.Lock()
    97  	defer r.mu.Unlock()
    98  	r.firstByteMillis.RecordValue(int64(d / time.Millisecond))
    99  }
   100  
   101  func (r *HistogramStatsRecorder) RecordDownloadAttemptStart(retry int, offset, size uint64) {
   102  	if retry == 0 {
   103  		r.mu.Lock()
   104  		defer r.mu.Unlock()
   105  		r.sizes.RecordValue(int64(size))
   106  	}
   107  	if retry > 0 {
   108  		r.mu.Lock()
   109  		defer r.mu.Unlock()
   110  		r.retryCount += 1
   111  	}
   112  }
   113  
   114  func (r *HistogramStatsRecorder) RecordDownloadComplete(retry int, size uint64, d time.Duration) {
   115  	r.mu.Lock()
   116  	defer r.mu.Unlock()
   117  	r.downloadTimeMillis.RecordValue(int64(d / time.Millisecond))
   118  }
   119  
   120  func (r *HistogramStatsRecorder) WriteSummaryTo(w io.Writer) error {
   121  	r.mu.Lock()
   122  	defer r.mu.Unlock()
   123  	_, err := fmt.Fprintf(w, "total downloads: %d, retries: %d\n", r.sizes.TotalCount(), r.retryCount)
   124  	if err != nil {
   125  		return err
   126  	}
   127  	err = writeHistogram(w, "sizes (bytes)", r.sizes)
   128  	if err != nil {
   129  		return err
   130  	}
   131  	err = writeHistogram(w, "time to first byte (millis)", r.firstByteMillis)
   132  	if err != nil {
   133  		return err
   134  	}
   135  	err = writeHistogram(w, "download time (millis)", r.downloadTimeMillis)
   136  	if err != nil {
   137  		return err
   138  	}
   139  	return nil
   140  }
   141  
   142  func writeHistogram(w io.Writer, prefix string, h *hdrhistogram.Histogram) error {
   143  	_, err := fmt.Fprintf(w, "%s: p50: %d, p90: %d, p99: %d, avg: %.2f, max: %d\n", prefix,
   144  		h.ValueAtQuantile(50), h.ValueAtQuantile(90), h.ValueAtQuantile(99), h.Mean(), h.Max())
   145  	return err
   146  }