github.com/whamcloud/lemur@v0.0.0-20190827193804-4655df8a52af/cmd/lhsmd/agent/action_stats.go (about)

     1  // Copyright (c) 2018 DDN. All rights reserved.
     2  // Use of this source code is governed by a MIT-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package agent
     6  
     7  import (
     8  	"fmt"
     9  	"sync"
    10  	"sync/atomic"
    11  	"time"
    12  
    13  	"golang.org/x/net/context"
    14  
    15  	"github.com/dustin/go-humanize"
    16  	"github.com/rcrowley/go-metrics"
    17  
    18  	"github.com/intel-hpdd/logging/audit"
    19  	"github.com/intel-hpdd/logging/debug"
    20  )
    21  
    22  // ActionStats is a synchronized container for ArchiveStats instances
    23  type ActionStats struct {
    24  	sync.Mutex
    25  	stats map[int]*ArchiveStats
    26  }
    27  
    28  // ArchiveStats is a per-archive container of statistics for that backend
    29  type ArchiveStats struct {
    30  	changes     uint64
    31  	queueLength metrics.Counter
    32  	completed   metrics.Timer
    33  }
    34  
    35  // NewActionStats initializes a new ActionStats container
    36  func NewActionStats() *ActionStats {
    37  	return &ActionStats{
    38  		stats: make(map[int]*ArchiveStats),
    39  	}
    40  }
    41  
    42  func (as *ActionStats) update() {
    43  	for _, k := range as.Archives() {
    44  		archive := as.GetIndex(k)
    45  		changes := atomic.LoadUint64(&archive.changes)
    46  		if changes != 0 {
    47  			atomic.AddUint64(&archive.changes, -changes)
    48  			audit.Logf("archive:%d %s", k, archive)
    49  		}
    50  	}
    51  }
    52  
    53  func (as *ActionStats) run(ctx context.Context) {
    54  	for {
    55  		select {
    56  		case <-ctx.Done():
    57  			debug.Print("Shutting down stats collector")
    58  			return
    59  		case <-time.After(10 * time.Second):
    60  			as.update()
    61  		}
    62  	}
    63  }
    64  
    65  // Start creates a new goroutine for collecting archive stats
    66  func (as *ActionStats) Start(ctx context.Context) {
    67  	go as.run(ctx)
    68  	debug.Print("Stats collector started in background")
    69  }
    70  
    71  // StartAction increments stats counters when an action starts
    72  func (as *ActionStats) StartAction(a *Action) {
    73  	s := as.GetIndex(int(a.aih.ArchiveID()))
    74  	s.queueLength.Inc(1)
    75  	atomic.AddUint64(&s.changes, 1)
    76  }
    77  
    78  // CompleteAction updates various stats when an action is complete
    79  func (as *ActionStats) CompleteAction(a *Action, rc int) {
    80  	s := as.GetIndex(int(a.aih.ArchiveID()))
    81  	s.queueLength.Dec(1)
    82  	s.completed.UpdateSince(a.start)
    83  	atomic.AddUint64(&s.changes, 1)
    84  }
    85  
    86  // GetIndex returns the *ArchiveStats corresponding to the supplied archive
    87  // number
    88  func (as *ActionStats) GetIndex(i int) *ArchiveStats {
    89  	as.Lock()
    90  	defer as.Unlock()
    91  	s, ok := as.stats[i]
    92  	if !ok {
    93  		s = &ArchiveStats{
    94  			queueLength: metrics.NewCounter(),
    95  			completed:   metrics.NewTimer(),
    96  		}
    97  		metrics.Register(fmt.Sprintf("archive%dCompleted", i), s.completed)
    98  		metrics.Register(fmt.Sprintf("archive%dQueueLength", i), s.queueLength)
    99  		as.stats[i] = s
   100  	}
   101  	return s
   102  }
   103  
   104  // Archives returns a slice of archive numbers corresponding to instrumented
   105  // backends
   106  func (as *ActionStats) Archives() (v []int) {
   107  	as.Lock()
   108  	defer as.Unlock()
   109  	for k := range as.stats {
   110  		v = append(v, k)
   111  	}
   112  	return
   113  }
   114  
   115  func (s *ArchiveStats) String() string {
   116  	ps := s.completed.Percentiles([]float64{0.5, .75, 0.95, 0.99, 0.999})
   117  	return fmt.Sprintf("total:%v queue:%v %v/%v/%v min:%v max:%v mean:%v median:%v 75%%:%v 95%%:%v 99%%:%v 99.9%%:%v",
   118  		humanize.Comma(s.completed.Count()),
   119  		humanize.Comma(s.queueLength.Count()),
   120  		humanize.Comma(int64(s.completed.Rate1())),
   121  		humanize.Comma(int64(s.completed.Rate5())),
   122  		humanize.Comma(int64(s.completed.Rate15())),
   123  		time.Duration(s.completed.Min()),
   124  		time.Duration(s.completed.Max()),
   125  		time.Duration(int64(s.completed.Mean())),
   126  		time.Duration(int64(ps[0])),
   127  		time.Duration(int64(ps[1])),
   128  		time.Duration(int64(ps[2])),
   129  		time.Duration(int64(ps[3])),
   130  		time.Duration(int64(ps[4])))
   131  }