github.com/muhammadn/cortex@v1.9.1-0.20220510110439-46bb7000d03d/integration/e2e/composite_service.go (about) 1 package e2e 2 3 import ( 4 "context" 5 "fmt" 6 "time" 7 8 "github.com/grafana/dskit/backoff" 9 "github.com/pkg/errors" 10 ) 11 12 // CompositeHTTPService abstract an higher-level service composed, under the hood, 13 // by 2+ HTTPService. 14 type CompositeHTTPService struct { 15 services []*HTTPService 16 17 // Generic retry backoff. 18 retryBackoff *backoff.Backoff 19 } 20 21 func NewCompositeHTTPService(services ...*HTTPService) *CompositeHTTPService { 22 return &CompositeHTTPService{ 23 services: services, 24 retryBackoff: backoff.New(context.Background(), backoff.Config{ 25 MinBackoff: 300 * time.Millisecond, 26 MaxBackoff: 600 * time.Millisecond, 27 MaxRetries: 50, // Sometimes the CI is slow ¯\_(ツ)_/¯ 28 }), 29 } 30 } 31 32 func (s *CompositeHTTPService) NumInstances() int { 33 return len(s.services) 34 } 35 36 func (s *CompositeHTTPService) Instances() []*HTTPService { 37 return s.services 38 } 39 40 // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true 41 // when passed to given isExpected(...). 42 func (s *CompositeHTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error { 43 return s.WaitSumMetricsWithOptions(isExpected, metricNames) 44 } 45 46 func (s *CompositeHTTPService) WaitSumMetricsWithOptions(isExpected func(sums ...float64) bool, metricNames []string, opts ...MetricsOption) error { 47 var ( 48 sums []float64 49 err error 50 options = buildMetricsOptions(opts) 51 ) 52 53 for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); { 54 sums, err = s.SumMetrics(metricNames, opts...) 55 if options.WaitMissingMetrics && errors.Is(err, errMissingMetric) { 56 continue 57 } 58 if err != nil { 59 return err 60 } 61 62 if isExpected(sums...) { 63 return nil 64 } 65 66 s.retryBackoff.Wait() 67 } 68 69 return fmt.Errorf("unable to find metrics %s with expected values. Last error: %v. Last values: %v", metricNames, err, sums) 70 } 71 72 // SumMetrics returns the sum of the values of each given metric names. 73 func (s *CompositeHTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) { 74 sums := make([]float64, len(metricNames)) 75 76 for _, service := range s.services { 77 partials, err := service.SumMetrics(metricNames, opts...) 78 if err != nil { 79 return nil, err 80 } 81 82 if len(partials) != len(sums) { 83 return nil, fmt.Errorf("unexpected mismatching sum metrics results (got %d, expected %d)", len(partials), len(sums)) 84 } 85 86 for i := 0; i < len(sums); i++ { 87 sums[i] += partials[i] 88 } 89 } 90 91 return sums, nil 92 }