github.com/uber/kraken@v0.1.4/lib/healthcheck/passive_filter.go (about)

     1  // Copyright (c) 2016-2019 Uber Technologies, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  package healthcheck
    15  
    16  import (
    17  	"sync"
    18  	"time"
    19  
    20  	"github.com/andres-erbsen/clock"
    21  	"github.com/uber/kraken/utils/stringset"
    22  )
    23  
    24  // PassiveFilter filters unhealthy hosts passively by tracking failed
    25  // requests to hosts. Clients are responsible for marking failures from
    26  // individual hosts, and PassiveFilter updates which hosts are unhealthy. It is
    27  // recommended that clients only mark failures for network errors, not HTTP
    28  // errors.
    29  type PassiveFilter interface {
    30  	Filter
    31  	Failed(addr string)
    32  }
    33  
    34  type passiveFilter struct {
    35  	sync.Mutex
    36  	config    PassiveFilterConfig
    37  	clk       clock.Clock
    38  	unhealthy map[string]time.Time
    39  	failures  map[string][]time.Time
    40  }
    41  
    42  // NewPassiveFilter creates a new PassiveFilter.
    43  func NewPassiveFilter(config PassiveFilterConfig, clk clock.Clock) PassiveFilter {
    44  	config.applyDefaults()
    45  	return &passiveFilter{
    46  		config:    config,
    47  		clk:       clk,
    48  		unhealthy: make(map[string]time.Time),
    49  		failures:  make(map[string][]time.Time),
    50  	}
    51  }
    52  
    53  // Run removes any unhealthy from addrs.
    54  func (f *passiveFilter) Run(addrs stringset.Set) stringset.Set {
    55  	f.Lock()
    56  	defer f.Unlock()
    57  
    58  	healthy := addrs.Copy()
    59  
    60  	for addr, t := range f.unhealthy {
    61  		if f.clk.Now().Sub(t) > f.config.FailTimeout {
    62  			delete(f.unhealthy, addr)
    63  		} else {
    64  			healthy.Remove(addr)
    65  		}
    66  	}
    67  
    68  	return healthy
    69  }
    70  
    71  // Failed marks a request to addr as failed.
    72  func (f *passiveFilter) Failed(addr string) {
    73  	f.Lock()
    74  	defer f.Unlock()
    75  
    76  	now := f.clk.Now()
    77  
    78  	failures := f.failures[addr]
    79  
    80  	// Pop off the expired failures.
    81  	for len(failures) > 0 {
    82  		if now.Sub(failures[0]) > f.config.FailTimeout {
    83  			failures = failures[1:]
    84  			continue
    85  		}
    86  		break
    87  	}
    88  
    89  	// Add latest failure.
    90  	failures = append(failures, now)
    91  
    92  	if len(failures) >= f.config.Fails {
    93  		f.unhealthy[addr] = now
    94  	}
    95  	f.failures[addr] = failures
    96  }