github.com/Jeffail/benthos/v3@v3.65.0/lib/processor/metric.go (about) 1 package processor 2 3 import ( 4 "errors" 5 "fmt" 6 "sort" 7 "strconv" 8 "strings" 9 "time" 10 11 "github.com/Jeffail/benthos/v3/internal/bloblang/field" 12 "github.com/Jeffail/benthos/v3/internal/docs" 13 "github.com/Jeffail/benthos/v3/internal/interop" 14 "github.com/Jeffail/benthos/v3/lib/log" 15 "github.com/Jeffail/benthos/v3/lib/metrics" 16 "github.com/Jeffail/benthos/v3/lib/types" 17 ) 18 19 //------------------------------------------------------------------------------ 20 21 func init() { 22 Constructors[TypeMetric] = TypeSpec{ 23 constructor: NewMetric, 24 Categories: []Category{ 25 CategoryUtility, 26 }, 27 Summary: "Emit custom metrics by extracting values from messages.", 28 Description: ` 29 This processor works by evaluating an [interpolated field ` + "`value`" + `](/docs/configuration/interpolation#bloblang-queries) for each message and updating a emitted metric according to the [type](#types). 30 31 Custom metrics such as these are emitted along with Benthos internal metrics, where you can customize where metrics are sent, which metric names are emitted and rename them as/when appropriate. For more information check out the [metrics docs here](/docs/components/metrics/about).`, 32 FieldSpecs: docs.FieldSpecs{ 33 docs.FieldCommon("type", "The metric [type](#types) to create.").HasOptions( 34 "counter", 35 "counter_by", 36 "gauge", 37 "timing", 38 ), 39 docs.FieldDeprecated("path"), 40 docs.FieldCommon("name", "The name of the metric to create, this must be unique across all Benthos components otherwise it will overwrite those other metrics."), 41 docs.FieldString( 42 "labels", "A map of label names and values that can be used to enrich metrics. Labels are not supported by some metric destinations, in which case the metrics series are combined.", 43 map[string]string{ 44 "type": "${! json(\"doc.type\") }", 45 "topic": "${! meta(\"kafka_topic\") }", 46 }, 47 ).IsInterpolated().Map(), 48 docs.FieldCommon("value", "For some metric types specifies a value to set, increment.").IsInterpolated(), 49 PartsFieldSpec, 50 }, 51 Examples: []docs.AnnotatedExample{ 52 { 53 Title: "Counter", 54 Summary: "In this example we emit a counter metric called `Foos`, which increments for every message processed, and we label the metric with some metadata about where the message came from and a field from the document that states what type it is. We also configure our metrics to emit to CloudWatch, and explicitly only allow our custom metric and some internal Benthos metrics to emit.", 55 Config: ` 56 pipeline: 57 processors: 58 - metric: 59 name: Foos 60 type: counter 61 labels: 62 topic: ${! meta("kafka_topic") } 63 partition: ${! meta("kafka_partition") } 64 type: ${! json("document.type").or("unknown") } 65 66 metrics: 67 aws_cloudwatch: 68 namespace: ProdConsumer 69 region: eu-west-1 70 path_mapping: | 71 root = if ![ 72 "Foos", 73 "input.received", 74 "output.sent" 75 ].contains(this) { deleted() } 76 `, 77 }, 78 { 79 Title: "Gauge", 80 Summary: "In this example we emit a gauge metric called `FooSize`, which is given a value extracted from JSON messages at the path `foo.size`. We then also configure our Prometheus metric exporter to only emit this custom metric and nothing else. We also label the metric with some metadata.", 81 Config: ` 82 pipeline: 83 processors: 84 - metric: 85 name: FooSize 86 type: gauge 87 labels: 88 topic: ${! meta("kafka_topic") } 89 value: ${! json("foo.size") } 90 91 metrics: 92 prometheus: 93 path_mapping: 'if this != "FooSize" { deleted() }' 94 `, 95 }, 96 }, 97 Footnotes: ` 98 ## Types 99 100 ### ` + "`counter`" + ` 101 102 Increments a counter by exactly 1, the contents of ` + "`value`" + ` are ignored 103 by this type. 104 105 ### ` + "`counter_by`" + ` 106 107 If the contents of ` + "`value`" + ` can be parsed as a positive integer value 108 then the counter is incremented by this value. 109 110 For example, the following configuration will increment the value of the 111 ` + "`count.custom.field` metric by the contents of `field.some.value`" + `: 112 113 ` + "```yaml" + ` 114 pipeline: 115 processors: 116 - metric: 117 type: counter_by 118 name: CountCustomField 119 value: ${!json("field.some.value")} 120 ` + "```" + ` 121 122 ### ` + "`gauge`" + ` 123 124 If the contents of ` + "`value`" + ` can be parsed as a positive integer value 125 then the gauge is set to this value. 126 127 For example, the following configuration will set the value of the 128 ` + "`gauge.custom.field` metric to the contents of `field.some.value`" + `: 129 130 ` + "```yaml" + ` 131 pipeline: 132 processors: 133 - metric: 134 type: gauge 135 path: GaugeCustomField 136 value: ${!json("field.some.value")} 137 ` + "```" + ` 138 139 ### ` + "`timing`" + ` 140 141 Equivalent to ` + "`gauge`" + ` where instead the metric is a timing.`, 142 } 143 } 144 145 //------------------------------------------------------------------------------ 146 147 // MetricConfig contains configuration fields for the Metric processor. 148 type MetricConfig struct { 149 Parts []int `json:"parts" yaml:"parts"` 150 Type string `json:"type" yaml:"type"` 151 Path string `json:"path" yaml:"path"` 152 Name string `json:"name" yaml:"name"` 153 Labels map[string]string `json:"labels" yaml:"labels"` 154 Value string `json:"value" yaml:"value"` 155 } 156 157 // NewMetricConfig returns a MetricConfig with default values. 158 func NewMetricConfig() MetricConfig { 159 return MetricConfig{ 160 Parts: []int{}, 161 Type: "counter", 162 Path: "", 163 Name: "", 164 Labels: map[string]string{}, 165 Value: "", 166 } 167 } 168 169 //------------------------------------------------------------------------------ 170 171 // Metric is a processor that creates a metric from extracted values from a message part. 172 type Metric struct { 173 parts []int 174 deprecated bool 175 176 conf Config 177 log log.Modular 178 stats metrics.Type 179 180 value *field.Expression 181 labels labels 182 183 mCounter metrics.StatCounter 184 mGauge metrics.StatGauge 185 mTimer metrics.StatTimer 186 187 mCounterVec metrics.StatCounterVec 188 mGaugeVec metrics.StatGaugeVec 189 mTimerVec metrics.StatTimerVec 190 191 handler func(string, int, types.Message) error 192 } 193 194 type labels []label 195 type label struct { 196 name string 197 value *field.Expression 198 } 199 200 func (l *label) val(index int, msg types.Message) string { 201 return l.value.String(index, msg) 202 } 203 204 func (l labels) names() []string { 205 var names []string 206 for i := range l { 207 names = append(names, l[i].name) 208 } 209 return names 210 } 211 212 func (l labels) values(index int, msg types.Message) []string { 213 var values []string 214 for i := range l { 215 values = append(values, l[i].val(index, msg)) 216 } 217 return values 218 } 219 220 func unwrapMetric(t metrics.Type) metrics.Type { 221 u, ok := t.(interface { 222 Unwrap() metrics.Type 223 }) 224 if ok { 225 t = u.Unwrap() 226 } 227 return t 228 } 229 230 // NewMetric returns a Metric processor. 231 func NewMetric( 232 conf Config, mgr types.Manager, log log.Modular, stats metrics.Type, 233 ) (Type, error) { 234 value, err := interop.NewBloblangField(mgr, conf.Metric.Value) 235 if err != nil { 236 return nil, fmt.Errorf("failed to parse value expression: %v", err) 237 } 238 239 m := &Metric{ 240 parts: conf.Metric.Parts, 241 conf: conf, 242 log: log, 243 stats: stats, 244 value: value, 245 } 246 247 name := conf.Metric.Name 248 if len(conf.Metric.Path) > 0 { 249 if len(conf.Metric.Name) > 0 { 250 return nil, errors.New("cannot combine deprecated path field with name field") 251 } 252 if len(conf.Metric.Parts) > 0 { 253 return nil, errors.New("cannot combine deprecated path field with parts field") 254 } 255 m.deprecated = true 256 name = conf.Metric.Path 257 } 258 if name == "" { 259 return nil, errors.New("metric name must not be empty") 260 } 261 if !m.deprecated { 262 // Remove any namespaces from the metric type. 263 stats = unwrapMetric(stats) 264 } 265 266 labelNames := make([]string, 0, len(conf.Metric.Labels)) 267 for n := range conf.Metric.Labels { 268 labelNames = append(labelNames, n) 269 } 270 sort.Strings(labelNames) 271 272 for _, n := range labelNames { 273 v, err := interop.NewBloblangField(mgr, conf.Metric.Labels[n]) 274 if err != nil { 275 return nil, fmt.Errorf("failed to parse label '%v' expression: %v", n, err) 276 } 277 m.labels = append(m.labels, label{ 278 name: n, 279 value: v, 280 }) 281 } 282 283 switch strings.ToLower(conf.Metric.Type) { 284 case "counter": 285 if len(m.labels) > 0 { 286 m.mCounterVec = stats.GetCounterVec(name, m.labels.names()) 287 } else { 288 m.mCounter = stats.GetCounter(name) 289 } 290 m.handler = m.handleCounter 291 case "counter_parts": 292 if len(m.labels) > 0 { 293 m.mCounterVec = stats.GetCounterVec(name, m.labels.names()) 294 } else { 295 m.mCounter = stats.GetCounter(name) 296 } 297 m.handler = m.handleCounterParts 298 case "counter_by": 299 if len(m.labels) > 0 { 300 m.mCounterVec = stats.GetCounterVec(name, m.labels.names()) 301 } else { 302 m.mCounter = stats.GetCounter(name) 303 } 304 m.handler = m.handleCounterBy 305 case "gauge": 306 if len(m.labels) > 0 { 307 m.mGaugeVec = stats.GetGaugeVec(name, m.labels.names()) 308 } else { 309 m.mGauge = stats.GetGauge(name) 310 } 311 m.handler = m.handleGauge 312 case "timing": 313 if len(m.labels) > 0 { 314 m.mTimerVec = stats.GetTimerVec(name, m.labels.names()) 315 } else { 316 m.mTimer = stats.GetTimer(name) 317 } 318 m.handler = m.handleTimer 319 default: 320 return nil, fmt.Errorf("metric type unrecognised: %v", conf.Metric.Type) 321 } 322 323 return m, nil 324 } 325 326 func (m *Metric) handleCounter(val string, index int, msg types.Message) error { 327 if len(m.labels) > 0 { 328 m.mCounterVec.With(m.labels.values(index, msg)...).Incr(1) 329 } else { 330 m.mCounter.Incr(1) 331 } 332 return nil 333 } 334 335 // TODO: V4 Remove this 336 func (m *Metric) handleCounterParts(val string, index int, msg types.Message) error { 337 if msg.Len() == 0 { 338 return nil 339 } 340 if len(m.labels) > 0 { 341 m.mCounterVec.With(m.labels.values(index, msg)...).Incr(int64(msg.Len())) 342 } else { 343 m.mCounter.Incr(int64(msg.Len())) 344 } 345 return nil 346 } 347 348 func (m *Metric) handleCounterBy(val string, index int, msg types.Message) error { 349 i, err := strconv.ParseInt(val, 10, 64) 350 if err != nil { 351 return err 352 } 353 if i < 0 { 354 return errors.New("value is negative") 355 } 356 if len(m.labels) > 0 { 357 m.mCounterVec.With(m.labels.values(index, msg)...).Incr(i) 358 } else { 359 m.mCounter.Incr(i) 360 } 361 return nil 362 } 363 364 func (m *Metric) handleGauge(val string, index int, msg types.Message) error { 365 i, err := strconv.ParseInt(val, 10, 64) 366 if err != nil { 367 return err 368 } 369 if i < 0 { 370 return errors.New("value is negative") 371 } 372 if len(m.labels) > 0 { 373 m.mGaugeVec.With(m.labels.values(index, msg)...).Set(i) 374 } else { 375 m.mGauge.Set(i) 376 } 377 return nil 378 } 379 380 func (m *Metric) handleTimer(val string, index int, msg types.Message) error { 381 i, err := strconv.ParseInt(val, 10, 64) 382 if err != nil { 383 return err 384 } 385 if i < 0 { 386 return errors.New("value is negative") 387 } 388 if len(m.labels) > 0 { 389 m.mTimerVec.With(m.labels.values(index, msg)...).Timing(i) 390 } else { 391 m.mTimer.Timing(i) 392 } 393 return nil 394 } 395 396 // ProcessMessage applies the processor to a message 397 func (m *Metric) ProcessMessage(msg types.Message) ([]types.Message, types.Response) { 398 if m.deprecated { 399 value := m.value.String(0, msg) 400 if err := m.handler(value, 0, msg); err != nil { 401 m.log.Errorf("Handler error: %v\n", err) 402 } 403 return []types.Message{msg}, nil 404 } 405 if err := iterateParts(m.parts, msg, func(index int, p types.Part) error { 406 value := m.value.String(index, msg) 407 if err := m.handler(value, index, msg); err != nil { 408 m.log.Errorf("Handler error: %v\n", err) 409 } 410 return nil 411 }); err != nil { 412 m.log.Errorf("Failed to iterate parts: %v\n", err) 413 } 414 return []types.Message{msg}, nil 415 } 416 417 // CloseAsync shuts down the processor and stops processing requests. 418 func (m *Metric) CloseAsync() { 419 } 420 421 // WaitForClose blocks until the processor has closed down. 422 func (m *Metric) WaitForClose(timeout time.Duration) error { 423 return nil 424 }