github.com/netdata/go.d.plugin@v0.58.1/modules/weblog/collect.go

// SPDX-License-Identifier: GPL-3.0-or-later

package weblog

import (
	"fmt"
	"io"
	"runtime"
	"strconv"
	"strings"

	"github.com/netdata/go.d.plugin/pkg/logs"
	"github.com/netdata/go.d.plugin/pkg/stm"

	"github.com/netdata/go.d.plugin/agent/module"
)

func (w *WebLog) logPanicStackIfAny() {
	err := recover()
	if err == nil {
		return
	}
	w.Errorf("[ERROR] %s\n", err)
	for depth := 0; ; depth++ {
		_, file, line, ok := runtime.Caller(depth)
		if !ok {
			break
		}
		w.Errorf("======> %d: %v:%d", depth, file, line)
	}
	panic(err)
}

func (w *WebLog) collect() (map[string]int64, error) {
	defer w.logPanicStackIfAny()
	w.mx.reset()

	var mx map[string]int64

	n, err := w.collectLogLines()

	if n > 0 || err == nil {
		mx = stm.ToMap(w.mx)
	}
	return mx, err
}

func (w *WebLog) collectLogLines() (int, error) {
	logOnce := true
	var n int
	for {
		w.line.reset()
		err := w.parser.ReadLine(w.line)
		if err != nil {
			if err == io.EOF {
				return n, nil
			}
			if !logs.IsParseError(err) {
				return n, err
			}
			n++
			if logOnce {
				w.Infof("unmatched line: %v (parser: %s)", err, w.parser.Info())
				logOnce = false
			}
			w.collectUnmatched()
			continue
		}
		n++
		if w.line.empty() {
			w.collectUnmatched()
		} else {
			w.collectLogLine()
		}
	}
}

func (w *WebLog) collectLogLine() {
	w.mx.Requests.Inc()
	w.collectVhost()
	w.collectPort()
	w.collectReqScheme()
	w.collectReqClient()
	w.collectReqMethod()
	w.collectReqURL()
	w.collectReqProto()
	w.collectRespCode()
	w.collectReqSize()
	w.collectRespSize()
	w.collectReqProcTime()
	w.collectUpsRespTime()
	w.collectSSLProto()
	w.collectSSLCipherSuite()
	w.collectCustomFields()
}

func (w *WebLog) collectUnmatched() {
	w.mx.Requests.Inc()
	w.mx.ReqUnmatched.Inc()
}

func (w *WebLog) collectVhost() {
	if !w.line.hasVhost() {
		return
	}
	c, ok := w.mx.ReqVhost.GetP(w.line.vhost)
	if !ok {
		w.addDimToVhostChart(w.line.vhost)
	}
	c.Inc()
}

func (w *WebLog) collectPort() {
	if !w.line.hasPort() {
		return
	}
	c, ok := w.mx.ReqPort.GetP(w.line.port)
	if !ok {
		w.addDimToPortChart(w.line.port)
	}
	c.Inc()
}

func (w *WebLog) collectReqClient() {
	if !w.line.hasReqClient() {
		return
	}
	if strings.ContainsRune(w.line.reqClient, ':') {
		w.mx.ReqIPv6.Inc()
		w.mx.UniqueIPv6.Insert(w.line.reqClient)
		return
	}
	// NOTE: count hostname as IPv4 address
	w.mx.ReqIPv4.Inc()
	w.mx.UniqueIPv4.Insert(w.line.reqClient)
}

func (w *WebLog) collectReqScheme() {
	if !w.line.hasReqScheme() {
		return
	}
	if w.line.reqScheme == "https" {
		w.mx.ReqHTTPSScheme.Inc()
	} else {
		w.mx.ReqHTTPScheme.Inc()
	}
}

func (w *WebLog) collectReqMethod() {
	if !w.line.hasReqMethod() {
		return
	}
	c, ok := w.mx.ReqMethod.GetP(w.line.reqMethod)
	if !ok {
		w.addDimToReqMethodChart(w.line.reqMethod)
	}
	c.Inc()
}

func (w *WebLog) collectReqURL() {
	if !w.line.hasReqURL() {
		return
	}
	for _, p := range w.urlPatterns {
		if !p.MatchString(w.line.reqURL) {
			continue
		}
		c, _ := w.mx.ReqURLPattern.GetP(p.name)
		c.Inc()

		w.collectURLPatternStats(p.name)
		return
	}
}

func (w *WebLog) collectReqProto() {
	if !w.line.hasReqProto() {
		return
	}
	c, ok := w.mx.ReqVersion.GetP(w.line.reqProto)
	if !ok {
		w.addDimToReqVersionChart(w.line.reqProto)
	}
	c.Inc()
}

func (w *WebLog) collectRespCode() {
	if !w.line.hasRespCode() {
		return
	}

	code := w.line.respCode
	switch {
	case code >= 100 && code < 300, code == 304, code == 401:
		w.mx.ReqSuccess.Inc()
	case code >= 300 && code < 400:
		w.mx.ReqRedirect.Inc()
	case code >= 400 && code < 500:
		w.mx.ReqBad.Inc()
	case code >= 500 && code < 600:
		w.mx.ReqError.Inc()
	}

	switch code / 100 {
	case 1:
		w.mx.Resp1xx.Inc()
	case 2:
		w.mx.Resp2xx.Inc()
	case 3:
		w.mx.Resp3xx.Inc()
	case 4:
		w.mx.Resp4xx.Inc()
	case 5:
		w.mx.Resp5xx.Inc()
	}

	codeStr := strconv.Itoa(code)
	c, ok := w.mx.RespCode.GetP(codeStr)
	if !ok {
		w.addDimToRespCodesChart(codeStr)
	}
	c.Inc()
}

func (w *WebLog) collectReqSize() {
	if !w.line.hasReqSize() {
		return
	}
	w.mx.BytesReceived.Add(float64(w.line.reqSize))
}

func (w *WebLog) collectRespSize() {
	if !w.line.hasRespSize() {
		return
	}
	w.mx.BytesSent.Add(float64(w.line.respSize))
}

func (w *WebLog) collectReqProcTime() {
	if !w.line.hasReqProcTime() {
		return
	}
	w.mx.ReqProcTime.Observe(w.line.reqProcTime)
	if w.mx.ReqProcTimeHist == nil {
		return
	}
	w.mx.ReqProcTimeHist.Observe(w.line.reqProcTime)
}

func (w *WebLog) collectUpsRespTime() {
	if !w.line.hasUpsRespTime() {
		return
	}
	w.mx.UpsRespTime.Observe(w.line.upsRespTime)
	if w.mx.UpsRespTimeHist == nil {
		return
	}
	w.mx.UpsRespTimeHist.Observe(w.line.upsRespTime)
}

func (w *WebLog) collectSSLProto() {
	if !w.line.hasSSLProto() {
		return
	}
	c, ok := w.mx.ReqSSLProto.GetP(w.line.sslProto)
	if !ok {
		w.addDimToSSLProtoChart(w.line.sslProto)
	}
	c.Inc()
}

func (w *WebLog) collectSSLCipherSuite() {
	if !w.line.hasSSLCipherSuite() {
		return
	}
	c, ok := w.mx.ReqSSLCipherSuite.GetP(w.line.sslCipherSuite)
	if !ok {
		w.addDimToSSLCipherSuiteChart(w.line.sslCipherSuite)
	}
	c.Inc()
}

func (w *WebLog) collectURLPatternStats(name string) {
	v, ok := w.mx.URLPatternStats[name]
	if !ok {
		return
	}
	if w.line.hasRespCode() {
		status := strconv.Itoa(w.line.respCode)
		c, ok := v.RespCode.GetP(status)
		if !ok {
			w.addDimToURLPatternRespCodesChart(name, status)
		}
		c.Inc()
	}

	if w.line.hasReqMethod() {
		c, ok := v.ReqMethod.GetP(w.line.reqMethod)
		if !ok {
			w.addDimToURLPatternReqMethodsChart(name, w.line.reqMethod)
		}
		c.Inc()
	}

	if w.line.hasReqSize() {
		v.BytesReceived.Add(float64(w.line.reqSize))
	}

	if w.line.hasRespSize() {
		v.BytesSent.Add(float64(w.line.respSize))
	}

	if w.line.hasReqProcTime() {
		v.ReqProcTime.Observe(w.line.reqProcTime)
	}
}

func (w *WebLog) collectCustomFields() {
	if !w.line.hasCustomFields() {
		return
	}

	for _, cv := range w.line.custom.values {
		_, _ = cv.name, cv.value

		if patterns, ok := w.customFields[cv.name]; ok {
			for _, pattern := range patterns {
				if !pattern.MatchString(cv.value) {
					continue
				}
				v, ok := w.mx.ReqCustomField[cv.name]
				if !ok {
					break
				}
				c, _ := v.GetP(pattern.name)
				c.Inc()
				break
			}
		} else if histogram, ok := w.customTimeFields[cv.name]; ok {
			v, ok := w.mx.ReqCustomTimeField[cv.name]
			if !ok {
				continue
			}
			ctf, err := strconv.ParseFloat(cv.value, 64)
			if err != nil || !isTimeValid(ctf) {
				continue
			}
			v.Time.Observe(ctf)
			if histogram != nil {
				v.TimeHist.Observe(ctf * timeMultiplier(cv.value))
			}
		} else if w.customNumericFields[cv.name] {
			m, ok := w.mx.ReqCustomNumericField[cv.name]
			if !ok {
				continue
			}
			v, err := strconv.ParseFloat(cv.value, 64)
			if err != nil {
				continue
			}
			v *= float64(m.multiplier)
			m.Summary.Observe(v)
		}
	}
}

func (w *WebLog) addDimToVhostChart(vhost string) {
	chart := w.Charts().Get(reqByVhost.ID)
	if chart == nil {
		w.Warningf("add dimension: no '%s' chart", reqByVhost.ID)
		return
	}
	dim := &Dim{
		ID:   "req_vhost_" + vhost,
		Name: vhost,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToPortChart(port string) {
	chart := w.Charts().Get(reqByPort.ID)
	if chart == nil {
		w.Warningf("add dimension: no '%s' chart", reqByPort.ID)
		return
	}
	dim := &Dim{
		ID:   "req_port_" + port,
		Name: port,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToReqMethodChart(method string) {
	chart := w.Charts().Get(reqByMethod.ID)
	if chart == nil {
		w.Warningf("add dimension: no '%s' chart", reqByMethod.ID)
		return
	}
	dim := &Dim{
		ID:   "req_method_" + method,
		Name: method,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToReqVersionChart(version string) {
	chart := w.Charts().Get(reqByVersion.ID)
	if chart == nil {
		w.Warningf("add dimension: no '%s' chart", reqByVersion.ID)
		return
	}
	dim := &Dim{
		ID:   "req_version_" + version,
		Name: version,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToSSLProtoChart(proto string) {
	chart := w.Charts().Get(reqBySSLProto.ID)
	if chart == nil {
		chart = reqBySSLProto.Copy()
		if err := w.Charts().Add(chart); err != nil {
			w.Warning(err)
			return
		}
	}
	dim := &Dim{
		ID:   "req_ssl_proto_" + proto,
		Name: proto,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToSSLCipherSuiteChart(cipher string) {
	chart := w.Charts().Get(reqBySSLCipherSuite.ID)
	if chart == nil {
		chart = reqBySSLCipherSuite.Copy()
		if err := w.Charts().Add(chart); err != nil {
			w.Warning(err)
			return
		}
	}
	dim := &Dim{
		ID:   "req_ssl_cipher_suite_" + cipher,
		Name: cipher,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToRespCodesChart(code string) {
	chart := w.findRespCodesChart(code)
	if chart == nil {
		w.Warning("add dimension: cant find resp codes chart")
		return
	}
	dim := &Dim{
		ID:   "resp_code_" + code,
		Name: code,
		Algo: module.Incremental,
	}
	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToURLPatternRespCodesChart(name, code string) {
	id := fmt.Sprintf(urlPatternRespCodes.ID, name)
	chart := w.Charts().Get(id)
	if chart == nil {
		w.Warningf("add dimension: no '%s' chart", id)
		return
	}
	dim := &Dim{
		ID:   fmt.Sprintf("url_ptn_%s_resp_code_%s", name, code),
		Name: code,
		Algo: module.Incremental,
	}

	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) addDimToURLPatternReqMethodsChart(name, method string) {
	id := fmt.Sprintf(urlPatternReqMethods.ID, name)
	chart := w.Charts().Get(id)
	if chart == nil {
		w.Warningf("add dimension: no '%s' chart", id)
		return
	}
	dim := &Dim{
		ID:   fmt.Sprintf("url_ptn_%s_req_method_%s", name, method),
		Name: method,
		Algo: module.Incremental,
	}

	if err := chart.AddDim(dim); err != nil {
		w.Warning(err)
		return
	}
	chart.MarkNotCreated()
}

func (w *WebLog) findRespCodesChart(code string) *Chart {
	if !w.GroupRespCodes {
		return w.Charts().Get(respCodes.ID)
	}

	var id string
	switch class := code[:1]; class {
	case "1":
		id = respCodes1xx.ID
	case "2":
		id = respCodes2xx.ID
	case "3":
		id = respCodes3xx.ID
	case "4":
		id = respCodes4xx.ID
	case "5":
		id = respCodes5xx.ID
	default:
		return nil
	}
	return w.Charts().Get(id)
}