github.com/weaviate/weaviate@v1.24.6/adapters/handlers/rest/panics_middleware.go (about)

     1  //                           _       _
     2  // __      _____  __ ___   ___  __ _| |_ ___
     3  // \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
     4  //  \ V  V /  __/ (_| |\ V /| | (_| | ||  __/
     5  //   \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
     6  //
     7  //  Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
     8  //
     9  //  CONTACT: hello@weaviate.io
    10  //
    11  
    12  package rest
    13  
    14  import (
    15  	"fmt"
    16  	"net"
    17  	"net/http"
    18  	"runtime/debug"
    19  	"syscall"
    20  
    21  	"github.com/pkg/errors"
    22  	"github.com/sirupsen/logrus"
    23  )
    24  
    25  func makeCatchPanics(logger logrus.FieldLogger, metricRequestsTotal restApiRequestsTotal) func(http.Handler) http.Handler {
    26  	return func(next http.Handler) http.Handler {
    27  		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    28  			defer handlePanics(logger, metricRequestsTotal, r)
    29  			next.ServeHTTP(w, r)
    30  		})
    31  	}
    32  }
    33  
    34  func handlePanics(logger logrus.FieldLogger, metricRequestsTotal restApiRequestsTotal, r *http.Request) {
    35  	recovered := recover()
    36  	if recovered == nil {
    37  		return
    38  	}
    39  
    40  	err, ok := recovered.(error)
    41  	if !ok {
    42  		// not a typed error, we can not handle this error other returning it to
    43  		// the user
    44  		logger.WithFields(logrus.Fields{
    45  			"error":  recovered,
    46  			"method": r.Method,
    47  			"path":   r.URL,
    48  		}).Errorf("%v", recovered)
    49  
    50  		// This was not expected, so we want to print the stack, this will help us
    51  		// find the source of the issue if the user sends their logs
    52  		metricRequestsTotal.logServerError("", fmt.Errorf("%v", recovered))
    53  		debug.PrintStack()
    54  		return
    55  	}
    56  
    57  	if errors.Is(err, syscall.EPIPE) {
    58  		metricRequestsTotal.logUserError("")
    59  		handleBrokenPipe(err, logger, r)
    60  		return
    61  	}
    62  
    63  	var netErr net.Error
    64  	if errors.As(err, &netErr) {
    65  		if netErr.Timeout() {
    66  			metricRequestsTotal.logUserError("")
    67  			handleTimeout(netErr, logger, r)
    68  			return
    69  		}
    70  	}
    71  
    72  	// typed as error, but none we are currently handling explicitly
    73  	logger.WithError(err).WithFields(logrus.Fields{
    74  		"method": r.Method,
    75  		"path":   r.URL,
    76  	}).Errorf(err.Error())
    77  	// This was not expected, so we want to print the stack, this will help us
    78  	// find the source of the issue if the user sends their logs
    79  	metricRequestsTotal.logServerError("", err)
    80  	debug.PrintStack()
    81  }
    82  
    83  func handleBrokenPipe(err error, logger logrus.FieldLogger, r *http.Request) {
    84  	logger.WithError(err).WithFields(logrus.Fields{
    85  		"method":      r.Method,
    86  		"path":        r.URL,
    87  		"description": "A broken pipe error occurs when Weaviate tries to write a response onto a connection that has already been closed or reset by the client. Typically, this is the case when the server was not able to respond within the configured client-side timeout.",
    88  		"hint":        "Either try increasing the client-side timeout, or sending a computationally cheaper request, for example by reducing a batch size, reducing a limit, using less complex filters, etc.",
    89  	}).Errorf("broken pipe")
    90  }
    91  
    92  func handleTimeout(err net.Error, logger logrus.FieldLogger, r *http.Request) {
    93  	logger.WithError(err).WithFields(logrus.Fields{
    94  		"method":      r.Method,
    95  		"path":        r.URL,
    96  		"description": "An I/O timeout occurs when the request takes longer than the specified server-side timeout.",
    97  		"hint":        "Either try increasing the server-side timeout using e.g. '--write-timeout=600s' as a command line flag when starting Weaviate, or try sending a computationally cheaper request, for example by reducing a batch size, reducing a limit, using less complex filters, etc. Note that this error is only thrown if client-side and server-side timeouts are not in sync, more precisely if the client-side timeout is longer than the server side timeout.",
    98  	}).Errorf("i/o timeout")
    99  }