github.com/cloudwego/kitex@v0.9.0/pkg/retry/util.go

/*
 * Copyright 2021 CloudWeGo Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package retry

import (
	"context"
	"errors"
	"fmt"
	"runtime/debug"
	"strconv"

	"github.com/bytedance/gopkg/cloud/metainfo"

	"github.com/cloudwego/kitex/pkg/kerrors"
	"github.com/cloudwego/kitex/pkg/klog"
	"github.com/cloudwego/kitex/pkg/rpcinfo"
	"github.com/cloudwego/kitex/pkg/rpcinfo/remoteinfo"
)

type ctxKey string

const (
	// TransitKey is transited persistently when the req is a retry call.
	// When a request with this key, the downstream will not do retry(just support in kitex now).
	TransitKey = "RetryReq"

	// CtxReqOp is used to ignore RPC Request concurrent write
	CtxReqOp ctxKey = "K_REQ_OP"

	// CtxRespOp is used to ignore RPC Response concurrent write/read.
	CtxRespOp ctxKey = "K_RESP_OP"

	// Wildcard stands for 'any method' when associated with a retryer.
	Wildcard = "*"
)

// Req or Resp operation state, just be useful when concurrent write may happen
const (
	OpNo int32 = iota
	OpDoing
	OpDone
)

var tagValueFirstTry = "0"

// DDLStopFunc is the definition of ddlStop func
type DDLStopFunc func(ctx context.Context, policy StopPolicy) (bool, string)

var ddlStopFunc DDLStopFunc

// RegisterDDLStop registers ddl stop.
func RegisterDDLStop(f DDLStopFunc) {
	ddlStopFunc = f
}

// If Ingress is turned on in the current node, check whether RPC_PERSIST_DDL_REMAIN_TIME exists,
// if it exists calculate handler time consumed by RPC_PERSIST_INGRESS_START_TIME and current time,
// if handler cost > ddl remain time, then do not execute retry.
func ddlStop(ctx context.Context, policy StopPolicy) (bool, string) {
	if !policy.DDLStop {
		return false, ""
	}
	if ddlStopFunc == nil {
		klog.Warnf("enable ddl stop for retry, but no ddlStopFunc is registered")
		return false, ""
	}
	return ddlStopFunc(ctx, policy)
}

func chainStop(ctx context.Context, policy StopPolicy) (bool, string) {
	if policy.DisableChainStop {
		return false, ""
	}
	if !IsRemoteRetryRequest(ctx) {
		return false, ""
	}
	return true, "chain stop retry"
}

func circuitBreakerStop(ctx context.Context, policy StopPolicy, cbC *cbContainer, request interface{}, cbKey string) (bool, string) {
	if cbC.cbCtl == nil || cbC.cbPanel == nil {
		return false, ""
	}
	metricer := cbC.cbPanel.GetMetricer(cbKey)
	errRate := metricer.ErrorRate()
	sample := metricer.Samples()
	if sample < cbMinSample || errRate < policy.CBPolicy.ErrorRate {
		return false, ""
	}
	return true, fmt.Sprintf("retry circuit break, errRate=%0.3f, sample=%d", errRate, sample)
}

func handleRetryInstance(retrySameNode bool, prevRI, retryRI rpcinfo.RPCInfo) {
	calledInst := remoteinfo.AsRemoteInfo(prevRI.To()).GetInstance()
	if calledInst == nil {
		return
	}
	if retrySameNode {
		remoteinfo.AsRemoteInfo(retryRI.To()).SetInstance(calledInst)
	} else {
		if me := remoteinfo.AsRemoteInfo(retryRI.To()); me != nil {
			me.SetTag(rpcinfo.RetryPrevInstTag, calledInst.Address().String())
		}
	}
}

func makeRetryErr(ctx context.Context, msg string, callTimes int32) error {
	var ctxErr string
	if ctx.Err() == context.Canceled {
		ctxErr = "context canceled by business."
	}

	ri := rpcinfo.GetRPCInfo(ctx)
	to := ri.To()

	errMsg := fmt.Sprintf("retry[%d] failed, %s, to=%s, method=%s", callTimes-1, msg, to.ServiceName(), to.Method())
	target := to.Address()
	if target != nil {
		errMsg = fmt.Sprintf("%s, remote=%s", errMsg, target.String())
	}
	if ctxErr != "" {
		errMsg = fmt.Sprintf("%s, %s", errMsg, ctxErr)
	}
	return kerrors.ErrRetry.WithCause(errors.New(errMsg))
}

func panicToErr(ctx context.Context, panicInfo interface{}, ri rpcinfo.RPCInfo) error {
	toService, toMethod := "unknown", "unknown"
	if ri != nil {
		toService, toMethod = ri.To().ServiceName(), ri.To().Method()
	}
	err := fmt.Errorf("KITEX: panic in retry, to_service=%s to_method=%s error=%v\nstack=%s",
		toService, toMethod, panicInfo, debug.Stack())
	klog.CtxErrorf(ctx, "%s", err.Error())
	return err
}

func appendErrMsg(err error, msg string) {
	if e, ok := err.(*kerrors.DetailedError); ok {
		// append no retry reason
		e.WithExtraMsg(msg)
	}
}

func recordRetryInfo(ri rpcinfo.RPCInfo, callTimes int32, lastCosts string) {
	if callTimes > 1 {
		if re := remoteinfo.AsRemoteInfo(ri.To()); re != nil {
			re.SetTag(rpcinfo.RetryTag, strconv.Itoa(int(callTimes)-1))
			// record last cost
			re.SetTag(rpcinfo.RetryLastCostTag, lastCosts)
		}
	}
}

// IsLocalRetryRequest checks whether it's a retry request by checking the RetryTag set in rpcinfo
// It's supposed to be used in client middlewares
func IsLocalRetryRequest(ctx context.Context) bool {
	ri := rpcinfo.GetRPCInfo(ctx)
	retryCountStr := ri.To().DefaultTag(rpcinfo.RetryTag, tagValueFirstTry)
	return retryCountStr != tagValueFirstTry
}

// IsRemoteRetryRequest checks whether it's a retry request by checking the TransitKey in metainfo
// It's supposed to be used in server side (handler/middleware)
func IsRemoteRetryRequest(ctx context.Context) bool {
	_, isRetry := metainfo.GetPersistentValue(ctx, TransitKey)
	return isRetry
}