github.com/cloudwego/kitex@v0.9.0/pkg/retry/retryer_test.go

/*
 * Copyright 2021 CloudWeGo Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package retry

import (
	"context"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/cloudwego/kitex/internal/test"
	"github.com/cloudwego/kitex/pkg/discovery"
	"github.com/cloudwego/kitex/pkg/kerrors"
	"github.com/cloudwego/kitex/pkg/remote"
	"github.com/cloudwego/kitex/pkg/rpcinfo"
	"github.com/cloudwego/kitex/pkg/rpcinfo/remoteinfo"
	"github.com/cloudwego/kitex/pkg/stats"
)

var (
	remoteTagKey   = "k"
	remoteTagValue = "v"
	remoteTags     = map[string]string{remoteTagKey: remoteTagValue}
)

// test new retry container
func TestNewRetryContainer(t *testing.T) {
	rc := NewRetryContainerWithCB(nil, nil)
	rc.NotifyPolicyChange(method, Policy{
		Enable:        true,
		BackupPolicy:  NewBackupPolicy(10),
		FailurePolicy: NewFailurePolicy(),
	})
	r := rc.getRetryer(genRPCInfo())
	_, ok := r.(*failureRetryer)
	test.Assert(t, ok)

	rc.NotifyPolicyChange(method, Policy{
		Enable:        false,
		BackupPolicy:  NewBackupPolicy(10),
		FailurePolicy: NewFailurePolicy(),
	})
	_, ok = r.(*failureRetryer)
	test.Assert(t, ok)
	_, allow := r.AllowRetry(context.Background())
	test.Assert(t, !allow)

	rc.NotifyPolicyChange(method, Policy{
		Enable:        true,
		BackupPolicy:  NewBackupPolicy(10),
		FailurePolicy: NewFailurePolicy(),
	})
	r = rc.getRetryer(genRPCInfo())
	_, ok = r.(*failureRetryer)
	test.Assert(t, ok)
	_, allow = r.AllowRetry(context.Background())
	test.Assert(t, allow)

	rc.NotifyPolicyChange(method, Policy{
		Enable:       true,
		Type:         1,
		BackupPolicy: NewBackupPolicy(20),
	})
	r = rc.getRetryer(genRPCInfo())
	_, ok = r.(*backupRetryer)
	test.Assert(t, ok)
	_, allow = r.AllowRetry(context.Background())
	test.Assert(t, allow)

	rc.NotifyPolicyChange(method, Policy{
		Enable:       false,
		Type:         1,
		BackupPolicy: NewBackupPolicy(20),
	})
	r = rc.getRetryer(genRPCInfo())
	_, ok = r.(*backupRetryer)
	test.Assert(t, ok)
	_, allow = r.AllowRetry(context.Background())
	test.Assert(t, !allow)

	// backupPolicy is nil
	rc = NewRetryContainer()
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   1,
	})
	msg := "new retryer[test-Backup] failed, err=newBackupRetryer failed, err=BackupPolicy is nil or retry type not match, cannot do update in backupRetryer, at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// backupPolicy config invalid
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   1,
		BackupPolicy: &BackupPolicy{
			RetryDelayMS: 0,
		},
	})
	msg = "new retryer[test-Backup] failed, err=newBackupRetryer failed, err=invalid backup request delay duration or retryTimes, at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// backupPolicy cBPolicy config invalid
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   1,
		BackupPolicy: &BackupPolicy{
			RetryDelayMS: 100,
			StopPolicy: StopPolicy{
				CBPolicy: CBPolicy{
					ErrorRate: 0.4,
				},
			},
		},
	})
	msg = "new retryer[test-Backup] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy config invalid
	rc = NewRetryContainer()
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 6,
			},
		},
	})
	msg = "new retryer[test-Failure] failed, err=newfailureRetryer failed, err=invalid failure MaxRetryTimes[6], at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy cBPolicy config invalid
	rc = NewRetryContainer()
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 5,
				CBPolicy: CBPolicy{
					ErrorRate: 0.4,
				},
			},
		},
	})
	msg = "new retryer[test-Failure] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy backOffPolicy fixedBackOffType cfg is nil
	rc = NewRetryContainerWithCB(nil, nil)
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 5,
			},
			BackOffPolicy: &BackOffPolicy{
				BackOffType: FixedBackOffType,
			},
		},
	})
	msg = "new retryer[test-Failure] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy backOffPolicy fixedBackOffType cfg invalid
	rc = NewRetryContainerWithCB(nil, nil)
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 5,
			},
			BackOffPolicy: &BackOffPolicy{
				BackOffType: FixedBackOffType,
				CfgItems: map[BackOffCfgKey]float64{
					FixMSBackOffCfgKey: 0,
				},
			},
		},
	})
	msg = "new retryer[test-Failure] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy backOffPolicy randomBackOffType cfg is nil
	rc = NewRetryContainerWithCB(nil, nil)
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 5,
			},
			BackOffPolicy: &BackOffPolicy{
				BackOffType: RandomBackOffType,
			},
		},
	})
	msg = "new retryer[test-Failure] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy backOffPolicy randomBackOffType cfg invalid
	rc = NewRetryContainerWithCB(nil, nil)
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 5,
			},
			BackOffPolicy: &BackOffPolicy{
				BackOffType: RandomBackOffType,
				CfgItems: map[BackOffCfgKey]float64{
					MinMSBackOffCfgKey: 20,
					MaxMSBackOffCfgKey: 10,
				},
			},
		},
	})
	msg = "new retryer[test-Failure] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// failurePolicy backOffPolicy randomBackOffType normal
	rc = NewRetryContainerWithCB(nil, nil)
	rc.NotifyPolicyChange(method, Policy{
		Enable: true,
		Type:   0,
		FailurePolicy: &FailurePolicy{
			StopPolicy: StopPolicy{
				MaxRetryTimes: 5,
			},
			BackOffPolicy: &BackOffPolicy{
				BackOffType: RandomBackOffType,
				CfgItems: map[BackOffCfgKey]float64{
					MinMSBackOffCfgKey: 10,
					MaxMSBackOffCfgKey: 20,
				},
			},
		},
	})
	msg = "new retryer[test-Failure] at "
	test.Assert(t, rc.msg[:len(msg)] == msg)

	// test init invalid case
	err := rc.Init(nil, nil)
	test.Assert(t, err == nil, err)
	err = rc.Init(map[string]Policy{Wildcard: {
		Enable: true,
		Type:   1,
		BackupPolicy: &BackupPolicy{
			RetryDelayMS: 0,
		},
	}}, nil)
	test.Assert(t, err != nil, err)

	rc.DeletePolicy(method)
	r = rc.getRetryer(genRPCInfo())
	test.Assert(t, r == nil)
}

// test container dump
func TestContainer_Dump(t *testing.T) {
	// test backupPolicy dump
	rc := NewRetryContainerWithCB(nil, nil)
	methodPolicies := map[string]Policy{
		method: {
			Enable:       true,
			Type:         1,
			BackupPolicy: NewBackupPolicy(20),
		},
	}
	rc.InitWithPolicies(methodPolicies)
	err := rc.Init(map[string]Policy{Wildcard: {
		Enable:       true,
		Type:         1,
		BackupPolicy: NewBackupPolicy(20),
	}}, nil)
	test.Assert(t, err == nil, err)
	rcDump, ok := rc.Dump().(map[string]interface{})
	test.Assert(t, ok)
	hasCodeCfg, err := jsoni.MarshalToString(rcDump["has_code_cfg"])
	test.Assert(t, err == nil, err)
	test.Assert(t, hasCodeCfg == "true", hasCodeCfg)
	testStr, err := jsoni.MarshalToString(rcDump["test"])
	msg := `{"backupRequest":{"retry_delay_ms":20,"stop_policy":{"max_retry_times":1,"max_duration_ms":0,"disable_chain_stop":false,"ddl_stop":false,"cb_policy":{"error_rate":0.1}},"retry_same_node":false},"enable":true}`
	test.Assert(t, err == nil, err)
	test.Assert(t, testStr == msg)

	// test failurePolicy dump
	rc = NewRetryContainerWithCB(nil, nil)
	methodPolicies = map[string]Policy{
		method: {
			Enable:        true,
			Type:          FailureType,
			FailurePolicy: NewFailurePolicy(),
		},
	}
	rc.InitWithPolicies(methodPolicies)
	err = rc.Init(map[string]Policy{Wildcard: {
		Enable:        true,
		Type:          FailureType,
		FailurePolicy: NewFailurePolicy(),
	}}, nil)
	test.Assert(t, err == nil, err)
	rcDump, ok = rc.Dump().(map[string]interface{})
	test.Assert(t, ok)
	hasCodeCfg, err = jsoni.MarshalToString(rcDump["has_code_cfg"])
	test.Assert(t, err == nil, err)
	test.Assert(t, hasCodeCfg == "true")
	testStr, err = jsoni.MarshalToString(rcDump["test"])
	msg = `{"enable":true,"failure_retry":{"stop_policy":{"max_retry_times":2,"max_duration_ms":0,"disable_chain_stop":false,"ddl_stop":false,"cb_policy":{"error_rate":0.1}},"backoff_policy":{"backoff_type":"none"},"retry_same_node":false,"extra":""},"specified_result_retry":{"error_retry":false,"resp_retry":false}}`
	test.Assert(t, err == nil, err)
	test.Assert(t, testStr == msg, testStr)
}

// test FailurePolicy call
func TestFailurePolicyCall(t *testing.T) {
	// call while rpc timeout
	ctx := context.Background()
	rc := NewRetryContainer()
	failurePolicy := NewFailurePolicy()
	failurePolicy.BackOffPolicy.BackOffType = FixedBackOffType
	failurePolicy.BackOffPolicy.CfgItems = map[BackOffCfgKey]float64{
		FixMSBackOffCfgKey: 100.0,
	}
	failurePolicy.StopPolicy.MaxDurationMS = 100
	err := rc.Init(map[string]Policy{Wildcard: {
		Enable:        true,
		Type:          0,
		FailurePolicy: failurePolicy,
	}}, nil)
	test.Assert(t, err == nil, err)
	ri := genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	_, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		return ri, nil, kerrors.ErrRPCTimeout
	}, ri, nil)
	test.Assert(t, err != nil, err)
	test.Assert(t, !ok)

	// call normal
	failurePolicy.StopPolicy.MaxDurationMS = 0
	err = rc.Init(map[string]Policy{Wildcard: {
		Enable:        true,
		Type:          0,
		FailurePolicy: failurePolicy,
	}}, nil)
	test.Assert(t, err == nil, err)
	_, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, ok)
}

// test retry with one time policy
func TestRetryWithOneTimePolicy(t *testing.T) {
	// call while rpc timeout and exceed MaxDurationMS cause BackOffPolicy is wait fix 100ms, it is invalid config
	failurePolicy := NewFailurePolicy()
	failurePolicy.BackOffPolicy.BackOffType = FixedBackOffType
	failurePolicy.BackOffPolicy.CfgItems = map[BackOffCfgKey]float64{
		FixMSBackOffCfgKey: 100.0,
	}
	failurePolicy.StopPolicy.MaxDurationMS = 100
	p := Policy{
		Enable:        true,
		Type:          0,
		FailurePolicy: failurePolicy,
	}
	ri := genRPCInfo()
	ctx := rpcinfo.NewCtxWithRPCInfo(context.Background(), ri)
	_, ok, err := NewRetryContainer().WithRetryIfNeeded(ctx, &p, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		return ri, nil, kerrors.ErrRPCTimeout
	}, ri, nil)
	test.Assert(t, err != nil, err)
	test.Assert(t, !ok)

	// call no MaxDurationMS limit, the retry will success
	failurePolicy.StopPolicy.MaxDurationMS = 0
	var callTimes int32
	ctx = rpcinfo.NewCtxWithRPCInfo(context.Background(), genRPCInfo())
	_, ok, err = NewRetryContainer().WithRetryIfNeeded(ctx, &p, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		if atomic.LoadInt32(&callTimes) == 0 {
			atomic.AddInt32(&callTimes, 1)
			return ri, nil, kerrors.ErrRPCTimeout
		}
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, !ok)

	// call backup request
	p = BuildBackupRequest(NewBackupPolicy(10))
	ctx = rpcinfo.NewCtxWithRPCInfo(context.Background(), genRPCInfo())
	callTimes = 0
	_, ok, err = NewRetryContainer().WithRetryIfNeeded(ctx, &p, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		if atomic.LoadInt32(&callTimes) == 0 || atomic.LoadInt32(&callTimes) == 1 {
			atomic.AddInt32(&callTimes, 1)
			time.Sleep(time.Millisecond * 100)
		}
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, !ok)
	test.Assert(t, atomic.LoadInt32(&callTimes) == 2)
}

// test specified error to retry
func TestSpecifiedErrorRetry(t *testing.T) {
	retryWithTransError := func(callTimes int32) RPCCallFunc {
		// fails for the first call if callTimes is initialized to 0
		return func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
			newVal := atomic.AddInt32(&callTimes, 1)
			if newVal == 1 {
				return genRPCInfo(), nil, remote.NewTransErrorWithMsg(1000, "mock")
			} else {
				return genRPCInfoWithRemoteTag(remoteTags), nil, nil
			}
		}
	}
	ri := genRPCInfo()
	ctx := rpcinfo.NewCtxWithRPCInfo(context.Background(), ri)

	// case1: specified method retry with error
	t.Run("case1", func(t *testing.T) {
		rc := NewRetryContainer()
		shouldResultRetry := &ShouldResultRetry{ErrorRetry: func(err error, ri rpcinfo.RPCInfo) bool {
			if ri.To().Method() == method {
				if te, ok := err.(*remote.TransError); ok && te.TypeID() == 1000 {
					return true
				}
			}
			return false
		}}
		err := rc.Init(map[string]Policy{Wildcard: BuildFailurePolicy(NewFailurePolicy())}, shouldResultRetry)
		test.Assert(t, err == nil, err)
		ri, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, retryWithTransError(0), ri, nil)
		test.Assert(t, err == nil, err)
		test.Assert(t, !ok)
		v, ok := ri.To().Tag(remoteTagKey)
		test.Assert(t, ok)
		test.Assert(t, v == remoteTagValue)
	})

	// case2: specified method retry with error, but use backup request config cannot be effective
	t.Run("case2", func(t *testing.T) {
		shouldResultRetry := &ShouldResultRetry{ErrorRetry: func(err error, ri rpcinfo.RPCInfo) bool {
			if ri.To().Method() == method {
				if te, ok := err.(*remote.TransError); ok && te.TypeID() == 1000 {
					return true
				}
			}
			return false
		}}
		rc := NewRetryContainer()
		err := rc.Init(map[string]Policy{Wildcard: BuildBackupRequest(NewBackupPolicy(10))}, shouldResultRetry)
		test.Assert(t, err == nil, err)
		ri = genRPCInfo()
		_, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, retryWithTransError(0), ri, nil)
		test.Assert(t, err != nil, err)
		test.Assert(t, !ok)
	})

	// case3: specified method retry with error, but method not match
	t.Run("case3", func(t *testing.T) {
		shouldResultRetry := &ShouldResultRetry{ErrorRetry: func(err error, ri rpcinfo.RPCInfo) bool {
			if ri.To().Method() != method {
				if te, ok := err.(*remote.TransError); ok && te.TypeID() == 1000 {
					return true
				}
			}
			return false
		}}
		rc := NewRetryContainer()
		err := rc.Init(map[string]Policy{method: BuildFailurePolicy(NewFailurePolicy())}, shouldResultRetry)
		test.Assert(t, err == nil, err)
		ri = genRPCInfo()
		ri, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, retryWithTransError(0), ri, nil)
		test.Assert(t, err != nil)
		test.Assert(t, !ok)
		_, ok = ri.To().Tag(remoteTagKey)
		test.Assert(t, !ok)
	})

	// case4: all error retry
	t.Run("case4", func(t *testing.T) {
		rc := NewRetryContainer()
		p := BuildFailurePolicy(NewFailurePolicyWithResultRetry(AllErrorRetry()))
		ri = genRPCInfo()
		ri, ok, err := rc.WithRetryIfNeeded(ctx, &p, retryWithTransError(0), ri, nil)
		test.Assert(t, err == nil, err)
		test.Assert(t, !ok)
		v, ok := ri.To().Tag(remoteTagKey)
		test.Assert(t, ok)
		test.Assert(t, v == remoteTagValue)
	})
}

// test specified resp to retry
func TestSpecifiedRespRetry(t *testing.T) {
	retryResult := &mockResult{}
	retryResp := mockResp{
		code: 500,
		msg:  "retry",
	}
	noRetryResp := mockResp{
		code: 0,
		msg:  "noretry",
	}
	var callTimes int32
	retryWithResp := func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		newVal := atomic.AddInt32(&callTimes, 1)
		if newVal == 1 {
			retryResult.SetResult(retryResp)
			return genRPCInfo(), retryResult, nil
		} else {
			retryResult.SetResult(noRetryResp)
			return genRPCInfoWithRemoteTag(remoteTags), retryResult, nil
		}
	}
	ctx := context.Background()
	ri := genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	rc := NewRetryContainer()
	// case1: specified method retry with resp
	shouldResultRetry := &ShouldResultRetry{RespRetry: func(resp interface{}, ri rpcinfo.RPCInfo) bool {
		if ri.To().Method() == method {
			if r, ok := resp.(*mockResult); ok && r.GetResult() == retryResp {
				return true
			}
		}
		return false
	}}
	err := rc.Init(map[string]Policy{Wildcard: BuildFailurePolicy(NewFailurePolicy())}, shouldResultRetry)
	test.Assert(t, err == nil, err)
	ri, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, retryWithResp, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, retryResult.GetResult() == noRetryResp, retryResult)
	test.Assert(t, !ok)
	v, ok := ri.To().Tag(remoteTagKey)
	test.Assert(t, ok)
	test.Assert(t, v == remoteTagValue)

	// case2 specified method retry with resp, but use backup request config cannot be effective
	atomic.StoreInt32(&callTimes, 0)
	rc = NewRetryContainer()
	err = rc.Init(map[string]Policy{Wildcard: BuildBackupRequest(NewBackupPolicy(100))}, shouldResultRetry)
	test.Assert(t, err == nil, err)
	ri = genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(context.Background(), ri)
	_, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, retryWithResp, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, retryResult.GetResult() == retryResp, retryResp)
	test.Assert(t, !ok)

	// case3: specified method retry with resp, but method not match
	atomic.StoreInt32(&callTimes, 0)
	shouldResultRetry = &ShouldResultRetry{RespRetry: func(resp interface{}, ri rpcinfo.RPCInfo) bool {
		if ri.To().Method() != method {
			if r, ok := resp.(*mockResult); ok && r.GetResult() == retryResp {
				return true
			}
		}
		return false
	}}
	rc = NewRetryContainer()
	err = rc.Init(map[string]Policy{method: BuildFailurePolicy(NewFailurePolicy())}, shouldResultRetry)
	test.Assert(t, err == nil, err)
	ri = genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(context.Background(), ri)
	ri, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, retryWithResp, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, retryResult.GetResult() == retryResp, retryResult)
	test.Assert(t, ok)
	_, ok = ri.To().Tag(remoteTagKey)
	test.Assert(t, !ok)
}

// test different method use different retry policy
func TestDifferentMethodConfig(t *testing.T) {
	var callTimes int32
	methodRetryer := make(map[string]Retryer)
	var lock sync.Mutex
	rpcCall := func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		atomic.AddInt32(&callTimes, 1)
		ri := rpcinfo.GetRPCInfo(ctx)
		if ri.To().Method() == method {
			lock.Lock()
			methodRetryer[ri.To().Method()] = r
			lock.Unlock()
			// specified method mock return error and do error retry
			if atomic.LoadInt32(&callTimes) == 1 {
				return genRPCInfo(), nil, remote.NewTransErrorWithMsg(1000, "mock")
			}
		} else {
			lock.Lock()
			methodRetryer[ri.To().Method()] = r
			lock.Unlock()
			if atomic.LoadInt32(&callTimes) == 1 {
				// other method do backup request
				time.Sleep(20 * time.Millisecond)
				return genRPCInfo(), nil, kerrors.ErrRPCTimeout
			}
		}
		return genRPCInfo(), nil, nil
	}
	rc := NewRetryContainer()
	shouldResultRetry := &ShouldResultRetry{ErrorRetry: func(err error, ri rpcinfo.RPCInfo) bool {
		if ri.To().Method() == method {
			if te, ok := err.(*remote.TransError); ok && te.TypeID() == 1000 {
				return true
			}
		}
		return false
	}}
	err := rc.Init(map[string]Policy{
		method:   BuildFailurePolicy(NewFailurePolicy()),
		Wildcard: BuildBackupRequest(NewBackupPolicy(10)),
	}, shouldResultRetry)
	test.Assert(t, err == nil, err)

	// case1: test method do error retry
	ri := genRPCInfo()
	ctx := rpcinfo.NewCtxWithRPCInfo(context.Background(), ri)
	_, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, rpcCall, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, !ok)
	lock.Lock()
	_, ok = methodRetryer[method].(*failureRetryer)
	lock.Unlock()
	test.Assert(t, ok)

	// case2: other method do backup request
	method2 := "method2"
	to := remoteinfo.NewRemoteInfo(&rpcinfo.EndpointBasicInfo{Method: method2}, method2).ImmutableView()
	ri = rpcinfo.NewRPCInfo(to, to, rpcinfo.NewInvocation("", method2), rpcinfo.NewRPCConfig(), rpcinfo.NewRPCStats())
	ctx = rpcinfo.NewCtxWithRPCInfo(context.Background(), ri)
	_, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, rpcCall, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, !ok)
	lock.Lock()
	_, ok = methodRetryer[method2].(*backupRetryer)
	lock.Unlock()
	test.Assert(t, ok)
}

func TestResultRetryWithPolicyChange(t *testing.T) {
	rc := NewRetryContainer()
	shouldResultRetry := &ShouldResultRetry{ErrorRetry: func(err error, ri rpcinfo.RPCInfo) bool {
		if ri.To().Method() == method {
			if te, ok := err.(*remote.TransError); ok && te.TypeID() == 1000 {
				return true
			}
		}
		return false
	}}
	err := rc.Init(nil, shouldResultRetry)
	test.Assert(t, err == nil, err)

	// case 1: first time trigger NotifyPolicyChange, the `initRetryer` will be executed, check if the ShouldResultRetry is not nil
	rc.NotifyPolicyChange(Wildcard, BuildFailurePolicy(NewFailurePolicy()))
	r := rc.getRetryer(genRPCInfo())
	fr, ok := r.(*failureRetryer)
	test.Assert(t, ok)
	test.Assert(t, fr.policy.ShouldResultRetry == shouldResultRetry)

	// case 2: second time trigger NotifyPolicyChange, the `UpdatePolicy` will be executed, check if the ShouldResultRetry is not nil
	rc.NotifyPolicyChange(Wildcard, BuildFailurePolicy(NewFailurePolicy()))
	r = rc.getRetryer(genRPCInfo())
	fr, ok = r.(*failureRetryer)
	test.Assert(t, ok)
	test.Assert(t, fr.policy.ShouldResultRetry == shouldResultRetry)
}

// test BackupPolicy call while rpcTime > delayTime
func TestBackupPolicyCall(t *testing.T) {
	ctx := context.Background()
	rc := NewRetryContainer()
	err := rc.Init(map[string]Policy{Wildcard: {
		Enable: true,
		Type:   1,
		BackupPolicy: &BackupPolicy{
			RetryDelayMS: 30,
			StopPolicy: StopPolicy{
				MaxRetryTimes:    2,
				DisableChainStop: false,
				CBPolicy: CBPolicy{
					ErrorRate: defaultCBErrRate,
				},
			},
		},
	}}, nil)
	test.Assert(t, err == nil, err)

	callTimes := int32(0)
	firstRI := genRPCInfo()
	secondRI := genRPCInfoWithRemoteTag(remoteTags)
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, firstRI)
	ri, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		atomic.AddInt32(&callTimes, 1)
		if atomic.LoadInt32(&callTimes) == 1 {
			// mock timeout for the first request and get the response of the backup request.
			time.Sleep(time.Millisecond * 50)
			return firstRI, nil, nil
		}
		return secondRI, nil, nil
	}, firstRI, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, atomic.LoadInt32(&callTimes) == 2)
	test.Assert(t, !ok)
	v, ok := ri.To().Tag(remoteTagKey)
	test.Assert(t, ok)
	test.Assert(t, v == remoteTagValue)
}

// test policy noRetry call
func TestPolicyNoRetryCall(t *testing.T) {
	ctx := context.Background()
	rc := NewRetryContainer()

	// case 1(default): no retry policy
	// no retry policy, call success
	var callTimes int32
	ri := genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	_, ok, err := rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		atomic.AddInt32(&callTimes, 1)
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, atomic.LoadInt32(&callTimes) == 1)
	test.Assert(t, ok)

	// no retry policy, call rpcTimeout
	atomic.StoreInt32(&callTimes, 0)
	ri = genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	_, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		atomic.AddInt32(&callTimes, 1)
		if atomic.LoadInt32(&callTimes) == 1 {
			return ri, nil, kerrors.ErrRPCTimeout
		}
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, kerrors.IsTimeoutError(err), err)
	test.Assert(t, atomic.LoadInt32(&callTimes) == 1)
	test.Assert(t, !ok)

	// case 2: setup retry policy, but not satisfy retry condition eg: circuit, retry times == 0, chain stop, ddl
	// failurePolicy DDLStop rpcTimeOut
	failurePolicy := NewFailurePolicy()
	failurePolicy.WithDDLStop()
	RegisterDDLStop(func(ctx context.Context, policy StopPolicy) (bool, string) {
		return true, "TestDDLStop"
	})
	err = rc.Init(map[string]Policy{Wildcard: {
		Enable:        true,
		Type:          0,
		FailurePolicy: failurePolicy,
	}}, nil)
	test.Assert(t, err == nil, err)
	atomic.StoreInt32(&callTimes, 0)
	ri = genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	_, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		atomic.AddInt32(&callTimes, 1)
		if atomic.LoadInt32(&callTimes) == 1 {
			return ri, nil, kerrors.ErrRPCTimeout
		}
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, kerrors.IsTimeoutError(err), err)
	test.Assert(t, atomic.LoadInt32(&callTimes) == 1)
	test.Assert(t, !ok)

	// backupPolicy MaxRetryTimes = 0
	backupPolicy := NewBackupPolicy(20)
	backupPolicy.StopPolicy.MaxRetryTimes = 0
	err = rc.Init(map[string]Policy{Wildcard: {
		Enable:       true,
		Type:         1,
		BackupPolicy: backupPolicy,
	}}, nil)
	test.Assert(t, err == nil, err)
	atomic.StoreInt32(&callTimes, 0)
	ri = genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	_, ok, err = rc.WithRetryIfNeeded(ctx, &Policy{}, func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		atomic.AddInt32(&callTimes, 1)
		time.Sleep(time.Millisecond * 100)
		return ri, nil, nil
	}, ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, atomic.LoadInt32(&callTimes) == 1)
	test.Assert(t, ok)
}

func retryCall(callTimes *int32, firstRI rpcinfo.RPCInfo, backup bool) RPCCallFunc {
	// prevRI represents a value of rpcinfo.RPCInfo type.
	var prevRI atomic.Value
	return func(ctx context.Context, r Retryer) (rpcinfo.RPCInfo, interface{}, error) {
		currCallTimes := int(atomic.AddInt32(callTimes, 1))
		cRI := firstRI
		if currCallTimes > 1 {
			cRI = genRPCInfoWithFirstStats(firstRI)
			cRI.Stats().Record(ctx, stats.RPCFinish, stats.StatusInfo, "")
			remoteInfo := remoteinfo.AsRemoteInfo(cRI.To())
			remoteInfo.SetInstance(discovery.NewInstance("tcp", "10.20.30.40:8888", 10, nil))
			if prevRI.Load() == nil {
				prevRI.Store(firstRI)
			}
			r.Prepare(ctx, prevRI.Load().(rpcinfo.RPCInfo), cRI)
			prevRI.Store(cRI)
			return cRI, nil, nil
		} else {
			if backup {
				time.Sleep(20 * time.Millisecond)
				return cRI, nil, nil
			} else {
				return cRI, nil, kerrors.ErrRPCTimeout
			}
		}
	}
}

func TestFailureRetryWithRPCInfo(t *testing.T) {
	// failure retry
	ctx := context.Background()
	rc := NewRetryContainer()

	ri := genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	rpcinfo.Record(ctx, ri, stats.RPCStart, nil)

	// call with retry policy
	var callTimes int32
	policy := BuildFailurePolicy(NewFailurePolicy())
	ri, ok, err := rc.WithRetryIfNeeded(ctx, &policy, retryCall(&callTimes, ri, false), ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, !ok)
	test.Assert(t, ri.Stats().GetEvent(stats.RPCStart).Status() == stats.StatusInfo)
	test.Assert(t, ri.Stats().GetEvent(stats.RPCFinish).Status() == stats.StatusInfo)
	test.Assert(t, ri.To().Address().String() == "10.20.30.40:8888")
	test.Assert(t, atomic.LoadInt32(&callTimes) == 2)
}

func TestBackupRetryWithRPCInfo(t *testing.T) {
	// backup retry
	ctx := context.Background()
	rc := NewRetryContainer()

	ri := genRPCInfo()
	ctx = rpcinfo.NewCtxWithRPCInfo(ctx, ri)
	rpcinfo.Record(ctx, ri, stats.RPCStart, nil)

	// call with retry policy
	var callTimes int32
	policy := BuildBackupRequest(NewBackupPolicy(10))
	ri, ok, err := rc.WithRetryIfNeeded(ctx, &policy, retryCall(&callTimes, ri, true), ri, nil)
	test.Assert(t, err == nil, err)
	test.Assert(t, !ok)
	test.Assert(t, ri.Stats().GetEvent(stats.RPCStart).Status() == stats.StatusInfo)
	test.Assert(t, ri.Stats().GetEvent(stats.RPCFinish).Status() == stats.StatusInfo)
	test.Assert(t, ri.To().Address().String() == "10.20.30.40:8888")
	test.Assert(t, atomic.LoadInt32(&callTimes) == 2)
}

type mockResult struct {
	result mockResp
	sync.RWMutex
}

type mockResp struct {
	code int
	msg  string
}

func (r *mockResult) GetResult() interface{} {
	r.RLock()
	defer r.RUnlock()
	return r.result
}

func (r *mockResult) SetResult(ret mockResp) {
	r.Lock()
	defer r.Unlock()
	r.result = ret
}

func TestNewRetryContainerWithOptions(t *testing.T) {
	t.Run("no_option", func(t *testing.T) {
		rc := NewRetryContainer()
		test.Assertf(t, rc.cbContainer.cbSuite != nil, "cb_suite nil")
		test.Assertf(t, rc.cbContainer.cbStat == true, "cb_stat not true")
	})

	t.Run("percentage_limit", func(t *testing.T) {
		rc := NewRetryContainer(WithContainerEnablePercentageLimit())
		test.Assertf(t, rc.cbContainer.enablePercentageLimit == true, "percentage_limit not true")
	})

	t.Run("percentage_limit&&cbOptions", func(t *testing.T) {
		cbSuite := newCBSuite()
		rc := NewRetryContainer(
			WithContainerEnablePercentageLimit(),
			WithContainerCBSuite(cbSuite),
			WithContainerCBControl(cbSuite.ServiceControl()),
			WithContainerCBPanel(cbSuite.ServicePanel()),
		)
		test.Assertf(t, rc.cbContainer.enablePercentageLimit == true, "percentage_limit not true")
		test.Assertf(t, rc.cbContainer.cbSuite != cbSuite, "cbSuite not ignored")
		test.Assertf(t, rc.cbContainer.cbCtl != cbSuite.ServiceControl(), "cbCtl not ignored")
		test.Assertf(t, rc.cbContainer.cbPanel != cbSuite.ServicePanel(), "cbPanel not ignored")
	})

	t.Run("cb_stat", func(t *testing.T) {
		rc := NewRetryContainer(WithContainerCBStat())
		test.Assertf(t, rc.cbContainer.cbStat == true, "cb_stat not true")
	})

	t.Run("cb_suite", func(t *testing.T) {
		cbs := newCBSuite()
		rc := NewRetryContainer(WithContainerCBSuite(cbs))
		test.Assert(t, rc.cbContainer.cbSuite == cbs, "cbSuite expected %p, got %p", cbs, rc.cbContainer.cbSuite)
	})

	t.Run("cb_control&cb_panel", func(t *testing.T) {
		cbs := newCBSuite()
		rc := NewRetryContainer(
			WithContainerCBControl(cbs.ServiceControl()),
			WithContainerCBPanel(cbs.ServicePanel()))
		test.Assert(t, rc.cbContainer.cbCtl == cbs.ServiceControl(), "cbControl not match")
		test.Assert(t, rc.cbContainer.cbPanel == cbs.ServicePanel(), "cbPanel not match")
	})
}

func TestNewRetryContainerWithCBStat(t *testing.T) {
	cbs := newCBSuite()
	rc := NewRetryContainerWithCBStat(cbs.ServiceControl(), cbs.ServicePanel())
	test.Assert(t, rc.cbContainer.cbCtl == cbs.ServiceControl(), "cbControl not match")
	test.Assert(t, rc.cbContainer.cbPanel == cbs.ServicePanel(), "cbPanel not match")
	test.Assertf(t, rc.cbContainer.cbStat == true, "cb_stat not true")
	rc.Close()
}

func TestNewRetryContainerWithCB(t *testing.T) {
	cbs := newCBSuite()
	rc := NewRetryContainerWithCB(cbs.ServiceControl(), cbs.ServicePanel())
	test.Assert(t, rc.cbContainer.cbCtl == cbs.ServiceControl(), "cbControl not match")
	test.Assert(t, rc.cbContainer.cbPanel == cbs.ServicePanel(), "cbPanel not match")
	test.Assertf(t, rc.cbContainer.cbStat == false, "cb_stat not false")
	rc.Close()
}