github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/cdc/api/v2/changefeed_test.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package v2
    15  
    16  import (
    17  	"bytes"
    18  	"context"
    19  	"encoding/json"
    20  	"fmt"
    21  	"net/http"
    22  	"net/http/httptest"
    23  	"path/filepath"
    24  	"sort"
    25  	"testing"
    26  
    27  	"github.com/golang/mock/gomock"
    28  	tidbkv "github.com/pingcap/tidb/pkg/kv"
    29  	mock_capture "github.com/pingcap/tiflow/cdc/capture/mock"
    30  	"github.com/pingcap/tiflow/cdc/controller"
    31  	mock_controller "github.com/pingcap/tiflow/cdc/controller/mock"
    32  	"github.com/pingcap/tiflow/cdc/model"
    33  	mock_owner "github.com/pingcap/tiflow/cdc/owner/mock"
    34  	"github.com/pingcap/tiflow/pkg/config"
    35  	cerrors "github.com/pingcap/tiflow/pkg/errors"
    36  	"github.com/pingcap/tiflow/pkg/etcd"
    37  	mock_etcd "github.com/pingcap/tiflow/pkg/etcd/mock"
    38  	"github.com/pingcap/tiflow/pkg/upstream"
    39  	"github.com/pingcap/tiflow/pkg/util"
    40  	"github.com/stretchr/testify/require"
    41  	pd "github.com/tikv/pd/client"
    42  	clientv3 "go.etcd.io/etcd/client/v3"
    43  	"go.etcd.io/etcd/tests/v3/integration"
    44  )
    45  
    46  var (
    47  	changeFeedID  = model.ChangeFeedID{Namespace: "abc", ID: "test-changeFeed"}
    48  	blackholeSink = "blackhole://"
    49  	mysqlSink     = "mysql://root:123456@127.0.0.1:3306"
    50  )
    51  
    52  func TestCreateChangefeed(t *testing.T) {
    53  	t.Parallel()
    54  	create := testCase{url: "/api/v2/changefeeds", method: "POST"}
    55  
    56  	pdClient := &mockPDClient{}
    57  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
    58  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
    59  	etcdClient := mock_etcd.NewMockCDCEtcdClient(gomock.NewController(t))
    60  	apiV2 := NewOpenAPIV2ForTest(cp, helpers)
    61  	router := newRouter(apiV2)
    62  	integration.BeforeTestExternal(t)
    63  	testEtcdCluster := integration.NewClusterV3(
    64  		t, &integration.ClusterConfig{Size: 2},
    65  	)
    66  	defer testEtcdCluster.Terminate(t)
    67  
    68  	mockUpManager := upstream.NewManager4Test(pdClient)
    69  	etcdClient.EXPECT().
    70  		GetEnsureGCServiceID(gomock.Any()).
    71  		Return(etcd.GcServiceIDForTest()).AnyTimes()
    72  	cp.EXPECT().GetEtcdClient().Return(etcdClient).AnyTimes()
    73  	cp.EXPECT().GetUpstreamManager().Return(mockUpManager, nil).AnyTimes()
    74  	cp.EXPECT().IsReady().Return(true).AnyTimes()
    75  	cp.EXPECT().IsController().Return(true).AnyTimes()
    76  	ctrl := mock_controller.NewMockController(gomock.NewController(t))
    77  	cp.EXPECT().GetController().Return(ctrl, nil).AnyTimes()
    78  
    79  	// case 1: json format mismatches with the spec.
    80  	errConfig := struct {
    81  		ID        string `json:"changefeed_id"`
    82  		Namespace string `json:"namespace"`
    83  		SinkURI   string `json:"sink_uri"`
    84  		PDAddrs   string `json:"pd_addrs"` // should be an array
    85  	}{
    86  		ID:        changeFeedID.ID,
    87  		Namespace: changeFeedID.Namespace,
    88  		SinkURI:   blackholeSink,
    89  		PDAddrs:   "http://127.0.0.1:2379",
    90  	}
    91  	bodyErr, err := json.Marshal(&errConfig)
    92  	require.Nil(t, err)
    93  	w := httptest.NewRecorder()
    94  	req, _ := http.NewRequestWithContext(context.Background(),
    95  		create.method, create.url, bytes.NewReader(bodyErr))
    96  	router.ServeHTTP(w, req)
    97  	require.Equal(t, http.StatusBadRequest, w.Code)
    98  	respErr := model.HTTPError{}
    99  	err = json.NewDecoder(w.Body).Decode(&respErr)
   100  	require.Nil(t, err)
   101  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   102  
   103  	cfConfig := struct {
   104  		ID        string   `json:"changefeed_id"`
   105  		Namespace string   `json:"namespace"`
   106  		SinkURI   string   `json:"sink_uri"`
   107  		PDAddrs   []string `json:"pd_addrs"`
   108  	}{
   109  		ID:        changeFeedID.ID,
   110  		Namespace: changeFeedID.Namespace,
   111  		SinkURI:   blackholeSink,
   112  		PDAddrs:   []string{},
   113  	}
   114  	body, err := json.Marshal(&cfConfig)
   115  	require.Nil(t, err)
   116  
   117  	// case 2: getPDClient failed, it may happen with wrong PDAddrs
   118  	helpers.EXPECT().
   119  		getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).
   120  		Return(nil, cerrors.ErrAPIGetPDClientFailed).Times(1)
   121  
   122  	w = httptest.NewRecorder()
   123  	req, _ = http.NewRequestWithContext(context.Background(), create.method,
   124  		create.url, bytes.NewReader(body))
   125  	router.ServeHTTP(w, req)
   126  	require.Equal(t, http.StatusInternalServerError, w.Code)
   127  	respErr = model.HTTPError{}
   128  	err = json.NewDecoder(w.Body).Decode(&respErr)
   129  	require.Nil(t, err)
   130  	require.Contains(t, respErr.Code, "ErrAPIGetPDClientFailed")
   131  
   132  	// case 3: failed to create TiStore
   133  	helpers.EXPECT().
   134  		getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).
   135  		Return(pdClient, nil).AnyTimes()
   136  	helpers.EXPECT().
   137  		createTiStore(gomock.Any(), gomock.Any()).
   138  		Return(nil, cerrors.ErrNewStore).
   139  		Times(1)
   140  	cfConfig.PDAddrs = []string{"http://127.0.0.1:2379", "http://127.0.0.1:2382"}
   141  	body, err = json.Marshal(&cfConfig)
   142  	require.Nil(t, err)
   143  	w = httptest.NewRecorder()
   144  	req, _ = http.NewRequestWithContext(context.Background(), create.method,
   145  		create.url, bytes.NewReader(body))
   146  	router.ServeHTTP(w, req)
   147  	respErr = model.HTTPError{}
   148  	err = json.NewDecoder(w.Body).Decode(&respErr)
   149  	require.Nil(t, err)
   150  	require.Contains(t, respErr.Code, "ErrNewStore")
   151  	require.Equal(t, http.StatusInternalServerError, w.Code)
   152  
   153  	// case 4: failed to verify tables
   154  	helpers.EXPECT().
   155  		createTiStore(gomock.Any(), gomock.Any()).
   156  		Return(nil, nil).
   157  		AnyTimes()
   158  	helpers.EXPECT().
   159  		verifyCreateChangefeedConfig(gomock.Any(), gomock.Any(), gomock.Any(),
   160  			gomock.Any(), gomock.Any(), gomock.Any()).
   161  		Return(nil, cerrors.ErrSinkURIInvalid.GenWithStackByArgs(
   162  			"sink_uri is empty, can't not create a changefeed without sink_uri"))
   163  	cfConfig.SinkURI = ""
   164  	body, err = json.Marshal(&cfConfig)
   165  	require.Nil(t, err)
   166  
   167  	w = httptest.NewRecorder()
   168  	req, _ = http.NewRequestWithContext(context.Background(), create.method,
   169  		create.url, bytes.NewReader(body))
   170  	router.ServeHTTP(w, req)
   171  	respErr = model.HTTPError{}
   172  	err = json.NewDecoder(w.Body).Decode(&respErr)
   173  	require.Nil(t, err)
   174  	require.Contains(t, respErr.Code, "ErrSinkURIInvalid")
   175  	require.Equal(t, http.StatusBadRequest, w.Code)
   176  
   177  	// case 5:
   178  	helpers.EXPECT().
   179  		getEtcdClient(gomock.Any(), gomock.Any()).
   180  		Return(testEtcdCluster.RandClient(), nil)
   181  	helpers.EXPECT().getVerifiedTables(gomock.Any(), gomock.Any(), gomock.Any(),
   182  		gomock.Any(), gomock.Any(), gomock.Any()).
   183  		Return(nil, nil, nil).
   184  		AnyTimes()
   185  	helpers.EXPECT().
   186  		verifyCreateChangefeedConfig(gomock.Any(), gomock.Any(), gomock.Any(),
   187  			gomock.Any(), gomock.Any(), gomock.Any()).
   188  		DoAndReturn(func(ctx context.Context,
   189  			cfg *ChangefeedConfig,
   190  			pdClient pd.Client,
   191  			ctrl controller.Controller,
   192  			ensureGCServiceID string,
   193  			kvStorage tidbkv.Storage,
   194  		) (*model.ChangeFeedInfo, error) {
   195  			require.EqualValues(t, cfg.ID, changeFeedID.ID)
   196  			require.EqualValues(t, cfg.Namespace, changeFeedID.Namespace)
   197  			require.EqualValues(t, cfg.SinkURI, mysqlSink)
   198  			return &model.ChangeFeedInfo{
   199  				UpstreamID: 1,
   200  				ID:         cfg.ID,
   201  				Namespace:  cfg.Namespace,
   202  				SinkURI:    cfg.SinkURI,
   203  			}, nil
   204  		}).AnyTimes()
   205  	ctrl.EXPECT().
   206  		CreateChangefeed(gomock.Any(), gomock.Any(), gomock.Any()).
   207  		Return(cerrors.ErrPDEtcdAPIError).Times(1)
   208  
   209  	cfConfig.SinkURI = mysqlSink
   210  	body, err = json.Marshal(&cfConfig)
   211  	require.Nil(t, err)
   212  	w = httptest.NewRecorder()
   213  	req, _ = http.NewRequestWithContext(context.Background(), create.method,
   214  		create.url, bytes.NewReader(body))
   215  	router.ServeHTTP(w, req)
   216  	respErr = model.HTTPError{}
   217  	err = json.NewDecoder(w.Body).Decode(&respErr)
   218  	require.Nil(t, err)
   219  	require.Contains(t, respErr.Code, "ErrPDEtcdAPIError")
   220  	require.Equal(t, http.StatusInternalServerError, w.Code)
   221  
   222  	// case 6: success
   223  	helpers.EXPECT().
   224  		getEtcdClient(gomock.Any(), gomock.Any()).
   225  		Return(testEtcdCluster.RandClient(), nil)
   226  	ctrl.EXPECT().
   227  		CreateChangefeed(gomock.Any(), gomock.Any(), gomock.Any()).
   228  		Return(nil).
   229  		AnyTimes()
   230  	w = httptest.NewRecorder()
   231  	req, _ = http.NewRequestWithContext(context.Background(), create.method,
   232  		create.url, bytes.NewReader(body))
   233  	router.ServeHTTP(w, req)
   234  	resp := ChangeFeedInfo{}
   235  	err = json.NewDecoder(w.Body).Decode(&resp)
   236  	require.Nil(t, err)
   237  	require.Equal(t, cfConfig.ID, resp.ID)
   238  	require.Equal(t, cfConfig.Namespace, resp.Namespace)
   239  	mysqlSink, err = util.MaskSinkURI(mysqlSink)
   240  	require.Nil(t, err)
   241  	require.Equal(t, mysqlSink, resp.SinkURI)
   242  	require.Equal(t, http.StatusOK, w.Code)
   243  }
   244  
   245  func TestGetChangeFeed(t *testing.T) {
   246  	t.Parallel()
   247  
   248  	cfInfo := testCase{url: "/api/v2/changefeeds/%s?namespace=%s", method: "GET"}
   249  	statusProvider := &mockStatusProvider{}
   250  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
   251  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   252  	cp.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   253  	cp.EXPECT().IsController().Return(true).AnyTimes()
   254  
   255  	apiV2 := NewOpenAPIV2ForTest(cp, APIV2HelpersImpl{})
   256  	router := newRouter(apiV2)
   257  
   258  	// case 1: invalid id
   259  	w := httptest.NewRecorder()
   260  	invalidID := "@^Invalid"
   261  	req, _ := http.NewRequestWithContext(
   262  		context.Background(),
   263  		cfInfo.method, fmt.Sprintf(cfInfo.url, invalidID, "test"),
   264  		nil,
   265  	)
   266  	router.ServeHTTP(w, req)
   267  	respErr := model.HTTPError{}
   268  	err := json.NewDecoder(w.Body).Decode(&respErr)
   269  	require.Nil(t, err)
   270  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   271  
   272  	// validId but not exists
   273  	validID := "changefeed-valid-id"
   274  	statusProvider.err = cerrors.ErrChangeFeedNotExists.GenWithStackByArgs(validID)
   275  	cp.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   276  
   277  	w = httptest.NewRecorder()
   278  	req, _ = http.NewRequestWithContext(
   279  		context.Background(),
   280  		cfInfo.method,
   281  		fmt.Sprintf(cfInfo.url, validID, "test"),
   282  		nil,
   283  	)
   284  	router.ServeHTTP(w, req)
   285  	require.Equal(t, http.StatusBadRequest, w.Code)
   286  	respErr = model.HTTPError{}
   287  	err = json.NewDecoder(w.Body).Decode(&respErr)
   288  	require.Nil(t, err)
   289  	require.Contains(t, respErr.Code, "ErrChangeFeedNotExists")
   290  
   291  	// valid but changefeed contains runtime error
   292  	statusProvider.err = nil
   293  	statusProvider.changefeedInfo = &model.ChangeFeedInfo{
   294  		ID:        validID,
   295  		Namespace: "abc",
   296  		Error: &model.RunningError{
   297  			Code: string(cerrors.ErrStartTsBeforeGC.RFCCode()),
   298  		},
   299  	}
   300  	statusProvider.changefeedStatus = &model.ChangeFeedStatusForAPI{
   301  		CheckpointTs: 1,
   302  	}
   303  	w = httptest.NewRecorder()
   304  	req, _ = http.NewRequestWithContext(context.Background(),
   305  		cfInfo.method, fmt.Sprintf(cfInfo.url, validID, "abc"), nil)
   306  	router.ServeHTTP(w, req)
   307  	require.Equal(t, http.StatusOK, w.Code)
   308  	resp := ChangeFeedInfo{}
   309  	err = json.NewDecoder(w.Body).Decode(&resp)
   310  	require.Nil(t, err)
   311  	require.Equal(t, resp.ID, validID)
   312  	require.Equal(t, resp.Namespace, "abc")
   313  	require.Contains(t, resp.Error.Code, "ErrStartTsBeforeGC")
   314  
   315  	// success
   316  	statusProvider.changefeedInfo = &model.ChangeFeedInfo{ID: validID}
   317  	w = httptest.NewRecorder()
   318  	req, _ = http.NewRequestWithContext(
   319  		context.Background(),
   320  		cfInfo.method,
   321  		fmt.Sprintf(cfInfo.url, validID, "abc"),
   322  		nil,
   323  	)
   324  	router.ServeHTTP(w, req)
   325  	require.Equal(t, http.StatusOK, w.Code)
   326  	resp = ChangeFeedInfo{}
   327  	err = json.NewDecoder(w.Body).Decode(&resp)
   328  	require.Nil(t, err)
   329  	require.Equal(t, resp.ID, validID)
   330  	require.Nil(t, resp.Error)
   331  }
   332  
   333  func TestUpdateChangefeed(t *testing.T) {
   334  	t.Parallel()
   335  	update := testCase{url: "/api/v2/changefeeds/%s", method: "PUT"}
   336  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
   337  	mockOwner := mock_owner.NewMockOwner(gomock.NewController(t))
   338  	mockCapture := mock_capture.NewMockCapture(gomock.NewController(t))
   339  	apiV2 := NewOpenAPIV2ForTest(mockCapture, helpers)
   340  	router := newRouter(apiV2)
   341  
   342  	statusProvider := &mockStatusProvider{}
   343  	mockCapture.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   344  	mockCapture.EXPECT().IsReady().Return(true).AnyTimes()
   345  	mockCapture.EXPECT().IsController().Return(true).AnyTimes()
   346  	mockCapture.EXPECT().GetOwner().Return(mockOwner, nil).AnyTimes()
   347  
   348  	// case 1 invalid id
   349  	invalidID := "Invalid_#"
   350  	w := httptest.NewRecorder()
   351  	req, _ := http.NewRequestWithContext(context.Background(), update.method,
   352  		fmt.Sprintf(update.url, invalidID), nil)
   353  	router.ServeHTTP(w, req)
   354  	respErr := model.HTTPError{}
   355  	err := json.NewDecoder(w.Body).Decode(&respErr)
   356  	require.Nil(t, err)
   357  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   358  	require.Equal(t, http.StatusBadRequest, w.Code)
   359  
   360  	// case 2: failed to get changefeedInfo
   361  	validID := changeFeedID.ID
   362  	statusProvider.err = cerrors.ErrChangeFeedNotExists.GenWithStackByArgs(validID)
   363  	w = httptest.NewRecorder()
   364  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   365  		fmt.Sprintf(update.url, validID), nil)
   366  	router.ServeHTTP(w, req)
   367  	respErr = model.HTTPError{}
   368  	err = json.NewDecoder(w.Body).Decode(&respErr)
   369  	require.Nil(t, err)
   370  	require.Contains(t, respErr.Code, "ErrChangeFeedNotExists")
   371  	require.Equal(t, http.StatusBadRequest, w.Code)
   372  
   373  	// case 3: changefeed not stopped
   374  	oldCfInfo := &model.ChangeFeedInfo{
   375  		ID:         validID,
   376  		State:      "normal",
   377  		UpstreamID: 1,
   378  		Namespace:  model.DefaultNamespace,
   379  		Config:     &config.ReplicaConfig{},
   380  	}
   381  	statusProvider.err = nil
   382  	statusProvider.changefeedInfo = oldCfInfo
   383  	w = httptest.NewRecorder()
   384  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   385  		fmt.Sprintf(update.url, validID), nil)
   386  	router.ServeHTTP(w, req)
   387  	respErr = model.HTTPError{}
   388  	err = json.NewDecoder(w.Body).Decode(&respErr)
   389  	require.Nil(t, err)
   390  	require.Contains(t, respErr.Code, "ErrChangefeedUpdateRefused")
   391  	require.Equal(t, http.StatusBadRequest, w.Code)
   392  
   393  	// case 4: changefeed stopped, but get upstream failed: not found
   394  	oldCfInfo.UpstreamID = 100
   395  	oldCfInfo.State = "stopped"
   396  	mockCapture.EXPECT().
   397  		GetUpstreamInfo(gomock.Any(), gomock.Eq(uint64(100)), gomock.Any()).
   398  		Return(nil, cerrors.ErrUpstreamNotFound).Times(1)
   399  
   400  	w = httptest.NewRecorder()
   401  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   402  		fmt.Sprintf(update.url, validID), nil)
   403  	router.ServeHTTP(w, req)
   404  	respErr = model.HTTPError{}
   405  	err = json.NewDecoder(w.Body).Decode(&respErr)
   406  	require.Nil(t, err)
   407  	require.Contains(t, respErr.Code, "ErrUpstreamNotFound")
   408  	require.Equal(t, http.StatusInternalServerError, w.Code)
   409  
   410  	// case 5: json failed
   411  	oldCfInfo.UpstreamID = 1
   412  	mockCapture.EXPECT().
   413  		GetUpstreamInfo(gomock.Any(), gomock.Eq(uint64(1)), gomock.Any()).
   414  		Return(nil, nil).AnyTimes()
   415  
   416  	w = httptest.NewRecorder()
   417  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   418  		fmt.Sprintf(update.url, validID), nil)
   419  	router.ServeHTTP(w, req)
   420  	respErr = model.HTTPError{}
   421  	err = json.NewDecoder(w.Body).Decode(&respErr)
   422  	require.Nil(t, err)
   423  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   424  	require.Equal(t, http.StatusBadRequest, w.Code)
   425  
   426  	// case 5: verify upstream failed
   427  	helpers.EXPECT().
   428  		verifyUpstream(gomock.Any(), gomock.Any(), gomock.Any()).
   429  		Return(cerrors.ErrUpstreamMissMatch).Times(1)
   430  	updateCfg := &ChangefeedConfig{}
   431  	body, err := json.Marshal(&updateCfg)
   432  	require.Nil(t, err)
   433  	w = httptest.NewRecorder()
   434  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   435  		fmt.Sprintf(update.url, validID), bytes.NewReader(body))
   436  	router.ServeHTTP(w, req)
   437  	respErr = model.HTTPError{}
   438  	err = json.NewDecoder(w.Body).Decode(&respErr)
   439  	require.Nil(t, err)
   440  	require.Contains(t, respErr.Code, "ErrUpstreamMissMatch")
   441  	require.Equal(t, http.StatusInternalServerError, w.Code)
   442  
   443  	// case 6: verify update changefeed info failed
   444  	helpers.EXPECT().
   445  		verifyUpstream(gomock.Any(), gomock.Any(), gomock.Any()).
   446  		Return(nil).AnyTimes()
   447  	helpers.EXPECT().
   448  		createTiStore(gomock.Any(), gomock.Any()).
   449  		Return(nil, nil).
   450  		AnyTimes()
   451  	mockCapture.EXPECT().GetUpstreamManager().Return(nil, nil).AnyTimes()
   452  	helpers.EXPECT().
   453  		verifyUpdateChangefeedConfig(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
   454  		Return(&model.ChangeFeedInfo{}, &model.UpstreamInfo{}, cerrors.ErrChangefeedUpdateRefused).
   455  		Times(1)
   456  
   457  	statusProvider.changefeedStatus = &model.ChangeFeedStatusForAPI{
   458  		CheckpointTs: 1,
   459  	}
   460  	w = httptest.NewRecorder()
   461  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   462  		fmt.Sprintf(update.url, validID), bytes.NewReader(body))
   463  	router.ServeHTTP(w, req)
   464  	respErr = model.HTTPError{}
   465  	err = json.NewDecoder(w.Body).Decode(&respErr)
   466  	require.Nil(t, err)
   467  	require.Contains(t, respErr.Code, "ErrChangefeedUpdateRefused")
   468  	require.Equal(t, http.StatusBadRequest, w.Code)
   469  
   470  	// case 7: update transaction failed
   471  	mockCapture.EXPECT().GetUpstreamManager().Return(upstream.NewManager4Test(&mockPDClient{}), nil).AnyTimes()
   472  	helpers.EXPECT().
   473  		verifyUpdateChangefeedConfig(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
   474  		Return(&model.ChangeFeedInfo{}, &model.UpstreamInfo{}, nil).
   475  		Times(1)
   476  	mockOwner.EXPECT().
   477  		UpdateChangefeedAndUpstream(gomock.Any(), gomock.Any(), gomock.Any()).
   478  		Return(cerrors.ErrEtcdAPIError).Times(1)
   479  
   480  	w = httptest.NewRecorder()
   481  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   482  		fmt.Sprintf(update.url, validID), bytes.NewReader(body))
   483  	router.ServeHTTP(w, req)
   484  	respErr = model.HTTPError{}
   485  	err = json.NewDecoder(w.Body).Decode(&respErr)
   486  	require.Nil(t, err)
   487  	require.Contains(t, respErr.Code, "ErrEtcdAPIError")
   488  	require.Equal(t, http.StatusInternalServerError, w.Code)
   489  
   490  	// case 8: success
   491  	helpers.EXPECT().
   492  		verifyUpdateChangefeedConfig(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
   493  		Return(oldCfInfo, &model.UpstreamInfo{}, nil).
   494  		Times(1)
   495  	mockCapture.EXPECT().GetUpstreamManager().Return(upstream.NewManager4Test(&mockPDClient{}), nil).AnyTimes()
   496  	mockOwner.EXPECT().
   497  		UpdateChangefeedAndUpstream(gomock.Any(), gomock.Any(), gomock.Any()).
   498  		Return(nil).Times(1)
   499  
   500  	w = httptest.NewRecorder()
   501  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   502  		fmt.Sprintf(update.url, validID), bytes.NewReader(body))
   503  	router.ServeHTTP(w, req)
   504  	require.Equal(t, http.StatusOK, w.Code)
   505  
   506  	// case 9: success with ChangeFeed.State equal to StateFailed
   507  	oldCfInfo.State = "failed"
   508  	helpers.EXPECT().
   509  		verifyUpdateChangefeedConfig(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
   510  		Return(oldCfInfo, &model.UpstreamInfo{}, nil).
   511  		Times(1)
   512  	mockCapture.EXPECT().GetUpstreamManager().Return(upstream.NewManager4Test(&mockPDClient{}), nil).AnyTimes()
   513  	mockOwner.EXPECT().
   514  		UpdateChangefeedAndUpstream(gomock.Any(), gomock.Any(), gomock.Any()).
   515  		Return(nil).Times(1)
   516  
   517  	w = httptest.NewRecorder()
   518  	req, _ = http.NewRequestWithContext(context.Background(), update.method,
   519  		fmt.Sprintf(update.url, validID), bytes.NewReader(body))
   520  	router.ServeHTTP(w, req)
   521  	require.Equal(t, http.StatusOK, w.Code)
   522  }
   523  
   524  func TestListChangeFeeds(t *testing.T) {
   525  	t.Parallel()
   526  
   527  	ctx := gomock.NewController(t)
   528  	cp := mock_capture.NewMockCapture(ctx)
   529  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   530  	cp.EXPECT().IsController().Return(true).AnyTimes()
   531  	controller := mock_controller.NewMockController(ctx)
   532  	cp.EXPECT().GetController().Return(controller, nil).AnyTimes()
   533  
   534  	apiV2 := NewOpenAPIV2ForTest(cp, APIV2HelpersImpl{})
   535  	router := newRouter(apiV2)
   536  	sorted := func(s []model.ChangefeedCommonInfo) bool {
   537  		return sort.SliceIsSorted(s, func(i, j int) bool {
   538  			cf1, cf2 := s[i], s[j]
   539  			if cf1.Namespace == cf2.Namespace {
   540  				return cf1.ID < cf2.ID
   541  			}
   542  			return cf1.Namespace < cf2.Namespace
   543  		})
   544  	}
   545  
   546  	// case 1: list all changefeeds regardless of the state
   547  	controller.EXPECT().GetAllChangeFeedInfo(gomock.Any()).Return(
   548  		map[model.ChangeFeedID]*model.ChangeFeedInfo{
   549  			model.DefaultChangeFeedID("cf1"): {
   550  				State: model.StateNormal,
   551  			},
   552  			model.DefaultChangeFeedID("cf2"): {
   553  				State: model.StateWarning,
   554  				Warning: &model.RunningError{
   555  					Code: "warning",
   556  				},
   557  			},
   558  			model.DefaultChangeFeedID("cf3"): {
   559  				State: model.StateStopped,
   560  			},
   561  			model.DefaultChangeFeedID("cf4"): {
   562  				State: model.StatePending,
   563  			},
   564  			model.DefaultChangeFeedID("cf5"): {
   565  				State: model.StateFinished,
   566  			},
   567  		}, nil,
   568  	).Times(2)
   569  	controller.EXPECT().GetAllChangeFeedCheckpointTs(gomock.Any()).Return(
   570  		map[model.ChangeFeedID]uint64{
   571  			model.DefaultChangeFeedID("cf1"): 1,
   572  			model.DefaultChangeFeedID("cf2"): 2,
   573  			model.DefaultChangeFeedID("cf3"): 3,
   574  			model.DefaultChangeFeedID("cf4"): 4,
   575  			model.DefaultChangeFeedID("cf5"): 5,
   576  		}, nil).Times(2)
   577  	w := httptest.NewRecorder()
   578  	metaInfo := testCase{
   579  		url:    "/api/v2/changefeeds?state=all",
   580  		method: "GET",
   581  	}
   582  	req, _ := http.NewRequestWithContext(
   583  		context.Background(),
   584  		metaInfo.method,
   585  		metaInfo.url,
   586  		nil,
   587  	)
   588  	router.ServeHTTP(w, req)
   589  	resp := ListResponse[model.ChangefeedCommonInfo]{}
   590  	err := json.NewDecoder(w.Body).Decode(&resp)
   591  	require.Nil(t, err)
   592  	require.Equal(t, 5, resp.Total)
   593  	// changefeed info must be sorted by ID
   594  	require.Equal(t, true, sorted(resp.Items))
   595  	// warning changefeed must have warning error message
   596  	require.Equal(t, model.StateWarning, resp.Items[1].FeedState)
   597  	require.Contains(t, resp.Items[1].RunningError.Code, "warning")
   598  
   599  	// case 2: only list changefeed with state 'normal', 'stopped' and 'failed', "pending", "warning"
   600  	metaInfo2 := testCase{
   601  		url:    "/api/v2/changefeeds",
   602  		method: "GET",
   603  	}
   604  	req2, _ := http.NewRequestWithContext(
   605  		context.Background(),
   606  		metaInfo2.method,
   607  		metaInfo2.url,
   608  		nil,
   609  	)
   610  	router.ServeHTTP(w, req2)
   611  	resp2 := ListResponse[model.ChangefeedCommonInfo]{}
   612  	err = json.NewDecoder(w.Body).Decode(&resp2)
   613  	require.Nil(t, err)
   614  	require.Equal(t, 4, resp2.Total)
   615  	// changefeed info must be sorted by ID
   616  	require.Equal(t, true, sorted(resp2.Items))
   617  }
   618  
   619  func TestVerifyTable(t *testing.T) {
   620  	t.Parallel()
   621  
   622  	verify := &testCase{url: "/api/v2/verify_table", method: "POST"}
   623  
   624  	pdClient := &mockPDClient{}
   625  	upManager := upstream.NewManager4Test(pdClient)
   626  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
   627  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
   628  	// statusProvider := &mockStatusProvider{}
   629  	// cp.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   630  	cp.EXPECT().GetUpstreamManager().Return(upManager, nil).AnyTimes()
   631  	cp.EXPECT().IsController().Return(true).AnyTimes()
   632  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   633  
   634  	apiV2 := NewOpenAPIV2ForTest(cp, helpers)
   635  	router := newRouter(apiV2)
   636  
   637  	// case 1: json format error
   638  	w := httptest.NewRecorder()
   639  	req, _ := http.NewRequestWithContext(context.Background(),
   640  		verify.method, verify.url, nil)
   641  	router.ServeHTTP(w, req)
   642  	respErr := model.HTTPError{}
   643  	err := json.NewDecoder(w.Body).Decode(&respErr)
   644  	require.Nil(t, err)
   645  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   646  
   647  	// case 2: kv create failed
   648  	updateCfg := getDefaultVerifyTableConfig()
   649  	body, err := json.Marshal(&updateCfg)
   650  	require.Nil(t, err)
   651  	helpers.EXPECT().
   652  		createTiStore(gomock.Any(), gomock.Any()).
   653  		Return(nil, cerrors.ErrNewStore).
   654  		Times(1)
   655  
   656  	w = httptest.NewRecorder()
   657  	req, _ = http.NewRequestWithContext(context.Background(),
   658  		verify.method, verify.url, bytes.NewReader(body))
   659  	router.ServeHTTP(w, req)
   660  	respErr = model.HTTPError{}
   661  	err = json.NewDecoder(w.Body).Decode(&respErr)
   662  	require.Nil(t, err)
   663  	require.Contains(t, respErr.Code, "ErrNewStore")
   664  
   665  	// case 3: getVerifiedTables failed
   666  	helpers.EXPECT().
   667  		createTiStore(gomock.Any(), gomock.Any()).
   668  		Return(nil, nil).
   669  		AnyTimes()
   670  	helpers.EXPECT().getVerifiedTables(gomock.Any(), gomock.Any(), gomock.Any(),
   671  		gomock.Any(), gomock.Any(), gomock.Any()).
   672  		Return(nil, nil, cerrors.ErrFilterRuleInvalid).
   673  		Times(1)
   674  
   675  	w = httptest.NewRecorder()
   676  	req, _ = http.NewRequestWithContext(context.Background(),
   677  		verify.method, verify.url, bytes.NewReader(body))
   678  	router.ServeHTTP(w, req)
   679  	respErr = model.HTTPError{}
   680  	err = json.NewDecoder(w.Body).Decode(&respErr)
   681  	require.Nil(t, err)
   682  	require.Contains(t, respErr.Code, "ErrFilterRuleInvalid")
   683  
   684  	// case 4: success
   685  	eligible := []model.TableName{
   686  		{Schema: "test", Table: "validTable1"},
   687  		{Schema: "test", Table: "validTable2"},
   688  	}
   689  	ineligible := []model.TableName{
   690  		{Schema: "test", Table: "invalidTable"},
   691  	}
   692  	helpers.EXPECT().getVerifiedTables(gomock.Any(), gomock.Any(), gomock.Any(),
   693  		gomock.Any(), gomock.Any(), gomock.Any()).
   694  		Return(eligible, ineligible, nil)
   695  
   696  	w = httptest.NewRecorder()
   697  	req, _ = http.NewRequestWithContext(context.Background(),
   698  		verify.method, verify.url, bytes.NewReader(body))
   699  	router.ServeHTTP(w, req)
   700  	resp := Tables{}
   701  	err = json.NewDecoder(w.Body).Decode(&resp)
   702  	require.Nil(t, err)
   703  	require.Equal(t, http.StatusOK, w.Code)
   704  }
   705  
   706  func TestResumeChangefeed(t *testing.T) {
   707  	resume := testCase{url: "/api/v2/changefeeds/%s/resume?namespace=abc", method: "POST"}
   708  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
   709  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
   710  	owner := mock_owner.NewMockOwner(gomock.NewController(t))
   711  	apiV2 := NewOpenAPIV2ForTest(cp, helpers)
   712  	router := newRouter(apiV2)
   713  
   714  	pdClient := &mockPDClient{}
   715  	etcdClient := mock_etcd.NewMockCDCEtcdClient(gomock.NewController(t))
   716  	mockUpManager := upstream.NewManager4Test(pdClient)
   717  	statusProvider := &mockStatusProvider{
   718  		changefeedStatus: &model.ChangeFeedStatusForAPI{},
   719  	}
   720  
   721  	etcdClient.EXPECT().
   722  		GetEnsureGCServiceID(gomock.Any()).
   723  		Return(etcd.GcServiceIDForTest()).AnyTimes()
   724  	cp.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   725  	cp.EXPECT().GetEtcdClient().Return(etcdClient).AnyTimes()
   726  	cp.EXPECT().GetUpstreamManager().Return(mockUpManager, nil).AnyTimes()
   727  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   728  	cp.EXPECT().IsController().Return(true).AnyTimes()
   729  	cp.EXPECT().GetOwner().Return(owner, nil).AnyTimes()
   730  	owner.EXPECT().EnqueueJob(gomock.Any(), gomock.Any()).
   731  		Do(func(adminJob model.AdminJob, done chan<- error) {
   732  			require.EqualValues(t, changeFeedID, adminJob.CfID)
   733  			require.EqualValues(t, model.AdminResume, adminJob.Type)
   734  			close(done)
   735  		}).AnyTimes()
   736  
   737  	// case 1: invalid changefeed id
   738  	w := httptest.NewRecorder()
   739  	invalidID := "@^Invalid"
   740  	req, _ := http.NewRequestWithContext(context.Background(),
   741  		resume.method, fmt.Sprintf(resume.url, invalidID), nil)
   742  	router.ServeHTTP(w, req)
   743  	respErr := model.HTTPError{}
   744  	err := json.NewDecoder(w.Body).Decode(&respErr)
   745  	require.Nil(t, err)
   746  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   747  
   748  	// case 2: failed to get changefeedInfo
   749  	validID := changeFeedID.ID
   750  	statusProvider.err = cerrors.ErrChangeFeedNotExists.GenWithStackByArgs(validID)
   751  	w = httptest.NewRecorder()
   752  	req, _ = http.NewRequestWithContext(context.Background(), resume.method,
   753  		fmt.Sprintf(resume.url, validID), nil)
   754  	router.ServeHTTP(w, req)
   755  	respErr = model.HTTPError{}
   756  	err = json.NewDecoder(w.Body).Decode(&respErr)
   757  	require.Nil(t, err)
   758  	require.Contains(t, respErr.Code, "ErrChangeFeedNotExists")
   759  	require.Equal(t, http.StatusBadRequest, w.Code)
   760  
   761  	// case 3: failed to verify config
   762  	statusProvider.err = nil
   763  	statusProvider.changefeedInfo = &model.ChangeFeedInfo{ID: validID}
   764  	helpers.EXPECT().
   765  		getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).
   766  		Return(pdClient, nil).AnyTimes()
   767  	helpers.EXPECT().
   768  		verifyResumeChangefeedConfig(gomock.Any(), gomock.Any(),
   769  			gomock.Any(), gomock.Any(), gomock.Any()).Return(cerrors.ErrStartTsBeforeGC).Times(1)
   770  	resumeCfg := &ResumeChangefeedConfig{}
   771  	resumeCfg.OverwriteCheckpointTs = 100
   772  	body, err := json.Marshal(&resumeCfg)
   773  	require.Nil(t, err)
   774  	w = httptest.NewRecorder()
   775  	req, _ = http.NewRequestWithContext(context.Background(), resume.method,
   776  		fmt.Sprintf(resume.url, validID), bytes.NewReader(body))
   777  	router.ServeHTTP(w, req)
   778  	respErr = model.HTTPError{}
   779  	err = json.NewDecoder(w.Body).Decode(&respErr)
   780  	require.Nil(t, err)
   781  	require.Contains(t, respErr.Code, "ErrStartTsBeforeGC")
   782  	require.Equal(t, http.StatusBadRequest, w.Code)
   783  
   784  	// case 4: success without overwriting checkpointTs
   785  	statusProvider.err = nil
   786  	statusProvider.changefeedInfo = &model.ChangeFeedInfo{ID: validID}
   787  	helpers.EXPECT().
   788  		verifyResumeChangefeedConfig(gomock.Any(), gomock.Any(),
   789  			gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
   790  	resumeCfg = &ResumeChangefeedConfig{}
   791  	body, err = json.Marshal(&resumeCfg)
   792  	require.Nil(t, err)
   793  	w = httptest.NewRecorder()
   794  	req, _ = http.NewRequestWithContext(context.Background(), resume.method,
   795  		fmt.Sprintf(resume.url, validID), bytes.NewReader(body))
   796  	router.ServeHTTP(w, req)
   797  	require.Equal(t, http.StatusOK, w.Code)
   798  
   799  	// case 5: success with overwriting checkpointTs
   800  	statusProvider.err = nil
   801  	statusProvider.changefeedInfo = &model.ChangeFeedInfo{ID: validID}
   802  	helpers.EXPECT().
   803  		getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).
   804  		Return(pdClient, nil).AnyTimes()
   805  	helpers.EXPECT().
   806  		verifyResumeChangefeedConfig(gomock.Any(), gomock.Any(),
   807  			gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).Times(1)
   808  	resumeCfg = &ResumeChangefeedConfig{}
   809  	resumeCfg.OverwriteCheckpointTs = 100
   810  	body, err = json.Marshal(&resumeCfg)
   811  	require.Nil(t, err)
   812  	w = httptest.NewRecorder()
   813  	req, _ = http.NewRequestWithContext(context.Background(), resume.method,
   814  		fmt.Sprintf(resume.url, validID), bytes.NewReader(body))
   815  	router.ServeHTTP(w, req)
   816  	require.Equal(t, http.StatusOK, w.Code)
   817  }
   818  
   819  func TestDeleteChangefeed(t *testing.T) {
   820  	remove := testCase{url: "/api/v2/changefeeds/%s?namespace=abc", method: "DELETE"}
   821  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
   822  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
   823  	owner := mock_owner.NewMockOwner(gomock.NewController(t))
   824  	apiV2 := NewOpenAPIV2ForTest(cp, helpers)
   825  	router := newRouter(apiV2)
   826  
   827  	pdClient := &mockPDClient{}
   828  	etcdClient := mock_etcd.NewMockCDCEtcdClient(gomock.NewController(t))
   829  	mockUpManager := upstream.NewManager4Test(pdClient)
   830  
   831  	etcdClient.EXPECT().
   832  		GetEnsureGCServiceID(gomock.Any()).
   833  		Return(etcd.GcServiceIDForTest()).AnyTimes()
   834  	ctrl := mock_controller.NewMockController(gomock.NewController(t))
   835  	cp.EXPECT().GetEtcdClient().Return(etcdClient).AnyTimes()
   836  	cp.EXPECT().GetUpstreamManager().Return(mockUpManager, nil).AnyTimes()
   837  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   838  	cp.EXPECT().IsController().Return(true).AnyTimes()
   839  	cp.EXPECT().GetOwner().Return(owner, nil).AnyTimes()
   840  	cp.EXPECT().GetController().Return(ctrl, nil).AnyTimes()
   841  	owner.EXPECT().EnqueueJob(gomock.Any(), gomock.Any()).
   842  		Do(func(adminJob model.AdminJob, done chan<- error) {
   843  			require.EqualValues(t, changeFeedID, adminJob.CfID)
   844  			require.EqualValues(t, model.AdminRemove, adminJob.Type)
   845  			close(done)
   846  		}).AnyTimes()
   847  
   848  	// case 1: invalid changefeed id
   849  	w := httptest.NewRecorder()
   850  	invalidID := "@^Invalid"
   851  	req, _ := http.NewRequestWithContext(context.Background(),
   852  		remove.method, fmt.Sprintf(remove.url, invalidID), nil)
   853  	router.ServeHTTP(w, req)
   854  	respErr := model.HTTPError{}
   855  	err := json.NewDecoder(w.Body).Decode(&respErr)
   856  	require.Nil(t, err)
   857  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   858  
   859  	// case 2: changefeed not exists
   860  	validID := changeFeedID.ID
   861  	ctrl.EXPECT().IsChangefeedExists(gomock.Any(), changeFeedID).Return(false, nil)
   862  	w = httptest.NewRecorder()
   863  	req, _ = http.NewRequestWithContext(context.Background(), remove.method,
   864  		fmt.Sprintf(remove.url, validID), nil)
   865  	router.ServeHTTP(w, req)
   866  	require.Equal(t, http.StatusOK, w.Code)
   867  
   868  	// case 3: query changefeed error
   869  	ctrl.EXPECT().IsChangefeedExists(gomock.Any(), changeFeedID).Return(false,
   870  		cerrors.ErrChangefeedUpdateRefused.GenWithStackByArgs(validID))
   871  	w = httptest.NewRecorder()
   872  	req, _ = http.NewRequestWithContext(context.Background(), remove.method,
   873  		fmt.Sprintf(remove.url, validID), nil)
   874  	router.ServeHTTP(w, req)
   875  	respErr = model.HTTPError{}
   876  	err = json.NewDecoder(w.Body).Decode(&respErr)
   877  	require.Nil(t, err)
   878  	require.Contains(t, respErr.Code, "ErrChangefeedUpdateRefused")
   879  
   880  	// case 4: remove changefeed
   881  	ctrl.EXPECT().IsChangefeedExists(gomock.Any(), changeFeedID).Return(true, nil)
   882  	ctrl.EXPECT().IsChangefeedExists(gomock.Any(), changeFeedID).Return(false,
   883  		cerrors.ErrChangeFeedNotExists.GenWithStackByArgs(validID))
   884  	w = httptest.NewRecorder()
   885  	req, _ = http.NewRequestWithContext(context.Background(), remove.method,
   886  		fmt.Sprintf(remove.url, validID), nil)
   887  	router.ServeHTTP(w, req)
   888  	require.Equal(t, http.StatusOK, w.Code)
   889  
   890  	// case 5: remove changefeed failed
   891  	ctrl.EXPECT().IsChangefeedExists(gomock.Any(), changeFeedID).Return(true, nil).AnyTimes()
   892  	w = httptest.NewRecorder()
   893  	req, _ = http.NewRequestWithContext(context.Background(), remove.method,
   894  		fmt.Sprintf(remove.url, validID), nil)
   895  	router.ServeHTTP(w, req)
   896  	respErr = model.HTTPError{}
   897  	err = json.NewDecoder(w.Body).Decode(&respErr)
   898  	require.Nil(t, err)
   899  	require.Contains(t, respErr.Code, "ErrReachMaxTry")
   900  	require.Equal(t, http.StatusInternalServerError, w.Code)
   901  }
   902  
   903  func TestPauseChangefeed(t *testing.T) {
   904  	resume := testCase{url: "/api/v2/changefeeds/%s/pause?namespace=abc", method: "POST"}
   905  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
   906  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
   907  	owner := mock_owner.NewMockOwner(gomock.NewController(t))
   908  	apiV2 := NewOpenAPIV2ForTest(cp, helpers)
   909  	router := newRouter(apiV2)
   910  
   911  	pdClient := &mockPDClient{}
   912  	etcdClient := mock_etcd.NewMockCDCEtcdClient(gomock.NewController(t))
   913  	mockUpManager := upstream.NewManager4Test(pdClient)
   914  	statusProvider := &mockStatusProvider{}
   915  
   916  	etcdClient.EXPECT().
   917  		GetEnsureGCServiceID(gomock.Any()).
   918  		Return(etcd.GcServiceIDForTest()).AnyTimes()
   919  	cp.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   920  	cp.EXPECT().GetEtcdClient().Return(etcdClient).AnyTimes()
   921  	cp.EXPECT().GetUpstreamManager().Return(mockUpManager, nil).AnyTimes()
   922  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   923  	cp.EXPECT().IsController().Return(true).AnyTimes()
   924  	cp.EXPECT().GetOwner().Return(owner, nil).AnyTimes()
   925  	owner.EXPECT().EnqueueJob(gomock.Any(), gomock.Any()).
   926  		Do(func(adminJob model.AdminJob, done chan<- error) {
   927  			require.EqualValues(t, changeFeedID, adminJob.CfID)
   928  			require.EqualValues(t, model.AdminStop, adminJob.Type)
   929  			close(done)
   930  		}).AnyTimes()
   931  
   932  	// case 1: invalid changefeed id
   933  	w := httptest.NewRecorder()
   934  	invalidID := "@^Invalid"
   935  	req, _ := http.NewRequestWithContext(context.Background(),
   936  		resume.method, fmt.Sprintf(resume.url, invalidID), nil)
   937  	router.ServeHTTP(w, req)
   938  	respErr := model.HTTPError{}
   939  	err := json.NewDecoder(w.Body).Decode(&respErr)
   940  	require.Nil(t, err)
   941  	require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   942  
   943  	// case 2: failed to get changefeedInfo
   944  	validID := changeFeedID.ID
   945  	statusProvider.err = cerrors.ErrChangeFeedNotExists.GenWithStackByArgs(validID)
   946  	w = httptest.NewRecorder()
   947  	req, _ = http.NewRequestWithContext(context.Background(), resume.method,
   948  		fmt.Sprintf(resume.url, validID), nil)
   949  	router.ServeHTTP(w, req)
   950  	respErr = model.HTTPError{}
   951  	err = json.NewDecoder(w.Body).Decode(&respErr)
   952  	require.Nil(t, err)
   953  	require.Contains(t, respErr.Code, "ErrChangeFeedNotExists")
   954  	require.Equal(t, http.StatusBadRequest, w.Code)
   955  
   956  	// case 4: success without overwriting checkpointTs
   957  	statusProvider.err = nil
   958  	statusProvider.changefeedInfo = &model.ChangeFeedInfo{ID: validID}
   959  	require.Nil(t, err)
   960  	w = httptest.NewRecorder()
   961  	req, _ = http.NewRequestWithContext(context.Background(), resume.method,
   962  		fmt.Sprintf(resume.url, validID), nil)
   963  	router.ServeHTTP(w, req)
   964  	require.Equal(t, http.StatusOK, w.Code)
   965  	require.Equal(t, "{}", w.Body.String())
   966  }
   967  
   968  func TestChangefeedSynced(t *testing.T) {
   969  	syncedInfo := testCase{url: "/api/v2/changefeeds/%s/synced?namespace=abc", method: "GET"}
   970  	helpers := NewMockAPIV2Helpers(gomock.NewController(t))
   971  	cp := mock_capture.NewMockCapture(gomock.NewController(t))
   972  	owner := mock_owner.NewMockOwner(gomock.NewController(t))
   973  	apiV2 := NewOpenAPIV2ForTest(cp, helpers)
   974  	router := newRouter(apiV2)
   975  
   976  	statusProvider := &mockStatusProvider{}
   977  
   978  	cp.EXPECT().StatusProvider().Return(statusProvider).AnyTimes()
   979  	cp.EXPECT().IsReady().Return(true).AnyTimes()
   980  	cp.EXPECT().IsController().Return(true).AnyTimes()
   981  	cp.EXPECT().GetOwner().Return(owner, nil).AnyTimes()
   982  
   983  	pdClient := &mockPDClient{}
   984  	mockUpManager := upstream.NewManager4Test(pdClient)
   985  	cp.EXPECT().GetUpstreamManager().Return(mockUpManager, nil).AnyTimes()
   986  
   987  	{
   988  		// case 1: invalid changefeed id
   989  		w := httptest.NewRecorder()
   990  		invalidID := "@^Invalid"
   991  		req, _ := http.NewRequestWithContext(context.Background(),
   992  			syncedInfo.method, fmt.Sprintf(syncedInfo.url, invalidID), nil)
   993  		router.ServeHTTP(w, req)
   994  		respErr := model.HTTPError{}
   995  		err := json.NewDecoder(w.Body).Decode(&respErr)
   996  		require.Nil(t, err)
   997  		require.Contains(t, respErr.Code, "ErrAPIInvalidParam")
   998  	}
   999  
  1000  	{
  1001  		// case 2: not existed changefeed id
  1002  		validID := changeFeedID.ID
  1003  		statusProvider.err = cerrors.ErrChangeFeedNotExists.GenWithStackByArgs(validID)
  1004  		w := httptest.NewRecorder()
  1005  		req, _ := http.NewRequestWithContext(context.Background(), syncedInfo.method,
  1006  			fmt.Sprintf(syncedInfo.url, validID), nil)
  1007  		router.ServeHTTP(w, req)
  1008  		respErr := model.HTTPError{}
  1009  		err := json.NewDecoder(w.Body).Decode(&respErr)
  1010  		require.Nil(t, err)
  1011  		require.Contains(t, respErr.Code, "ErrChangeFeedNotExists")
  1012  		require.Equal(t, http.StatusBadRequest, w.Code)
  1013  	}
  1014  
  1015  	validID := changeFeedID.ID
  1016  	cfInfo := &model.ChangeFeedInfo{
  1017  		ID: validID,
  1018  	}
  1019  	statusProvider.err = nil
  1020  	statusProvider.changefeedInfo = cfInfo
  1021  	{
  1022  		helpers.EXPECT().getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, cerrors.ErrAPIGetPDClientFailed).Times(1)
  1023  		// case3: pd is offline,resolvedTs - checkpointTs > 15s
  1024  		statusProvider.changeFeedSyncedStatus = &model.ChangeFeedSyncedStatusForAPI{
  1025  			CheckpointTs:     1701153217279 << 18,
  1026  			LastSyncedTs:     1701153217279 << 18,
  1027  			PullerResolvedTs: 1701153247279 << 18,
  1028  		}
  1029  		w := httptest.NewRecorder()
  1030  		req, _ := http.NewRequestWithContext(
  1031  			context.Background(),
  1032  			syncedInfo.method,
  1033  			fmt.Sprintf(syncedInfo.url, validID),
  1034  			nil,
  1035  		)
  1036  		router.ServeHTTP(w, req)
  1037  		require.Equal(t, http.StatusOK, w.Code)
  1038  		resp := SyncedStatus{}
  1039  		err := json.NewDecoder(w.Body).Decode(&resp)
  1040  		require.Nil(t, err)
  1041  		require.Equal(t, false, resp.Synced)
  1042  		require.Equal(t, "[CDC:ErrAPIGetPDClientFailed]failed to get PDClient to connect PD, "+
  1043  			"please recheck. Besides the data is not finish syncing", resp.Info)
  1044  	}
  1045  
  1046  	{
  1047  		helpers.EXPECT().getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, cerrors.ErrAPIGetPDClientFailed).Times(1)
  1048  		// case4: pd is offline,resolvedTs - checkpointTs < 15s
  1049  		statusProvider.changeFeedSyncedStatus = &model.ChangeFeedSyncedStatusForAPI{
  1050  			CheckpointTs:     1701153217279 << 18,
  1051  			LastSyncedTs:     1701153217279 << 18,
  1052  			PullerResolvedTs: 1701153217479 << 18,
  1053  		}
  1054  		w := httptest.NewRecorder()
  1055  		req, _ := http.NewRequestWithContext(
  1056  			context.Background(),
  1057  			syncedInfo.method,
  1058  			fmt.Sprintf(syncedInfo.url, validID),
  1059  			nil,
  1060  		)
  1061  		router.ServeHTTP(w, req)
  1062  		require.Equal(t, http.StatusOK, w.Code)
  1063  		resp := SyncedStatus{}
  1064  		err := json.NewDecoder(w.Body).Decode(&resp)
  1065  		require.Nil(t, err)
  1066  		require.Equal(t, false, resp.Synced)
  1067  		require.Equal(t, "[CDC:ErrAPIGetPDClientFailed]failed to get PDClient to connect PD, please recheck. "+
  1068  			"You should check the pd status first. If pd status is normal, means we don't finish sync data. "+
  1069  			"If pd is offline, please check whether we satisfy the condition that "+
  1070  			"the time difference from lastSyncedTs to the current time from the time zone of pd is greater than 300 secs. "+
  1071  			"If it's satisfied, means the data syncing is totally finished", resp.Info)
  1072  	}
  1073  
  1074  	helpers.EXPECT().getPDClient(gomock.Any(), gomock.Any(), gomock.Any()).Return(pdClient, nil).AnyTimes()
  1075  	pdClient.logicTime = 1000
  1076  	pdClient.timestamp = 1701153217279
  1077  	{
  1078  		// case5: pdTs - lastSyncedTs > 5min, pdTs - checkpointTs < 15s
  1079  		statusProvider.changeFeedSyncedStatus = &model.ChangeFeedSyncedStatusForAPI{
  1080  			CheckpointTs:     1701153217209 << 18,
  1081  			LastSyncedTs:     1701152217279 << 18,
  1082  			PullerResolvedTs: 1701153217229 << 18,
  1083  		}
  1084  		w := httptest.NewRecorder()
  1085  		req, _ := http.NewRequestWithContext(
  1086  			context.Background(),
  1087  			syncedInfo.method,
  1088  			fmt.Sprintf(syncedInfo.url, validID),
  1089  			nil,
  1090  		)
  1091  		router.ServeHTTP(w, req)
  1092  		require.Equal(t, http.StatusOK, w.Code)
  1093  		resp := SyncedStatus{}
  1094  		err := json.NewDecoder(w.Body).Decode(&resp)
  1095  		require.Nil(t, err)
  1096  		require.Equal(t, true, resp.Synced)
  1097  		require.Equal(t, "Data syncing is finished", resp.Info)
  1098  	}
  1099  
  1100  	{
  1101  		// case6: pdTs - lastSyncedTs > 5min, pdTs - checkpointTs > 15s, resolvedTs - checkpointTs < 15s
  1102  		statusProvider.changeFeedSyncedStatus = &model.ChangeFeedSyncedStatusForAPI{
  1103  			CheckpointTs:     1701153201279 << 18,
  1104  			LastSyncedTs:     1701152217279 << 18,
  1105  			PullerResolvedTs: 1701153201379 << 18,
  1106  		}
  1107  		w := httptest.NewRecorder()
  1108  		req, _ := http.NewRequestWithContext(
  1109  			context.Background(),
  1110  			syncedInfo.method,
  1111  			fmt.Sprintf(syncedInfo.url, validID),
  1112  			nil,
  1113  		)
  1114  		router.ServeHTTP(w, req)
  1115  		require.Equal(t, http.StatusOK, w.Code)
  1116  		resp := SyncedStatus{}
  1117  		err := json.NewDecoder(w.Body).Decode(&resp)
  1118  		require.Nil(t, err)
  1119  		require.Equal(t, false, resp.Synced)
  1120  		require.Equal(t, "Please check whether PD is online and TiKV Regions are all available. "+
  1121  			"If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. "+
  1122  			"To check whether TiKV regions are all available, you can view "+
  1123  			"'TiKV-Details' > 'Resolved-Ts' > 'Max Leader Resolved TS gap' on Grafana. "+
  1124  			"If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. "+
  1125  			"Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait", resp.Info)
  1126  	}
  1127  
  1128  	{
  1129  		// case7: pdTs - lastSyncedTs > 5min, pdTs - checkpointTs > 15s, resolvedTs - checkpointTs > 15s
  1130  		statusProvider.changeFeedSyncedStatus = &model.ChangeFeedSyncedStatusForAPI{
  1131  			CheckpointTs:     1701153201279 << 18,
  1132  			LastSyncedTs:     1701152207279 << 18,
  1133  			PullerResolvedTs: 1701153218279 << 18,
  1134  		}
  1135  		w := httptest.NewRecorder()
  1136  		req, _ := http.NewRequestWithContext(
  1137  			context.Background(),
  1138  			syncedInfo.method,
  1139  			fmt.Sprintf(syncedInfo.url, validID),
  1140  			nil,
  1141  		)
  1142  		router.ServeHTTP(w, req)
  1143  		require.Equal(t, http.StatusOK, w.Code)
  1144  		resp := SyncedStatus{}
  1145  		err := json.NewDecoder(w.Body).Decode(&resp)
  1146  		require.Nil(t, err)
  1147  		require.Equal(t, false, resp.Synced)
  1148  		require.Equal(t, "The data syncing is not finished, please wait", resp.Info)
  1149  	}
  1150  
  1151  	{
  1152  		// case8: pdTs - lastSyncedTs < 5min
  1153  		statusProvider.changeFeedSyncedStatus = &model.ChangeFeedSyncedStatusForAPI{
  1154  			CheckpointTs:     1701153217279 << 18,
  1155  			LastSyncedTs:     1701153213279 << 18,
  1156  			PullerResolvedTs: 1701153217279 << 18,
  1157  		}
  1158  		w := httptest.NewRecorder()
  1159  		req, _ := http.NewRequestWithContext(
  1160  			context.Background(),
  1161  			syncedInfo.method,
  1162  			fmt.Sprintf(syncedInfo.url, validID),
  1163  			nil,
  1164  		)
  1165  		router.ServeHTTP(w, req)
  1166  		require.Equal(t, http.StatusOK, w.Code)
  1167  		resp := SyncedStatus{}
  1168  		err := json.NewDecoder(w.Body).Decode(&resp)
  1169  		require.Nil(t, err)
  1170  		require.Equal(t, false, resp.Synced)
  1171  		require.Equal(t, "The data syncing is not finished, please wait", resp.Info)
  1172  	}
  1173  }
  1174  
  1175  func TestHasRunningImport(t *testing.T) {
  1176  	integration.BeforeTestExternal(t)
  1177  	testEtcdCluster := integration.NewClusterV3(
  1178  		t, &integration.ClusterConfig{Size: 1},
  1179  	)
  1180  	defer testEtcdCluster.Terminate(t)
  1181  
  1182  	ctx := context.Background()
  1183  	client := testEtcdCluster.RandClient()
  1184  	hasImport := hasRunningImport(ctx, client)
  1185  	require.NoError(t, hasImport)
  1186  
  1187  	lease, err := client.Lease.Grant(ctx, 3*60)
  1188  	require.NoError(t, err)
  1189  
  1190  	_, err = client.KV.Put(
  1191  		ctx, filepath.Join(RegisterImportTaskPrefix, "pitr"),
  1192  		"", clientv3.WithLease(lease.ID),
  1193  	)
  1194  	require.NoError(t, err)
  1195  
  1196  	hasImport = hasRunningImport(ctx, client)
  1197  	require.NotNil(t, hasImport)
  1198  	require.Contains(
  1199  		t, hasImport.Error(), "There are lightning/restore tasks running",
  1200  	)
  1201  }