github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/http_api/util/test_case.py (about)

     1  import sys
     2  import os
     3  import requests as rq
     4  from requests.exceptions import RequestException
     5  import time
     6  import json
     7  
     8  # the max retry time
     9  RETRY_TIME = 10
    10  
    11  BASE_URL0 = "http://127.0.0.1:8300/api/v1"
    12  BASE_URL1 = "http://127.0.0.1:8301/api/v1"
    13  
    14  
    15  BASE_URL0_V2 = "http://127.0.0.1:8300/api/v2"
    16  BASE_URL1_V2 = "https://127.0.0.1:8301/api/v2"
    17  
    18  PD_ADDR = "http://127.0.0.1:2379"
    19  # This value is generated by:
    20  # echo -n '123456' | base64
    21  # MTIzNDU2
    22  # Use this value here to test redo apply function works well
    23  # when use base64 encoded password
    24  ENPASSWORD="MTIzNDU2"
    25  SINK_URI="mysql://normal:%s@127.0.0.1:3306/" % ENPASSWORD
    26  
    27  physicalShiftBits = 18
    28  
    29  def requests_get_with_retry(url, max_retries=RETRY_TIME, delay_seconds=1):
    30      """
    31      requests get with retry
    32  
    33      :param url: request url
    34      :param max_retries: max retry times
    35      :param delay_seconds: retry delay seconds
    36      :return: when success, return response, else return None
    37      """
    38      for retry in range(max_retries):
    39          try:
    40              response = rq.get(url)
    41              if response.status_code == 200 or response.status_code == 202:
    42                  return response
    43          except RequestException as e:
    44              print(f"request fails {retry + 1}/{max_retries} time retry...")
    45              time.sleep(delay_seconds)
    46      return None
    47  
    48  # we should write some SQLs in the run.sh after call create_changefeed
    49  def create_changefeed(sink_uri):
    50      url = BASE_URL1+"/changefeeds"
    51      # create changefeed
    52      for i in range(1, 4):
    53          data = {
    54              "changefeed_id": "changefeed-test"+str(i),
    55              "sink_uri": "blackhole://",
    56              "ignore_ineligible_table": True
    57          }
    58          # set sink_uri
    59          if i == 1 and sink_uri != "":
    60              data["sink_uri"] = sink_uri
    61  
    62          data = json.dumps(data)
    63          headers = {"Content-Type": "application/json"}
    64          resp = rq.post(url, data=data, headers=headers)
    65          assert resp.status_code == rq.codes.accepted
    66  
    67      # create changefeed fail because sink_uri is invalid
    68      data = json.dumps({
    69          "changefeed_id": "changefeed-test",
    70          "sink_uri": "mysql://127.0.0.1:1111",
    71          "ignore_ineligible_table": True
    72      })
    73      headers = {"Content-Type": "application/json"}
    74      resp = rq.post(url, data=data, headers=headers)
    75      assert resp.status_code == rq.codes.bad_request
    76  
    77      print("pass test: create changefeed")
    78  
    79  
    80  def list_changefeed():
    81      # test state: all
    82      url = BASE_URL0+"/changefeeds?state=all"
    83      resp = rq.get(url)
    84      assert resp.status_code == rq.codes.ok
    85  
    86      # test state: normal
    87      url = BASE_URL0+"/changefeeds?state=normal"
    88      resp = rq.get(url)
    89      assert resp.status_code == rq.codes.ok
    90      data = resp.json()
    91      for changefeed in data:
    92          assert changefeed["state"] == "normal"
    93  
    94      # test state: stopped
    95      url = BASE_URL0+"/changefeeds?state=stopped"
    96      resp = rq.get(url)
    97      assert resp.status_code == rq.codes.ok
    98      data = resp.json()
    99      for changefeed in data:
   100          assert changefeed["state"] == "stopped"
   101  
   102      print("pass test: list changefeed")
   103  
   104  def get_changefeed():
   105      # test get changefeed success
   106      url = BASE_URL0+"/changefeeds/changefeed-test1"
   107      resp = rq.get(url)
   108      assert resp.status_code == rq.codes.ok
   109  
   110      url = BASE_URL0+"/changefeeds/changefeed-test2"
   111      resp = rq.get(url)
   112      assert resp.status_code == rq.codes.ok
   113  
   114      # test get changefeed failed
   115      url = BASE_URL0+"/changefeeds/changefeed-not-exists"
   116      resp = rq.get(url)
   117      assert resp.status_code == rq.codes.bad_request
   118      data = resp.json()
   119      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   120  
   121      print("pass test: get changefeed")
   122  
   123  
   124  def pause_changefeed():
   125      # pause changefeed
   126      url = BASE_URL0+"/changefeeds/changefeed-test2/pause"
   127      for i in range(RETRY_TIME):
   128          resp = rq.post(url)
   129          if resp.status_code == rq.codes.accepted:
   130              break
   131          time.sleep(1)
   132      assert resp.status_code == rq.codes.accepted
   133      # check if pause changefeed success
   134      url = BASE_URL0+"/changefeeds/changefeed-test2"
   135      for i in range(RETRY_TIME):
   136          resp = rq.get(url)
   137          assert resp.status_code == rq.codes.ok
   138          data = resp.json()
   139          if data["state"] == "stopped":
   140              break
   141          time.sleep(1)
   142      assert data["state"] == "stopped"
   143      # test pause changefeed failed
   144      url = BASE_URL0+"/changefeeds/changefeed-not-exists/pause"
   145      resp = rq.post(url)
   146      assert resp.status_code == rq.codes.bad_request
   147      data = resp.json()
   148      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   149  
   150      print("pass test: pause changefeed")
   151  
   152  def update_changefeed():
   153      # update fail
   154      # can only update a stopped changefeed
   155      url = BASE_URL0+"/changefeeds/changefeed-test1"
   156      data = json.dumps({"mounter_worker_num": 32})
   157      headers = {"Content-Type": "application/json"}
   158      resp = rq.put(url, data=data, headers=headers)
   159      assert resp.status_code == rq.codes.bad_request
   160  
   161      # update success
   162      url = BASE_URL0+"/changefeeds/changefeed-test2"
   163      data = json.dumps({"mounter_worker_num": 32})
   164      headers = {"Content-Type": "application/json"}
   165      resp = rq.put(url, data=data, headers=headers)
   166      assert resp.status_code == rq.codes.accepted
   167  
   168      # update fail
   169      # can't update start_ts
   170      url = BASE_URL0+"/changefeeds/changefeed-test2"
   171      data = json.dumps({"start_ts": 0})
   172      headers = {"Content-Type": "application/json"}
   173      resp = rq.put(url, data=data, headers=headers)
   174      assert resp.status_code == rq.codes.bad_request
   175  
   176      print("pass test: update changefeed")
   177  
   178  
   179  def resume_changefeed():
   180      # resume changefeed
   181      url = BASE_URL1+"/changefeeds/changefeed-test2/resume"
   182      resp = rq.post(url)
   183      assert resp.status_code == rq.codes.accepted
   184  
   185      # check if resume changefeed success
   186      url = BASE_URL1+"/changefeeds/changefeed-test2"
   187      for i in range(RETRY_TIME):
   188          resp = rq.get(url)
   189          assert resp.status_code == rq.codes.ok
   190          data = resp.json()
   191          if data["state"] == "normal":
   192              break
   193          time.sleep(1)
   194      assert data["state"] == "normal"
   195  
   196      # test resume changefeed failed
   197      url = BASE_URL0+"/changefeeds/changefeed-not-exists/resume"
   198      resp = rq.post(url)
   199      assert resp.status_code == rq.codes.bad_request
   200      data = resp.json()
   201      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   202  
   203      print("pass test: resume changefeed")
   204  
   205  
   206  def remove_changefeed():
   207      # remove changefeed
   208      url = BASE_URL0+"/changefeeds/changefeed-test3"
   209      resp = rq.delete(url)
   210      assert resp.status_code == rq.codes.accepted
   211  
   212      # check if remove changefeed success
   213      url = BASE_URL0+"/changefeeds/changefeed-test3"
   214      for i in range(RETRY_TIME):
   215          resp = rq.get(url)
   216          if resp.status_code == rq.codes.bad_request:
   217              break
   218          time.sleep(1)
   219      assert resp.status_code == rq.codes.bad_request
   220      assert resp.json()["error_code"] == "CDC:ErrChangeFeedNotExists"
   221  
   222      # test remove changefeed failed
   223      url = BASE_URL0+"/changefeeds/changefeed-not-exists"
   224      resp = rq.delete(url)
   225      assert (resp.status_code == rq.codes.bad_request or resp.status_code == rq.codes.internal_server_error)
   226      data = resp.json()
   227      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   228  
   229      print("pass test: remove changefeed")
   230  
   231  
   232  def rebalance_table():
   233      # rebalance_table
   234      url = BASE_URL0 + "/changefeeds/changefeed-test1/tables/rebalance_table"
   235      resp = rq.post(url)
   236      assert resp.status_code == rq.codes.accepted
   237  
   238      print("pass test: rebalance table")
   239  
   240  
   241  def move_table():
   242      # move table
   243      url = BASE_URL0 + "/changefeeds/changefeed-test1/tables/move_table"
   244      data = json.dumps({"capture_id": "test-aaa-aa", "table_id": 11})
   245      headers = {"Content-Type": "application/json"}
   246      resp = rq.post(url, data=data, headers=headers)
   247      assert resp.status_code == rq.codes.accepted
   248  
   249      # move table fail
   250      # not allow empty capture_id
   251      data = json.dumps({"capture_id": "", "table_id": 11})
   252      headers = {"Content-Type": "application/json"}
   253      resp = rq.post(url, data=data, headers=headers)
   254      assert resp.status_code == rq.codes.bad_request
   255  
   256      print("pass test: move table")
   257  
   258  
   259  def resign_owner():
   260      url = BASE_URL1 + "/owner/resign"
   261      resp = rq.post(url)
   262      assert resp.status_code == rq.codes.accepted
   263  
   264      print("pass test: resign owner")
   265  
   266  
   267  def list_capture():
   268      url = BASE_URL0 + "/captures"
   269      resp = requests_get_with_retry(url)
   270      assert resp.status_code == rq.codes.ok
   271  
   272      print("pass test: list captures")
   273  
   274  
   275  def list_processor():
   276      url = BASE_URL0 + "/processors"
   277      resp = requests_get_with_retry(url)
   278      assert resp.status_code == rq.codes.ok
   279  
   280      print("pass test: list processors")
   281  
   282  
   283  def get_processor():
   284      # list processor to get changefeed_id and capture_id 
   285      base_url = BASE_URL0 + "/processors"
   286      resp = requests_get_with_retry(base_url)
   287      assert resp.status_code == rq.codes.ok
   288      data = resp.json()[0]
   289      time.sleep(2)
   290      url = base_url + "/changefeed-test1/" + data["capture_id"]
   291      resp = requests_get_with_retry(url)
   292      # print error message for debug 
   293      if (resp.status_code != rq.codes.ok):
   294          print("request url", url)
   295          print("response status code:", resp.status_code)
   296      assert resp.status_code == rq.codes.ok
   297  
   298      # test capture_id error and cdc server no panic
   299      url = base_url + "/" + data["changefeed_id"] + "/" + "non-exist-capture-id"
   300      resp = rq.get(url)
   301      assert resp.status_code == rq.codes.bad_request
   302  
   303      print("pass test: get processors")
   304  
   305  
   306  def check_health():
   307      url = BASE_URL0 + "/health"
   308      for i in range(RETRY_TIME):
   309          resp = rq.get(url)
   310          if resp.status_code == rq.codes.ok:
   311              break
   312          time.sleep(1)
   313      assert resp.status_code == rq.codes.ok
   314  
   315      print("pass test: check health")
   316  
   317  
   318  def get_status():
   319      url = BASE_URL0 + "/status"
   320      resp = requests_get_with_retry(url)
   321      assert resp.status_code == rq.codes.ok
   322      assert resp.json()["is_owner"]
   323  
   324      print("pass test: get status")
   325  
   326  
   327  def set_log_level():
   328      url = BASE_URL0 + "/log"
   329      data = json.dumps({"log_level": "debug"})
   330      headers = {"Content-Type": "application/json"}
   331      resp = rq.post(url, data=data, headers=headers)
   332      assert resp.status_code == rq.codes.ok
   333  
   334      data = json.dumps({"log_level": "info"})
   335      resp = rq.post(url, data=data, headers=headers)
   336      assert resp.status_code == rq.codes.ok
   337  
   338      print("pass test: set log level")
   339  
   340  def get_tso():
   341      # test state: all
   342      url = BASE_URL0_V2+"/tso"
   343      data = json.dumps({})
   344      headers = {"Content-Type": "application/json"}
   345      resp = rq.post(url, data=data, headers=headers)
   346      assert resp.status_code == rq.codes.ok
   347  
   348      data = json.dumps({"pd_addrs": [PD_ADDR]})
   349      headers = {"Content-Type": "application/json"}
   350      resp = rq.post(url, data=data, headers=headers)
   351      assert resp.status_code == rq.codes.ok
   352  
   353      # wrong pd address
   354      data = json.dumps({"pd_addrs": ["http://127.0.0.1:2233"]})
   355      headers = {"Content-Type": "application/json"}
   356      resp = rq.post(url, data=data, headers=headers)
   357      assert resp.status_code != rq.codes.ok
   358  
   359      print("pass test: get tso")
   360  
   361  # util functions define belows
   362  
   363  # compose physical time and logical time into tso
   364  def compose_tso(ps, ls):
   365      return (ps << physicalShiftBits) + ls
   366  
   367  # arg1: test case name
   368  # arg2: certificates dir
   369  # arg3: sink uri
   370  if __name__ == "__main__":
   371  
   372      # test all the case as the order list in this map
   373      FUNC_MAP = {
   374          # api v1
   375          "check_health": check_health,
   376          "get_status": get_status,
   377          "create_changefeed": create_changefeed,
   378          "list_changefeed": list_changefeed,
   379          "get_changefeed": get_changefeed,
   380          "pause_changefeed": pause_changefeed,
   381          "update_changefeed": update_changefeed,
   382          "resume_changefeed": resume_changefeed,
   383          "rebalance_table": rebalance_table,
   384          "move_table": move_table,
   385          "get_processor": get_processor,
   386          "list_processor": list_processor,
   387          "set_log_level": set_log_level,
   388          "remove_changefeed": remove_changefeed,
   389          "resign_owner": resign_owner,
   390          # api v2
   391          "get_tso": get_tso
   392      }
   393  
   394      func = FUNC_MAP[sys.argv[1]]
   395      if len(sys.argv) >= 2:
   396          func(*sys.argv[2:])
   397      else:
   398          func()
   399