github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/tests/integration_tests/http_api_tls/util/test_case.py (about)

     1  import sys
     2  import os
     3  import requests as rq
     4  import time
     5  import json
     6  
     7  # the max retry time
     8  RETRY_TIME = 10
     9  
    10  BASE_URL0 = "https://127.0.0.1:8300/api/v1"
    11  BASE_URL1 = "https://127.0.0.1:8301/api/v1"
    12  
    13  BASE_URL0_V2 = "https://127.0.0.1:8300/api/v2"
    14  BASE_URL1_V2 = "https://127.0.0.1:8301/api/v2"
    15  
    16  TLS_PD_ADDR = "https://127.0.0.1:2579"
    17  SINK_URI="mysql://normal:123456@127.0.0.1:3306/"
    18  
    19  physicalShiftBits = 18
    20  # we should write some SQLs in the run.sh after call create_changefeed
    21  def create_changefeed(sink_uri):
    22      url = BASE_URL1+"/changefeeds"
    23      # create changefeed
    24      for i in range(1, 5):
    25          data = {
    26              "changefeed_id": "changefeed-test"+str(i),
    27              "sink_uri": "blackhole://",
    28              "ignore_ineligible_table": True
    29          }
    30          # set sink_uri
    31          if i == 1 and sink_uri != "":
    32              data["sink_uri"] = sink_uri
    33  
    34          data = json.dumps(data)
    35          headers = {"Content-Type": "application/json"}
    36          resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
    37          assert resp.status_code == rq.codes.accepted
    38  
    39      # create changefeed fail because sink_uri is invalid
    40      data = json.dumps({
    41          "changefeed_id": "changefeed-test",
    42          "sink_uri": "mysql://127.0.0.1:1111",
    43          "ignore_ineligible_table": True
    44      })
    45      headers = {"Content-Type": "application/json"}
    46      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
    47      assert resp.status_code == rq.codes.bad_request
    48  
    49      print("pass test: create changefeed")
    50  
    51  
    52  def list_changefeed():
    53      # test state: all
    54      url = BASE_URL0+"/changefeeds?state=all"
    55      resp = rq.get(url, cert=CERT, verify=VERIFY)
    56      assert resp.status_code == rq.codes.ok
    57  
    58      # test state: normal
    59      url = BASE_URL0+"/changefeeds?state=normal"
    60      resp = rq.get(url, cert=CERT, verify=VERIFY)
    61      assert resp.status_code == rq.codes.ok
    62      data = resp.json()
    63      for changefeed in data:
    64          assert changefeed["state"] == "normal"
    65  
    66      # test state: stopped
    67      url = BASE_URL0+"/changefeeds?state=stopped"
    68      resp = rq.get(url, cert=CERT, verify=VERIFY)
    69      assert resp.status_code == rq.codes.ok
    70      data = resp.json()
    71      for changefeed in data:
    72          assert changefeed["state"] == "stopped"
    73  
    74      print("pass test: list changefeed")
    75  
    76  def get_changefeed():
    77      # test get changefeed success
    78      url = BASE_URL0+"/changefeeds/changefeed-test1"
    79      resp = rq.get(url, cert=CERT, verify=VERIFY)
    80      assert resp.status_code == rq.codes.ok
    81  
    82      # test get changefeed failed
    83      url = BASE_URL0+"/changefeeds/changefeed-not-exists"
    84      resp = rq.get(url, cert=CERT, verify=VERIFY)
    85      assert resp.status_code == rq.codes.bad_request
    86      data = resp.json()
    87      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
    88  
    89      print("pass test: get changefeed")
    90  
    91  
    92  def pause_changefeed():
    93      # pause changefeed
    94      url = BASE_URL0+"/changefeeds/changefeed-test2/pause"
    95      for i in range(RETRY_TIME):
    96          resp = rq.post(url, cert=CERT, verify=VERIFY)
    97          if resp.status_code == rq.codes.accepted:
    98              break
    99          time.sleep(1)
   100      assert resp.status_code == rq.codes.accepted
   101      # check if pause changefeed success
   102      url = BASE_URL0+"/changefeeds/changefeed-test2"
   103      for i in range(RETRY_TIME):
   104          resp = rq.get(url, cert=CERT, verify=VERIFY)
   105          assert resp.status_code == rq.codes.ok
   106          data = resp.json()
   107          if data["state"] == "stopped":
   108              break
   109          time.sleep(1)
   110      assert data["state"] == "stopped"
   111      # test pause changefeed failed
   112      url = BASE_URL0+"/changefeeds/changefeed-not-exists/pause"
   113      resp = rq.post(url, cert=CERT, verify=VERIFY)
   114      assert resp.status_code == rq.codes.bad_request
   115      data = resp.json()
   116      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   117  
   118      print("pass test: pause changefeed")
   119  
   120  def update_changefeed():
   121      # update fail
   122      # can only update a stopped changefeed
   123      url = BASE_URL0+"/changefeeds/changefeed-test1"
   124      data = json.dumps({"mounter_worker_num": 32})
   125      headers = {"Content-Type": "application/json"}
   126      resp = rq.put(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   127      assert resp.status_code == rq.codes.bad_request
   128  
   129      # update success
   130      url = BASE_URL0+"/changefeeds/changefeed-test2"
   131      data = json.dumps({"mounter_worker_num": 32})
   132      headers = {"Content-Type": "application/json"}
   133      resp = rq.put(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   134      assert resp.status_code == rq.codes.accepted
   135  
   136      # update fail
   137      # can't update start_ts
   138      url = BASE_URL0+"/changefeeds/changefeed-test2"
   139      data = json.dumps({"start_ts": 0})
   140      headers = {"Content-Type": "application/json"}
   141      resp = rq.put(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   142      assert resp.status_code == rq.codes.bad_request
   143  
   144      print("pass test: update changefeed")
   145  
   146  
   147  def resume_changefeed():
   148      # resume changefeed
   149      url = BASE_URL1+"/changefeeds/changefeed-test2/resume"
   150      resp = rq.post(url, cert=CERT, verify=VERIFY)
   151      assert resp.status_code == rq.codes.accepted
   152  
   153      # check if resume changefeed success
   154      url = BASE_URL1+"/changefeeds/changefeed-test2"
   155      for i in range(RETRY_TIME):
   156          resp = rq.get(url, cert=CERT, verify=VERIFY)
   157          assert resp.status_code == rq.codes.ok
   158          data = resp.json()
   159          if data["state"] == "normal":
   160              break
   161          time.sleep(1)
   162      assert data["state"] == "normal"
   163  
   164      # test resume changefeed failed
   165      url = BASE_URL0+"/changefeeds/changefeed-not-exists/resume"
   166      resp = rq.post(url, cert=CERT, verify=VERIFY)
   167      assert resp.status_code == rq.codes.bad_request
   168      data = resp.json()
   169      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   170  
   171      print("pass test: resume changefeed")
   172  
   173  
   174  def remove_changefeed():
   175      # remove changefeed
   176      url = BASE_URL0+"/changefeeds/changefeed-test3"
   177      resp = rq.delete(url, cert=CERT, verify=VERIFY)
   178      assert resp.status_code == rq.codes.accepted
   179  
   180      # check if remove changefeed success
   181      url = BASE_URL0+"/changefeeds/changefeed-test3"
   182      for i in range(RETRY_TIME):
   183          resp = rq.get(url, cert=CERT, verify=VERIFY)
   184          if resp.status_code == rq.codes.bad_request:
   185              break
   186          time.sleep(1)
   187      assert resp.status_code == rq.codes.bad_request
   188      assert resp.json()["error_code"] == "CDC:ErrChangeFeedNotExists"
   189  
   190      # test remove changefeed failed
   191      url = BASE_URL0+"/changefeeds/changefeed-not-exists"
   192      resp = rq.delete(url, cert=CERT, verify=VERIFY)
   193      assert (resp.status_code == rq.codes.bad_request or resp.status_code == rq.codes.internal_server_error)
   194      data = resp.json()
   195      assert data["error_code"] == "CDC:ErrChangeFeedNotExists"
   196  
   197      print("pass test: remove changefeed")
   198  
   199  
   200  def rebalance_table():
   201      # rebalance_table
   202      url = BASE_URL0 + "/changefeeds/changefeed-test1/tables/rebalance_table"
   203      resp = rq.post(url, cert=CERT, verify=VERIFY)
   204      assert resp.status_code == rq.codes.accepted
   205  
   206      print("pass test: rebalance table")
   207  
   208  
   209  def move_table():
   210      # move table
   211      url = BASE_URL0 + "/changefeeds/changefeed-test1/tables/move_table"
   212      data = json.dumps({"capture_id": "test-aaa-aa", "table_id": 11})
   213      headers = {"Content-Type": "application/json"}
   214      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   215      assert resp.status_code == rq.codes.accepted
   216  
   217      # move table fail
   218      # not allow empty capture_id
   219      data = json.dumps({"capture_id": "", "table_id": 11})
   220      headers = {"Content-Type": "application/json"}
   221      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   222      assert resp.status_code == rq.codes.bad_request
   223  
   224      print("pass test: move table")
   225  
   226  
   227  def resign_owner():
   228      url = BASE_URL1 + "/owner/resign"
   229      resp = rq.post(url, cert=CERT, verify=VERIFY)
   230      assert resp.status_code == rq.codes.accepted
   231  
   232      print("pass test: resign owner")
   233  
   234  
   235  def list_capture():
   236      url = BASE_URL0 + "/captures"
   237      resp = rq.get(url, cert=CERT, verify=VERIFY)
   238      assert resp.status_code == rq.codes.ok
   239  
   240      print("pass test: list captures")
   241  
   242  
   243  def list_processor():
   244      url = BASE_URL0 + "/processors"
   245      resp = rq.get(url, cert=CERT, verify=VERIFY)
   246      assert resp.status_code == rq.codes.ok
   247  
   248      print("pass test: list processors")
   249  
   250  
   251  def get_processor():
   252      # list processor to get changefeed_id and capture_id 
   253      base_url = BASE_URL0 + "/processors"
   254      resp = rq.get(base_url, cert=CERT, verify=VERIFY)
   255      assert resp.status_code == rq.codes.ok
   256      data = resp.json()[0]
   257      time.sleep(2)
   258      url = base_url + "/changefeed-test1/" + data["capture_id"]
   259      resp = rq.get(url, cert=CERT, verify=VERIFY)
   260      # print error message for debug 
   261      if (resp.status_code != rq.codes.ok):
   262          print("request url", url)
   263          print("response status code:", resp.status_code)
   264          print("response body:", resp.text())
   265      assert resp.status_code == rq.codes.ok
   266  
   267      # test capture_id error and cdc server no panic
   268      url = base_url + "/" + data["changefeed_id"] + "/" + "non-exist-capture-id"
   269      resp = rq.get(url, cert=CERT, verify=VERIFY)
   270      assert resp.status_code == rq.codes.bad_request
   271  
   272      print("pass test: get processor")
   273  
   274  
   275  def check_health():
   276      url = BASE_URL0 + "/health"
   277      for i in range(RETRY_TIME):
   278          resp = rq.get(url, cert=CERT, verify=VERIFY)
   279          if resp.status_code == rq.codes.ok:
   280              break
   281          time.sleep(1)
   282      assert resp.status_code == rq.codes.ok
   283  
   284      print("pass test: check health")
   285  
   286  
   287  def get_status():
   288      url = BASE_URL0 + "/status"
   289      resp = rq.get(url,cert=CERT, verify=VERIFY)
   290      assert resp.status_code == rq.codes.ok
   291      assert resp.json()["is_owner"]
   292  
   293      print("pass test: get status")
   294  
   295  
   296  def set_log_level():
   297      url = BASE_URL0 + "/log"
   298      data = json.dumps({"log_level": "debug"})
   299      headers = {"Content-Type": "application/json"}
   300      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   301      assert resp.status_code == rq.codes.ok
   302  
   303      data = json.dumps({"log_level": "info"})
   304      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   305      assert resp.status_code == rq.codes.ok
   306  
   307      print("pass test: set log level")
   308  
   309  def verify_table():
   310      url = BASE_URL0_V2+"/tso"
   311      # we need to retry since owner resign before this func call
   312      i = 0
   313      while i < 10:
   314          try:
   315              data = json.dumps({})
   316              headers = {"Content-Type": "application/json"}
   317              resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY, timeout=5)
   318              if resp.status_code == rq.codes.ok:
   319                break
   320              else:
   321                   continue
   322          except rq.exceptions.RequestException:
   323              i += 1
   324      assert resp.status_code == rq.codes.ok
   325  
   326      ps = resp.json()["timestamp"]
   327      ls = resp.json()["logic_time"]
   328      tso = compose_tso(ps,ls)
   329  
   330      url = BASE_URL0_V2 + "/verify_table"
   331      data = json.dumps({
   332      "pd_addrs": [TLS_PD_ADDR],
   333      "ca_path":CA_PEM_PATH,
   334      "cert_path":CLIENT_PEM_PATH,
   335      "key_path":CLIENT_KEY_PEM_PATH,
   336      "cert_allowed_cn":["client"],
   337      "start_ts": tso,
   338      "replica_config": {
   339          "filter": {
   340              "rules": ["test.verify*"]
   341              }
   342          }
   343      })
   344      headers = {"Content-Type": "application/json"}
   345      for i in range(RETRY_TIME):
   346          resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   347          if resp.status_code == rq.codes.ok:
   348              break
   349          time.sleep(1)
   350      assert resp.status_code == rq.codes.ok
   351      eligible_table_name = resp.json()["eligible_tables"][0]["table_name"]
   352      ineligible_table_name = resp.json()["ineligible_tables"][0]["table_name"]
   353      assert eligible_table_name == "verify_table_eligible"
   354      assert ineligible_table_name == "verify_table_ineligible"
   355  
   356      print("pass test: verify table")
   357  
   358  
   359  def get_tso():
   360      # test state: all
   361      url = BASE_URL0_V2+"/tso"
   362      data = json.dumps({})
   363      headers = {"Content-Type": "application/json"}
   364      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   365      assert resp.status_code == rq.codes.ok
   366  
   367      print("pass test: get tso")
   368  
   369  def create_changefeed_v2():
   370      url = BASE_URL1_V2+"/changefeeds"
   371      # create changefeed 1
   372      data = {
   373          "changefeed_id": "changefeed-test-v2-black-hole-1",
   374          "sink_uri": "blackhole://",
   375          "replica_config":{
   376              "ignore_ineligible_table": True
   377              },
   378          "pd_addrs": [TLS_PD_ADDR],
   379          "ca_path":CA_PEM_PATH,
   380          "cert_path":CLIENT_PEM_PATH,
   381          "key_path":CLIENT_KEY_PEM_PATH,
   382          "cert_allowed_cn":["client"],
   383      }
   384      data = json.dumps(data)
   385      headers = {"Content-Type": "application/json"}
   386      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   387      assert resp.status_code == rq.codes.ok
   388  
   389      # create changefeed 2
   390      data = {
   391          "changefeed_id": "changefeed-test-v2-black-hole-2",
   392          "sink_uri": SINK_URI,
   393          "replica_config":{
   394              "ignore_ineligible_table": True,
   395              "filter": {
   396              "rules": ["test.verify*"]
   397              }
   398          },
   399          "pd_addrs": [TLS_PD_ADDR],
   400          "ca_path":CA_PEM_PATH,
   401          "cert_path":CLIENT_PEM_PATH,
   402          "key_path":CLIENT_KEY_PEM_PATH,
   403          "cert_allowed_cn":["client"],
   404      }
   405      data = json.dumps(data)
   406      headers = {"Content-Type": "application/json"}
   407      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   408      assert resp.status_code == rq.codes.ok
   409  
   410      # create changefeed fail because sink_uri is invalid
   411      data = json.dumps({
   412          "changefeed_id": "changefeed-test",
   413          "sink_uri": "mysql://127.0.0.1:1111",
   414          "replica_config":{
   415              "ignore_ineligible_table": True
   416              },
   417          "pd_addrs": [TLS_PD_ADDR],
   418          "ca_path":CA_PEM_PATH,
   419          "cert_path":CLIENT_PEM_PATH,
   420          "key_path":CLIENT_KEY_PEM_PATH,
   421          "cert_allowed_cn":["client"],
   422      })
   423      headers = {"Content-Type": "application/json"}
   424      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   425      assert resp.status_code == rq.codes.bad_request
   426  
   427      print("pass test: create changefeed v2")
   428  
   429  def unsafe_apis():
   430      url = BASE_URL1_V2+"/unsafe/metadata"
   431      resp = rq.get(url, cert=CERT, verify=VERIFY)
   432      assert resp.status_code == rq.codes.ok
   433      print("status code", resp.status_code)
   434      print("pass test: show metadata")
   435  
   436      # service_gc_safepoint 1
   437      url = BASE_URL1_V2+"/unsafe/service_gc_safepoint"
   438      data = {
   439          "pd_addrs": [TLS_PD_ADDR],
   440          "ca_path":CA_PEM_PATH,
   441          "cert_path":CLIENT_PEM_PATH,
   442          "key_path":CLIENT_KEY_PEM_PATH,
   443          "cert_allowed_cn":["client"],
   444      }
   445      data = json.dumps(data)
   446      headers = {"Content-Type": "application/json"}
   447      resp = rq.delete(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   448      print("status code", resp.status_code)
   449      assert resp.status_code == rq.codes.ok
   450  
   451      data = json.dumps({})
   452      headers = {"Content-Type": "application/json"}
   453      resp = rq.delete(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   454      print("status code", resp.status_code)
   455      assert resp.status_code == rq.codes.ok
   456      print("pass test: delete service_gc_safepoint")
   457  
   458      # create changefeed fail because sink_uri is invalid
   459      data = json.dumps({})
   460      url = BASE_URL1_V2+"/unsafe/resolve_lock"
   461      headers = {"Content-Type": "application/json"}
   462      resp = rq.post(url, data=data, headers=headers, cert=CERT, verify=VERIFY)
   463      print("status code", resp.status_code)
   464      assert resp.status_code != rq.codes.not_found
   465      print("pass test: resolve lock")
   466  
   467  def delete_changefeed_v2():
   468      # remove changefeed
   469      url = BASE_URL0_V2+"/changefeeds/changefeed-test4"
   470      resp = rq.delete(url, cert=CERT, verify=VERIFY)
   471      assert resp.status_code == rq.codes.ok
   472  
   473      # check if remove changefeed success
   474      url = BASE_URL0+"/changefeeds/changefeed-test4"
   475      for i in range(RETRY_TIME):
   476          resp = rq.get(url, cert=CERT, verify=VERIFY)
   477          if resp.status_code == rq.codes.bad_request:
   478              break
   479          time.sleep(1)
   480      assert resp.status_code == rq.codes.bad_request
   481      assert resp.json()["error_code"] == "CDC:ErrChangeFeedNotExists"
   482  
   483      # test remove changefeed not exists
   484      url = BASE_URL0_V2+"/changefeeds/changefeed-not-exists"
   485      resp = rq.delete(url, cert=CERT, verify=VERIFY)
   486      assert (resp.status_code == rq.codes.ok)
   487  
   488      print("pass test: remove changefeed v2")
   489  
   490  # util functions define belows
   491  
   492  # compose physical time and logical time into tso
   493  def compose_tso(ps, ls):
   494      return (ps << physicalShiftBits) + ls
   495  
   496  # arg1: test case name
   497  # arg2: cetificates dir
   498  # arg3: sink uri
   499  if __name__ == "__main__":
   500  
   501      CERTIFICATE_PATH = sys.argv[2]
   502      CLIENT_PEM_PATH = CERTIFICATE_PATH + '/client.pem'
   503      CLIENT_KEY_PEM_PATH = CERTIFICATE_PATH + '/client-key.pem'
   504      CA_PEM_PATH = CERTIFICATE_PATH + '/ca.pem'
   505      CERT=(CLIENT_PEM_PATH, CLIENT_KEY_PEM_PATH)
   506      VERIFY=(CA_PEM_PATH)
   507  
   508      # test all the case as the order list in this map
   509      FUNC_MAP = {
   510          "check_health": check_health,
   511          "get_status": get_status,
   512          "create_changefeed": create_changefeed,
   513          "list_changefeed": list_changefeed,
   514          "get_changefeed": get_changefeed,
   515          "pause_changefeed": pause_changefeed,
   516          "update_changefeed": update_changefeed,
   517          "resume_changefeed": resume_changefeed,
   518          "rebalance_table": rebalance_table,
   519          "move_table": move_table,
   520          "get_processor": get_processor,
   521          "list_processor": list_processor,
   522          "set_log_level": set_log_level,
   523          "remove_changefeed": remove_changefeed,
   524          "resign_owner": resign_owner,
   525          # api v2
   526          "get_tso": get_tso,
   527          "verify_table": verify_table,
   528          "create_changefeed_v2": create_changefeed_v2,
   529          "delete_changefeed_v2": delete_changefeed_v2,
   530          "unsafe_apis": unsafe_apis
   531      }
   532  
   533      func = FUNC_MAP[sys.argv[1]]
   534      if len(sys.argv) >= 3:
   535          func(*sys.argv[3:])
   536      else:
   537          func()