github.com/platonnetwork/platon-go@v0.7.6/cases/environment/node.py (about) 1 import os 2 import json 3 from client_sdk_python import Web3 4 from client_sdk_python.admin import Admin 5 from client_sdk_python.debug import Debug 6 from client_sdk_python.eth import Eth 7 from client_sdk_python.personal import Personal 8 from client_sdk_python.ppos import Ppos 9 from client_sdk_python.pip import Pip 10 11 from common.connect import run_ssh, connect_linux, wait_connect_web3 12 from common.load_file import LoadFile 13 from environment.config import TestConfig 14 from environment.mock import mock_connect_linux 15 from common.log import log 16 17 18 failed_msg = "Node-{} do {} failed:{}" 19 success_msg = "Node-{} do {} success" 20 21 22 class Node: 23 def __init__(self, node_conf, cfg: TestConfig, chain_id): 24 self.cfg = cfg 25 # node identity parameter 26 self.blspubkey = node_conf["blspubkey"] 27 self.blsprikey = node_conf["blsprikey"] 28 self.node_id = node_conf["id"] 29 self.nodekey = node_conf["nodekey"] 30 # node startup necessary parameters 31 self.p2p_port = str(node_conf["port"]) 32 self.rpc_port = str(node_conf["rpcport"]) 33 # node starts non essential parameters 34 self.wsport = node_conf.get("wsport") 35 self.wsurl = node_conf.get("wsurl") 36 self.pprofport = node_conf.get("pprofport") 37 self.fail_point = node_conf.get("fail_point") 38 # node server information 39 self.host = node_conf["host"] 40 self.username = node_conf["username"] 41 self.password = node_conf["password"] 42 self.ssh_port = node_conf.get("sshport", 22) 43 if self.cfg.can_deploy: 44 self.ssh, self.sftp, self.t = connect_linux(self.host, self.username, self.password, self.ssh_port) 45 else: 46 self.ssh, self.sftp, self.t = mock_connect_linux() 47 # node identification information 48 self.url = node_conf["url"] 49 self.node_name = "node-" + self.p2p_port 50 self.node_mark = self.host + ":" + self.p2p_port 51 # node remote directory information 52 if os.path.isabs(self.cfg.deploy_path): 53 self.remote_node_path = "{}/{}".format(self.cfg.deploy_path, self.node_name) 54 else: 55 self.remote_node_path = "{}/{}/{}".format(self.pwd, self.cfg.deploy_path, self.node_name) 56 57 self.remote_log_dir = '{}/log'.format(self.remote_node_path) 58 self.remote_bin_file = self.remote_node_path + "/platon" 59 self.remote_genesis_file = self.remote_node_path + "/genesis.json" 60 self.remote_config_file = self.remote_node_path + "/config.json" 61 self.remote_data_dir = self.remote_node_path + "/data" 62 63 self.remote_blskey_file = '{}/blskey'.format(self.remote_data_dir) 64 self.remote_nodekey_file = '{}/nodekey'.format(self.remote_data_dir) 65 self.remote_keystore_dir = '{}/keystore'.format(self.remote_data_dir) 66 self.remote_static_nodes_file = '{}/static-nodes.json'.format(self.remote_data_dir) 67 self.remote_db_dir = '{}/platon'.format(self.remote_data_dir) 68 69 self.remote_supervisor_node_file = '{}/{}.conf'.format(self.cfg.remote_supervisor_tmp, self.node_name) 70 71 # RPC connection 72 self.__is_connected = False 73 self.__rpc = None 74 75 self.__is_connected_ppos = False 76 self.__ppos = None 77 78 self.__is_connected_pip = False 79 self.__pip = None 80 81 self.__is_ws_connected = False 82 self.__ws_rpc = None 83 84 # remote directory 85 self.make_remote_dir() 86 87 # node local tmp 88 self.local_node_tmp = self.gen_node_tmp() 89 90 # self.genesis_config = LoadFile(self.cfg.genesis_file).get_data() 91 self.chain_id = chain_id 92 93 @property 94 def pwd(self): 95 pwd_list = self.run_ssh("pwd") 96 pwd = pwd_list[0].strip("\r\n") 97 return pwd 98 99 def make_remote_dir(self): 100 self.run_ssh("mkdir -p {}".format(self.remote_node_path)) 101 self.run_ssh('mkdir -p {}/log'.format(self.remote_node_path)) 102 self.run_ssh("mkdir -p {}".format(self.remote_data_dir)) 103 self.run_ssh('mkdir -p {}'.format(self.remote_keystore_dir)) 104 105 def gen_node_tmp(self): 106 """ 107 generate local node cache directory 108 :return: 109 """ 110 tmp = os.path.join(self.cfg.node_tmp, self.host + "_" + self.p2p_port) 111 if not os.path.exists(tmp): 112 os.makedirs(tmp) 113 return tmp 114 115 @property 116 def enode(self): 117 return r"enode://" + self.node_id + "@" + self.host + ":" + self.p2p_port 118 119 def try_do(self, func): 120 try: 121 func() 122 except Exception as e: 123 raise Exception(failed_msg.format(self.node_mark, func.__name__, e)) 124 125 def try_do_resturn(self, func): 126 try: 127 func() 128 except Exception as e: 129 return False, failed_msg.format(self.node_mark, func.__name__, e) 130 return True, success_msg.format(self.node_mark, func.__name__) 131 132 def init(self): 133 """ 134 Initialize 135 :return: 136 """ 137 def __init(): 138 cmd = '{} --datadir {} init {}'.format(self.remote_bin_file, self.remote_data_dir, self.remote_genesis_file) 139 result = self.run_ssh(cmd) 140 # todo :fix init complete 141 # Adding a query here can only alleviate the problem of starting deployment without initialization. 142 self.run_ssh("ls {}".format(self.remote_data_dir)) 143 if len(result) > 0: 144 log.error(failed_msg.format(self.node_mark, "init", result[0])) 145 raise Exception("Init failed:{}".format(result[0])) 146 log.debug("node-{} init success".format(self.node_mark)) 147 self.try_do(__init) 148 149 def run_ssh(self, cmd, need_password=False): 150 if need_password: 151 return run_ssh(self.ssh, cmd, self.password) 152 return run_ssh(self.ssh, cmd) 153 154 def clean(self): 155 """ 156 clear node data 157 :return: 158 """ 159 log.debug("Clean node:{}".format(self.node_mark)) 160 161 def __clean(): 162 is_success = self.stop() 163 if not is_success: 164 raise Exception("Stop failed") 165 self.run_ssh("sudo -S -p '' rm -rf {};mkdir -p {}".format(self.remote_node_path, self.remote_node_path), 166 True) 167 self.run_ssh("ls {}".format(self.remote_node_path)) 168 return self.try_do_resturn(__clean) 169 170 def clean_db(self): 171 """ 172 clear the node database 173 :return: 174 """ 175 def __clean_db(): 176 is_success = self.stop() 177 if not is_success: 178 raise Exception("Stop failed") 179 self.run_ssh("sudo -S -p '' rm -rf {}".format(self.remote_db_dir), True) 180 return self.try_do_resturn(__clean_db) 181 182 def clean_log(self): 183 """ 184 clear node log 185 :return: 186 """ 187 def __clean_log(): 188 is_success = self.stop() 189 if not is_success: 190 raise Exception("Stop failed") 191 self.run_ssh("rm -rf {}".format(self.remote_log_dir)) 192 self.append_log_file() 193 self.try_do(__clean_log) 194 195 def append_log_file(self): 196 """ 197 append log id 198 :return: 199 """ 200 def __append_log_file(): 201 self.run_ssh("mkdir -p {};echo {} >> {}/platon.log".format(self.remote_log_dir, self.cfg.env_id, 202 self.remote_log_dir)) 203 self.try_do(__append_log_file) 204 205 def stop(self): 206 """ 207 close node 208 :return: 209 """ 210 log.debug("Stop node:{}".format(self.node_mark)) 211 212 def __stop(): 213 self.__is_connected = False 214 self.__is_ws_connected = False 215 if not self.running: 216 return True, "{}-node is not running".format(self.node_mark) 217 self.run_ssh("sudo -S -p '' supervisorctl stop {}".format(self.node_name), True) 218 return self.try_do_resturn(__stop) 219 220 def start(self, is_init=False) -> tuple: 221 """ 222 boot node 223 :param is_init: 224 :return: 225 """ 226 log.debug("Start node:{}".format(self.node_mark)) 227 228 def __start(): 229 is_success = self.stop() 230 if not is_success: 231 raise Exception("Stop failed") 232 if is_init: 233 self.init() 234 self.append_log_file() 235 result = self.run_ssh("sudo -S -p '' supervisorctl start " + self.node_name, True) 236 for r in result: 237 if "ERROR" in r or "Command 'supervisorctl' not found" in r: 238 raise Exception("Start failed:{}".format(r.strip("\n"))) 239 240 return self.try_do_resturn(__start) 241 242 def restart(self) -> tuple: 243 """ 244 restart node 245 :return: 246 """ 247 def __restart(): 248 self.append_log_file() 249 result = self.run_ssh("sudo -S -p '' supervisorctl restart " + self.node_name, True) 250 for r in result: 251 if "ERROR" in r: 252 raise Exception("restart failed:{}".format(r.strip("\n"))) 253 return self.try_do_resturn(__restart) 254 255 def update(self) -> tuple: 256 """ 257 update node 258 :return: 259 """ 260 def __update(): 261 # todo fix me 262 self.stop() 263 self.put_bin() 264 self.start() 265 return self.try_do_resturn(__update) 266 267 def close(self): 268 """ 269 Close the node, delete the node data, 270 delete the node supervisor configuration 271 :return: 272 """ 273 is_success = True 274 msg = "close success" 275 try: 276 self.clean() 277 self.run_ssh("sudo -S -p '' rm -rf /etc/supervisor/conf.d/{}.conf".format(self.node_name), True) 278 except Exception as e: 279 is_success = False 280 msg = "{}-close failed:{}".format(self.node_mark, e) 281 finally: 282 self.t.close() 283 return is_success, msg 284 285 def put_bin(self): 286 """ 287 upload binary package 288 :return: 289 """ 290 def __put_bin(): 291 self.run_ssh("rm -rf {}".format(self.remote_bin_file)) 292 self.sftp.put(self.cfg.platon_bin_file, self.remote_node_path) 293 self.run_ssh('chmod +x {}'.format(self.remote_bin_file)) 294 self.try_do(__put_bin) 295 296 def put_nodekey(self): 297 """ 298 upload nodekey 299 :return: 300 """ 301 def __put_nodekey(): 302 nodekey_file = os.path.join(self.local_node_tmp, "nodekey") 303 with open(nodekey_file, 'w', encoding="utf-8") as f: 304 f.write(self.nodekey) 305 self.run_ssh('mkdir -p {}'.format(self.remote_data_dir)) 306 self.sftp.put(nodekey_file, self.remote_nodekey_file) 307 self.try_do(__put_nodekey) 308 309 def put_blskey(self): 310 """ 311 upload blskey 312 :return: 313 """ 314 def __put_blskey(): 315 blskey_file = os.path.join(self.local_node_tmp, "blskey") 316 with open(blskey_file, 'w', encoding="utf-8") as f: 317 f.write(self.blsprikey) 318 self.run_ssh('mkdir -p {}'.format(self.remote_data_dir)) 319 self.sftp.put(blskey_file, self.remote_blskey_file) 320 self.try_do(__put_blskey) 321 322 def create_keystore(self, password="88888888"): 323 """ 324 create a wallet 325 :param password: 326 :return: 327 """ 328 def __create_keystore(): 329 cmd = "{} account new --datadir {}".format(self.remote_bin_file, self.remote_data_dir) 330 stdin, stdout, _ = self.ssh.exec_command("source /etc/profile;%s" % cmd) 331 stdin.write(str(password) + "\n") 332 stdin.write(str(password) + "\n") 333 self.try_do(__create_keystore) 334 335 def put_genesis(self, genesis_file): 336 """ 337 upload genesis 338 :param genesis_file: 339 :return: 340 """ 341 def __put_genesis(): 342 self.run_ssh("rm -rf {}".format(self.remote_genesis_file)) 343 self.sftp.put(genesis_file, self.remote_genesis_file) 344 self.try_do(__put_genesis) 345 346 def put_config(self): 347 """ 348 upload config 349 :return: 350 """ 351 def __put_config(): 352 self.run_ssh("rm -rf {}".format(self.remote_config_file)) 353 self.sftp.put(self.cfg.config_json_tmp, self.remote_config_file) 354 self.try_do(__put_config) 355 356 def put_static(self): 357 """ 358 upload static 359 :return: 360 """ 361 def __put_static(): 362 self.sftp.put(self.cfg.static_node_tmp, self.remote_static_nodes_file) 363 self.try_do(__put_static) 364 365 def put_deploy_conf(self): 366 """ 367 upload node deployment supervisor configuration 368 :return: 369 """ 370 def __put_deploy_conf(): 371 log.debug("{}-generate supervisor deploy conf...".format(self.node_mark)) 372 supervisor_tmp_file = os.path.join(self.local_node_tmp, "{}.conf".format(self.node_name)) 373 self.__gen_deploy_conf(supervisor_tmp_file) 374 log.debug("{}-upload supervisor deploy conf...".format(self.node_mark)) 375 self.run_ssh("rm -rf {}".format(self.remote_supervisor_node_file)) 376 self.run_ssh("mkdir -p {}".format(self.cfg.remote_supervisor_tmp)) 377 self.sftp.put(supervisor_tmp_file, self.remote_supervisor_node_file) 378 self.run_ssh("sudo -S -p '' cp {} /etc/supervisor/conf.d".format(self.remote_supervisor_node_file), True) 379 self.try_do(__put_deploy_conf) 380 381 def upload_file(self, local_file, remote_file): 382 if local_file and os.path.exists(local_file): 383 self.sftp.put(local_file, remote_file) 384 else: 385 log.info("file: {} not found".format(local_file)) 386 387 def __gen_deploy_conf(self, sup_tmp_file): 388 """ 389 Generate a supervisor configuration for node deployment 390 :param sup_tmp_file: 391 :return: 392 """ 393 with open(sup_tmp_file, "w") as fp: 394 fp.write("[program:" + self.node_name + "]\n") 395 go_fail_point = "" 396 if self.fail_point: 397 go_fail_point = " GO_FAILPOINTS='{}' ".format(self.fail_point) 398 cmd = "{} --identity platon --datadir".format(self.remote_bin_file) 399 cmd = cmd + " {} --port ".format(self.remote_data_dir) + self.p2p_port 400 cmd = cmd + " --db.nogc" 401 cmd = cmd + " --gcmode archive --nodekey " + self.remote_nodekey_file 402 cmd = cmd + " --cbft.blskey " + self.remote_blskey_file 403 cmd = cmd + " --config " + self.remote_config_file 404 cmd = cmd + " --syncmode '{}'".format(self.cfg.syncmode) 405 cmd = cmd + " --debug --verbosity {}".format(self.cfg.log_level) 406 if self.pprofport: 407 cmd = cmd + " --pprof --pprofaddr 0.0.0.0 --pprofport " + str(self.pprofport) 408 if self.wsport: 409 cmd = cmd + " --ws --wsorigins '*' --wsaddr 0.0.0.0 --wsport " + str(self.wsport) 410 cmd = cmd + " --wsapi platon,debug,personal,admin,net,web3" 411 cmd = cmd + " --rpc --rpcaddr 0.0.0.0 --rpcport " + str(self.rpc_port) 412 cmd = cmd + " --rpcapi platon,debug,personal,admin,net,web3" 413 cmd = cmd + " --txpool.nolocals" 414 if self.cfg.append_cmd: 415 cmd = cmd + " " + self.cfg.append_cmd 416 fp.write("command=" + cmd + "\n") 417 if go_fail_point: 418 fp.write("environment={}\n".format(go_fail_point)) 419 supervisor_default_conf = "numprocs=1\n" + "autostart=false\n" + "startsecs=1\n" + "startretries=3\n" + \ 420 "autorestart=unexpected\n" + "exitcode=0\n" + "stopsignal=TERM\n" + \ 421 "stopwaitsecs=10\n" + "redirect_stderr=true\n" + \ 422 "stdout_logfile_maxbytes=200MB\n" + "stdout_logfile_backups=20\n" 423 fp.write(supervisor_default_conf) 424 fp.write("stdout_logfile={}/platon.log\n".format(self.remote_log_dir)) 425 426 def deploy_me(self, genesis_file) -> tuple: 427 """ 428 deploy this node 429 1. Empty environmental data 430 2. According to the node server to determine whether it is necessary to upload files 431 3. Determine whether to initialize, choose to upload genesis 432 4. Upload the node key file 433 5. Upload the inter-node supervisor configuration 434 6. Start the node 435 :param genesis_file: 436 :return: 437 """ 438 log.debug("{}-clean node path...".format(self.node_mark)) 439 is_success, msg = self.clean() 440 if not is_success: 441 return is_success, msg 442 self.clean_log() 443 is_success, msg = self.put_all_file(genesis_file) 444 if not is_success: 445 return is_success, msg 446 return self.start(self.cfg.init_chain) 447 448 def put_all_file(self, genesis_file): 449 """ 450 upload or copy the base file 451 :param genesis_file: 452 :return: 453 """ 454 def __pre_env(): 455 ls = self.run_ssh("cd {};ls".format(self.cfg.remote_compression_tmp_path)) 456 if self.cfg.env_id and (self.cfg.env_id + ".tar.gz\n") in ls: 457 log.debug("{}-copy bin...".format(self.remote_node_path)) 458 cmd = "cp -r {}/{}/* {}".format(self.cfg.remote_compression_tmp_path, self.cfg.env_id, 459 self.remote_node_path) 460 self.run_ssh(cmd) 461 self.run_ssh("chmod +x {};mkdir {}".format(self.remote_bin_file, self.remote_log_dir)) 462 else: 463 self.put_bin() 464 self.put_config() 465 # self.put_static() 466 self.create_keystore() 467 if self.cfg.init_chain: 468 log.debug("{}-upload genesis...".format(self.node_mark)) 469 self.put_genesis(genesis_file) 470 if self.cfg.is_need_static: 471 self.put_static() 472 log.debug("{}-upload blskey...".format(self.node_mark)) 473 self.put_blskey() 474 log.debug("{}-upload nodekey...".format(self.node_mark)) 475 self.put_nodekey() 476 self.put_deploy_conf() 477 self.run_ssh("sudo -S -p '' supervisorctl update " + self.node_name, True) 478 return self.try_do_resturn(__pre_env) 479 480 def backup_log(self): 481 """ 482 download log 483 :return: 484 """ 485 def __backup_log(): 486 self.run_ssh("cd {};tar zcvf log.tar.gz ./log".format(self.remote_node_path)) 487 self.sftp.get("{}/log.tar.gz".format(self.remote_node_path), 488 "{}/{}_{}.tar.gz".format(self.cfg.tmp_log, self.host, self.p2p_port)) 489 self.run_ssh("cd {};rm -rf ./log.tar.gz".format(self.remote_node_path)) 490 return self.try_do_resturn(__backup_log) 491 492 @property 493 def running(self) -> bool: 494 p_id = self.run_ssh("ps -ef|grep platon|grep port|grep %s|grep -v grep|awk {'print $2'}" % self.p2p_port) 495 if len(p_id) == 0: 496 return False 497 return True 498 499 @property 500 def web3(self) -> Web3: 501 if not self.__is_connected: 502 self.__rpc = wait_connect_web3(self.url, self.chain_id) 503 self.__is_connected = True 504 return self.__rpc 505 506 @property 507 def ws_web3(self) -> Web3: 508 if not self.__is_ws_connected: 509 self.__ws_rpc = wait_connect_web3(self.wsurl, self.chain_id) 510 self.__is_ws_connected = True 511 return self.__ws_rpc 512 513 @property 514 def eth(self) -> Eth: 515 return Eth(self.web3) 516 517 @property 518 def admin(self) -> Admin: 519 return Admin(self.web3) 520 521 @property 522 def debug(self) -> Debug: 523 return Debug(self.web3) 524 525 @property 526 def personal(self) -> Personal: 527 return Personal(self.web3) 528 529 @property 530 def ppos(self) -> Ppos: 531 if not self.__is_connected_ppos: 532 self.__ppos = Ppos(self.web3) 533 self.__is_connected_ppos = True 534 return self.__ppos 535 536 @property 537 def pip(self) -> Pip: 538 if not self.__is_connected_pip: 539 self.__pip = Pip(self.web3) 540 self.__is_connected_pip = True 541 return self.__pip 542 543 @property 544 def block_number(self) -> int: 545 return self.eth.blockNumber 546 547 @property 548 def program_version(self): 549 return self.admin.getProgramVersion()['Version'] 550 551 @property 552 def program_version_sign(self): 553 return self.admin.getProgramVersion()['Sign'] 554 555 @property 556 def schnorr_NIZK_prove(self): 557 return self.admin.getSchnorrNIZKProve() 558 559 @property 560 def staking_address(self): 561 """ 562 staking wallet address 563 """ 564 result = self.ppos.getCandidateInfo(self.node_id) 565 candidate_info = result.get('Ret', {}) 566 address = candidate_info.get('StakingAddress') 567 return self.web3.toChecksumAddress(address)