github.com/dustinrc/deis@v1.10.1-0.20150917223407-0894a5fb979e/controller/scheduler/k8s.py (about) 1 import copy 2 import httplib 3 import json 4 import random 5 import re 6 import string 7 import time 8 9 from django.conf import settings 10 from docker import Client 11 from .states import JobState 12 from . import AbstractSchedulerClient 13 14 15 POD_TEMPLATE = '''{ 16 "kind": "Pod", 17 "apiVersion": "$version", 18 "metadata": { 19 "name": "$id" 20 }, 21 "spec": { 22 "containers": [ 23 { 24 "name": "$id", 25 "image": "$image" 26 } 27 ], 28 "restartPolicy":"Never" 29 } 30 }''' 31 32 RC_TEMPLATE = '''{ 33 "kind":"ReplicationController", 34 "apiVersion":"$version", 35 "metadata":{ 36 "name":"$name", 37 "labels":{ 38 "name":"$id" 39 } 40 }, 41 "spec":{ 42 "replicas":$num, 43 "selector":{ 44 "name":"$id", 45 "version":"$appversion", 46 "type":"$type" 47 }, 48 "template":{ 49 "metadata":{ 50 "labels":{ 51 "name":"$id", 52 "version":"$appversion", 53 "type":"$type" 54 } 55 }, 56 "spec":{ 57 "containers":[ 58 { 59 "name":"$containername", 60 "image":"$image" 61 } 62 ] 63 } 64 } 65 } 66 }''' 67 68 SERVICE_TEMPLATE = '''{ 69 "kind":"Service", 70 "apiVersion":"$version", 71 "metadata":{ 72 "name":"$name", 73 "labels":{ 74 "name":"$label" 75 } 76 }, 77 "spec":{ 78 "ports": [ 79 { 80 "port":80, 81 "targetPort":$port, 82 "protocol":"TCP" 83 } 84 ], 85 "selector":{ 86 "name":"$label", 87 "type":"$type" 88 } 89 } 90 }''' 91 92 POD_DELETE = '''{ 93 }''' 94 95 96 RETRIES = 3 97 MATCH = re.compile( 98 r'(?P<app>[a-z0-9-]+)_?(?P<version>v[0-9]+)?\.?(?P<c_type>[a-z-_]+)') 99 100 101 class KubeHTTPClient(AbstractSchedulerClient): 102 103 def __init__(self, target, auth, options, pkey): 104 super(KubeHTTPClient, self).__init__(target, auth, options, pkey) 105 self.target = settings.K8S_MASTER 106 self.port = "8080" 107 self.registry = settings.REGISTRY_HOST+":"+settings.REGISTRY_PORT 108 self.apiversion = "v1" 109 self.conn = httplib.HTTPConnection(self.target+":"+self.port) 110 111 def _get_old_rc(self, name, app_type): 112 con_app = httplib.HTTPConnection(self.target+":"+self.port) 113 con_app.request('GET', '/api/'+self.apiversion + 114 '/namespaces/'+name+'/replicationcontrollers') 115 resp = con_app.getresponse() 116 data = resp.read() 117 reason = resp.reason 118 status = resp.status 119 con_app.close() 120 if not 200 <= status <= 299: 121 errmsg = "Failed to get Replication Controllers: {} {} - {}".format( 122 status, reason, data) 123 raise RuntimeError(errmsg) 124 parsed_json = json.loads(data) 125 exists = False 126 prev_rc = [] 127 for rc in parsed_json['items']: 128 if('name' in rc['metadata']['labels'] and name == rc['metadata']['labels']['name'] and 129 'type' in rc['spec']['selector'] and app_type == rc['spec']['selector']['type']): 130 exists = True 131 prev_rc = rc 132 break 133 if exists: 134 return prev_rc 135 else: 136 return 0 137 138 def _get_rc_status(self, name, namespace): 139 conn_rc = httplib.HTTPConnection(self.target+":"+self.port) 140 conn_rc.request('GET', '/api/'+self.apiversion+'/' + 141 'namespaces/'+namespace+'/replicationcontrollers/'+name) 142 resp = conn_rc.getresponse() 143 status = resp.status 144 conn_rc.close() 145 return status 146 147 def _get_rc_(self, name, namespace): 148 conn_rc_resver = httplib.HTTPConnection(self.target+":"+self.port) 149 conn_rc_resver.request('GET', '/api/'+self.apiversion+'/' + 150 'namespaces/'+namespace+'/replicationcontrollers/'+name) 151 resp = conn_rc_resver.getresponse() 152 data = resp.read() 153 reason = resp.reason 154 status = resp.status 155 conn_rc_resver.close() 156 if not 200 <= status <= 299: 157 errmsg = "Failed to get Replication Controller:{} {} {} - {}".format( 158 name, status, reason, data) 159 raise RuntimeError(errmsg) 160 parsed_json = json.loads(data) 161 return parsed_json 162 163 def deploy(self, name, image, command, **kwargs): 164 app_name = kwargs.get('aname', {}) 165 app_type = name.split(".")[1] 166 old_rc = self._get_old_rc(app_name, app_type) 167 new_rc = self._create_rc(name, image, command, **kwargs) 168 desired = int(old_rc["spec"]["replicas"]) 169 old_rc_name = old_rc["metadata"]["name"] 170 new_rc_name = new_rc["metadata"]["name"] 171 try: 172 count = 1 173 while desired >= count: 174 new_rc = self._scale_app(new_rc_name, count, app_name) 175 old_rc = self._scale_app(old_rc_name, desired-count, app_name) 176 count += 1 177 except Exception as e: 178 self._scale_app(new_rc["metadata"]["name"], 0, app_name) 179 self._delete_rc(new_rc["metadata"]["name"], app_name) 180 self._scale_app(old_rc["metadata"]["name"], desired, app_name) 181 err = '{} (deploy): {}'.format(name, e) 182 raise RuntimeError(err) 183 self._delete_rc(old_rc_name, app_name) 184 185 def _get_events(self, namespace): 186 con_get = httplib.HTTPConnection(self.target+":"+self.port) 187 con_get.request('GET', '/api/'+self.apiversion+'/namespaces/'+namespace+'/events') 188 resp = con_get.getresponse() 189 reason = resp.reason 190 status = resp.status 191 data = resp.read() 192 con_get.close() 193 if not 200 <= status <= 299: 194 errmsg = "Failed to get events: {} {} - {}".format( 195 status, reason, data) 196 raise RuntimeError(errmsg) 197 return (status, data, reason) 198 199 def _get_schedule_status(self, name, num, namespace): 200 pods = [] 201 for _ in xrange(120): 202 count = 0 203 pods = [] 204 status, data, reason = self._get_pods(namespace) 205 parsed_json = json.loads(data) 206 for pod in parsed_json['items']: 207 if pod['metadata']['generateName'] == name+'-': 208 count += 1 209 pods.append(pod['metadata']['name']) 210 if count == num: 211 break 212 time.sleep(1) 213 for _ in xrange(120): 214 count = 0 215 status, data, reason = self._get_events(namespace) 216 parsed_json = json.loads(data) 217 for event in parsed_json['items']: 218 if(event['involvedObject']['name'] in pods and 219 event['source']['component'] == 'scheduler'): 220 if event['reason'] == 'scheduled': 221 count += 1 222 else: 223 raise RuntimeError(event['message']) 224 if count == num: 225 break 226 time.sleep(1) 227 228 def _scale_rc(self, rc, namespace): 229 name = rc['metadata']['name'] 230 num = rc["spec"]["replicas"] 231 headers = {'Content-Type': 'application/json'} 232 conn_scalepod = httplib.HTTPConnection(self.target+":"+self.port) 233 conn_scalepod.request('PUT', '/api/'+self.apiversion+'/namespaces/'+namespace+'/' + 234 'replicationcontrollers/'+name, headers=headers, body=json.dumps(rc)) 235 resp = conn_scalepod.getresponse() 236 data = resp.read() 237 reason = resp.reason 238 status = resp.status 239 conn_scalepod.close() 240 if not 200 <= status <= 299: 241 errmsg = "Failed to scale Replication Controller:{} {} {} - {}".format( 242 name, status, reason, data) 243 raise RuntimeError(errmsg) 244 resource_ver = rc['metadata']['resourceVersion'] 245 for _ in xrange(30): 246 js_template = self._get_rc_(name, namespace) 247 if js_template["metadata"]["resourceVersion"] != resource_ver: 248 break 249 time.sleep(1) 250 self._get_schedule_status(name, num, namespace) 251 for _ in xrange(120): 252 count = 0 253 status, data, reason = self._get_pods(namespace) 254 parsed_json = json.loads(data) 255 for pod in parsed_json['items']: 256 if(pod['metadata']['generateName'] == name+'-' and 257 pod['status']['phase'] == 'Running'): 258 count += 1 259 if count == num: 260 break 261 time.sleep(1) 262 263 def _scale_app(self, name, num, namespace): 264 js_template = self._get_rc_(name, namespace) 265 js_template["spec"]["replicas"] = num 266 self._scale_rc(js_template, namespace) 267 268 def scale(self, name, image, command, **kwargs): 269 app_name = kwargs.get('aname', {}) 270 rc_name = name.replace(".", "-") 271 rc_name = rc_name.replace("_", "-") 272 if not 200 <= self._get_rc_status(rc_name, app_name) <= 299: 273 self.create(name, image, command, **kwargs) 274 return 275 name = name.replace(".", "-") 276 name = name.replace("_", "-") 277 num = kwargs.get('num', {}) 278 js_template = self._get_rc_(name, app_name) 279 old_replicas = js_template["spec"]["replicas"] 280 try: 281 self._scale_app(name, num, app_name) 282 except Exception as e: 283 self._scale_app(name, old_replicas, app_name) 284 err = '{} (Scale): {}'.format(name, e) 285 raise RuntimeError(err) 286 287 def _create_rc(self, name, image, command, **kwargs): 288 container_fullname = name 289 app_name = kwargs.get('aname', {}) 290 app_type = name.split(".")[1] 291 container_name = app_name+"-"+app_type 292 name = name.replace(".", "-") 293 name = name.replace("_", "-") 294 args = command.split() 295 296 num = kwargs.get('num', {}) 297 l = {} 298 l["name"] = name 299 l["id"] = app_name 300 l["appversion"] = kwargs.get('version', {}) 301 l["version"] = self.apiversion 302 l["image"] = self.registry+"/"+image 303 l['num'] = num 304 l['containername'] = container_name 305 l['type'] = app_type 306 template = string.Template(RC_TEMPLATE).substitute(l) 307 js_template = json.loads(template) 308 containers = js_template["spec"]["template"]["spec"]["containers"] 309 containers[0]['args'] = args 310 loc = locals().copy() 311 loc.update(re.match(MATCH, container_fullname).groupdict()) 312 mem = kwargs.get('memory', {}).get(loc['c_type']) 313 cpu = kwargs.get('cpu', {}).get(loc['c_type']) 314 if mem or cpu: 315 containers[0]["resources"] = {"limits": {}} 316 if mem: 317 if mem[-2:-1].isalpha() and mem[-1].isalpha(): 318 mem = mem[:-1] 319 mem = mem+"i" 320 containers[0]["resources"]["limits"]["memory"] = mem 321 if cpu: 322 cpu = float(cpu)/1024 323 containers[0]["resources"]["limits"]["cpu"] = cpu 324 headers = {'Content-Type': 'application/json'} 325 conn_rc = httplib.HTTPConnection(self.target+":"+self.port) 326 conn_rc.request('POST', '/api/'+self.apiversion+'/namespaces/'+app_name+'/' + 327 'replicationcontrollers', headers=headers, body=json.dumps(js_template)) 328 resp = conn_rc.getresponse() 329 data = resp.read() 330 reason = resp.reason 331 status = resp.status 332 conn_rc.close() 333 if not 200 <= status <= 299: 334 errmsg = "Failed to create Replication Controller:{} {} {} - {}".format( 335 name, status, reason, data) 336 raise RuntimeError(errmsg) 337 create = False 338 for _ in xrange(30): 339 if not create and self._get_rc_status(name, app_name) == 404: 340 time.sleep(1) 341 continue 342 create = True 343 rc = self._get_rc_(name, app_name) 344 if ("observedGeneration" in rc["status"] 345 and rc["metadata"]["generation"] == rc["status"]["observedGeneration"]): 346 break 347 time.sleep(1) 348 return json.loads(data) 349 350 def create(self, name, image, command, **kwargs): 351 """Create a container.""" 352 self._create_rc(name, image, command, **kwargs) 353 app_type = name.split(".")[1] 354 name = name.replace(".", "-") 355 name = name.replace("_", "-") 356 app_name = kwargs.get('aname', {}) 357 try: 358 self._create_service(name, app_name, app_type) 359 except Exception as e: 360 self._scale_app(name, 0, app_name) 361 self._delete_rc(name, app_name) 362 err = '{} (create): {}'.format(name, e) 363 raise RuntimeError(err) 364 365 def _get_service(self, name, namespace): 366 con_get = httplib.HTTPConnection(self.target+":"+self.port) 367 con_get.request('GET', '/api/'+self.apiversion+'/namespaces/'+namespace+'/services/'+name) 368 resp = con_get.getresponse() 369 reason = resp.reason 370 status = resp.status 371 data = resp.read() 372 con_get.close() 373 if not 200 <= status <= 299: 374 errmsg = "Failed to get Service: {} {} - {}".format( 375 status, reason, data) 376 raise RuntimeError(errmsg) 377 return (status, data, reason) 378 379 def _create_service(self, name, app_name, app_type): 380 random.seed(app_name) 381 app_id = random.randint(1, 100000) 382 appname = "app-"+str(app_id) 383 actual_pod = {} 384 for _ in xrange(300): 385 status, data, reason = self._get_pods(app_name) 386 parsed_json = json.loads(data) 387 for pod in parsed_json['items']: 388 if('generateName' in pod['metadata'] and 389 pod['metadata']['generateName'] == name+'-'): 390 actual_pod = pod 391 break 392 if actual_pod and actual_pod['status']['phase'] == 'Running': 393 break 394 time.sleep(1) 395 container_id = actual_pod['status']['containerStatuses'][0]['containerID'].split("//")[1] 396 ip = actual_pod['status']['hostIP'] 397 docker_cli = Client("tcp://{}:2375".format(ip), timeout=1200, version='1.17') 398 container = docker_cli.inspect_container(container_id) 399 port = int(container['Config']['ExposedPorts'].keys()[0].split("/")[0]) 400 l = {} 401 l["version"] = self.apiversion 402 l["label"] = app_name 403 l["port"] = port 404 l['type'] = app_type 405 l["name"] = appname 406 template = string.Template(SERVICE_TEMPLATE).substitute(l) 407 headers = {'Content-Type': 'application/json'} 408 conn_serv = httplib.HTTPConnection(self.target+":"+self.port) 409 conn_serv.request('POST', '/api/'+self.apiversion+'/namespaces/'+app_name+'/services', 410 headers=headers, body=copy.deepcopy(template)) 411 resp = conn_serv.getresponse() 412 data = resp.read() 413 reason = resp.reason 414 status = resp.status 415 conn_serv.close() 416 if status == 409: 417 status, data, reason = self._get_service(appname, app_name) 418 srv = json.loads(data) 419 if srv['spec']['selector']['type'] == 'web': 420 return 421 srv['spec']['selector']['type'] = app_type 422 srv['spec']['ports'][0]['targetPort'] = port 423 headers = {'Content-Type': 'application/json'} 424 conn_scalepod = httplib.HTTPConnection(self.target+":"+self.port) 425 conn_scalepod.request('PUT', '/api/'+self.apiversion+'/namespaces/'+app_name+'/' + 426 'services/'+appname, headers=headers, body=json.dumps(srv)) 427 resp = conn_scalepod.getresponse() 428 data = resp.read() 429 reason = resp.reason 430 status = resp.status 431 conn_scalepod.close() 432 if not 200 <= status <= 299: 433 errmsg = "Failed to update the Service:{} {} {} - {}".format( 434 name, status, reason, data) 435 raise RuntimeError(errmsg) 436 elif not 200 <= status <= 299: 437 errmsg = "Failed to create Service:{} {} {} - {}".format( 438 name, status, reason, data) 439 raise RuntimeError(errmsg) 440 441 def start(self, name): 442 """Start a container.""" 443 pass 444 445 def stop(self, name): 446 """Stop a container.""" 447 pass 448 449 def _delete_rc(self, name, namespace): 450 headers = {'Content-Type': 'application/json'} 451 con_dest = httplib.HTTPConnection(self.target+":"+self.port) 452 con_dest.request('DELETE', '/api/'+self.apiversion+'/namespaces/'+namespace+'/' + 453 'replicationcontrollers/'+name, headers=headers, body=POD_DELETE) 454 resp = con_dest.getresponse() 455 reason = resp.reason 456 status = resp.status 457 data = resp.read() 458 con_dest.close() 459 if not 200 <= status <= 299: 460 errmsg = "Failed to delete Replication Controller:{} {} {} - {}".format( 461 name, status, reason, data) 462 raise RuntimeError(errmsg) 463 464 def destroy(self, name): 465 """Destroy a container.""" 466 appname = name.split("_")[0] 467 name = name.split(".") 468 name = name[0]+'-'+name[1] 469 name = name.replace("_", "-") 470 471 headers = {'Content-Type': 'application/json'} 472 con_dest = httplib.HTTPConnection(self.target+":"+self.port) 473 con_dest.request('DELETE', '/api/'+self.apiversion+'/namespaces/'+appname+'/' + 474 'replicationcontrollers/'+name, headers=headers, body=POD_DELETE) 475 resp = con_dest.getresponse() 476 reason = resp.reason 477 status = resp.status 478 data = resp.read() 479 con_dest.close() 480 if status == 404: 481 return 482 if not 200 <= status <= 299: 483 errmsg = "Failed to delete Replication Controller:{} {} {} - {}".format( 484 name, status, reason, data) 485 raise RuntimeError(errmsg) 486 487 random.seed(appname) 488 app_id = random.randint(1, 100000) 489 app_name = "app-"+str(app_id) 490 con_serv = httplib.HTTPConnection(self.target+":"+self.port) 491 con_serv.request('DELETE', '/api/'+self.apiversion + 492 '/namespaces/'+appname+'/services/'+app_name) 493 resp = con_serv.getresponse() 494 reason = resp.reason 495 status = resp.status 496 data = resp.read() 497 con_serv.close() 498 if status != 404 and not 200 <= status <= 299: 499 errmsg = "Failed to delete service:{} {} {} - {}".format( 500 name, status, reason, data) 501 raise RuntimeError(errmsg) 502 503 status, data, reason = self._get_pods(appname) 504 parsed_json = json.loads(data) 505 for pod in parsed_json['items']: 506 if 'generateName' in pod['metadata'] and pod['metadata']['generateName'] == name+'-': 507 self._delete_pod(pod['metadata']['name'], appname) 508 con_ns = httplib.HTTPConnection(self.target+":"+self.port) 509 con_ns.request('DELETE', '/api/'+self.apiversion+'/namespaces/'+appname) 510 resp = con_ns.getresponse() 511 reason = resp.reason 512 status = resp.status 513 data = resp.read() 514 con_ns.close() 515 if not 200 <= status <= 299: 516 errmsg = "Failed to delete namespace:{} {} {} - {}".format( 517 appname, status, reason, data) 518 raise RuntimeError(errmsg) 519 520 def _get_pod(self, name, namespace): 521 conn_pod = httplib.HTTPConnection(self.target+":"+self.port) 522 conn_pod.request('GET', '/api/'+self.apiversion+'/namespaces/'+namespace+'/pods/'+name) 523 resp = conn_pod.getresponse() 524 status = resp.status 525 data = resp.read() 526 reason = resp.reason 527 conn_pod.close() 528 return (status, data, reason) 529 530 def _get_pods(self, namespace): 531 con_get = httplib.HTTPConnection(self.target+":"+self.port) 532 con_get.request('GET', '/api/'+self.apiversion+'/namespaces/'+namespace+'/pods') 533 resp = con_get.getresponse() 534 reason = resp.reason 535 status = resp.status 536 data = resp.read() 537 con_get.close() 538 if not 200 <= status <= 299: 539 errmsg = "Failed to get Pods: {} {} - {}".format( 540 status, reason, data) 541 raise RuntimeError(errmsg) 542 return (status, data, reason) 543 544 def _delete_pod(self, name, namespace): 545 headers = {'Content-Type': 'application/json'} 546 con_dest_pod = httplib.HTTPConnection(self.target+":"+self.port) 547 con_dest_pod.request('DELETE', '/api/'+self.apiversion+'/namespaces/' + 548 namespace+'/pods/'+name, headers=headers, body=POD_DELETE) 549 resp = con_dest_pod.getresponse() 550 reason = resp.reason 551 status = resp.status 552 data = resp.read() 553 con_dest_pod.close() 554 if not 200 <= status <= 299: 555 errmsg = "Failed to delete Pod: {} {} - {}".format( 556 status, reason, data) 557 raise RuntimeError(errmsg) 558 for _ in xrange(5): 559 status, data, reason = self._get_pod(name, namespace) 560 if status != 404: 561 time.sleep(1) 562 continue 563 break 564 if status != 404: 565 errmsg = "Failed to delete Pod: {} {} - {}".format( 566 status, reason, data) 567 raise RuntimeError(errmsg) 568 569 def _pod_log(self, name, namespace): 570 conn_log = httplib.HTTPConnection(self.target+":"+self.port) 571 conn_log.request('GET', '/api/'+self.apiversion+'/namespaces/' + 572 namespace+'/pods/'+name+'/log') 573 resp = conn_log.getresponse() 574 status = resp.status 575 data = resp.read() 576 reason = resp.reason 577 conn_log.close() 578 if not 200 <= status <= 299: 579 errmsg = "Failed to get the log: {} {} - {}".format( 580 status, reason, data) 581 raise RuntimeError(errmsg) 582 return (status, data, reason) 583 584 def logs(self, name): 585 appname = name.split("_")[0] 586 name = name.replace(".", "-") 587 name = name.replace("_", "-") 588 status, data, reason = self._get_pods(appname) 589 parsed_json = json.loads(data) 590 log_data = '' 591 for pod in parsed_json['items']: 592 if name in pod['metadata']['generateName'] and pod['status']['phase'] == 'Running': 593 status, data, reason = self._pod_log(pod['metadata']['name'], appname) 594 log_data += data 595 return log_data 596 597 def run(self, name, image, entrypoint, command): 598 """Run a one-off command.""" 599 appname = name.split("_")[0] 600 name = name.replace(".", "-") 601 name = name.replace("_", "-") 602 l = {} 603 l["id"] = name 604 l["version"] = self.apiversion 605 l["image"] = self.registry+"/"+image 606 template = string.Template(POD_TEMPLATE).substitute(l) 607 if command.startswith("-c "): 608 args = command.split(' ', 1) 609 args[1] = args[1][1:-1] 610 else: 611 args = [command[1:-1]] 612 js_template = json.loads(template) 613 js_template['spec']['containers'][0]['command'] = [entrypoint] 614 js_template['spec']['containers'][0]['args'] = args 615 616 con_dest = httplib.HTTPConnection(self.target+":"+self.port) 617 headers = {'Content-Type': 'application/json'} 618 con_dest.request('POST', '/api/'+self.apiversion+'/namespaces/'+appname+'/pods', 619 headers=headers, body=json.dumps(js_template)) 620 resp = con_dest.getresponse() 621 data = resp.read() 622 status = resp.status 623 reason = resp.reason 624 con_dest.close() 625 if not 200 <= status <= 299: 626 errmsg = "Failed to create a Pod: {} {} - {}".format( 627 status, reason, data) 628 raise RuntimeError(errmsg) 629 while(1): 630 parsed_json = {} 631 status = 404 632 reason = '' 633 data = '' 634 for _ in xrange(5): 635 status, data, reason = self._get_pod(name, appname) 636 if not 200 <= status <= 299: 637 time.sleep(1) 638 continue 639 parsed_json = json.loads(data) 640 break 641 if not 200 <= status <= 299: 642 errmsg = "Failed to create a Pod: {} {} - {}".format( 643 status, reason, data) 644 raise RuntimeError(errmsg) 645 if parsed_json['status']['phase'] == 'Succeeded': 646 status, data, reason = self._pod_log(name, appname) 647 self._delete_pod(name, appname) 648 return 0, data 649 elif parsed_json['status']['phase'] == 'Failed': 650 pod_state = parsed_json['status']['containerStatuses'][0]['state'] 651 err_code = pod_state['terminated']['exitCode'] 652 self._delete_pod(name, appname) 653 return err_code, data 654 time.sleep(1) 655 return 0, data 656 657 def _get_pod_state(self, name): 658 try: 659 appname = name.split("_")[0] 660 name = name.split(".") 661 name = name[0]+'-'+name[1] 662 name = name.replace("_", "-") 663 for _ in xrange(120): 664 status, data, reason = self._get_pods(appname) 665 parsed_json = json.loads(data) 666 for pod in parsed_json['items']: 667 if pod['metadata']['generateName'] == name+'-': 668 actual_pod = pod 669 break 670 if actual_pod and actual_pod['status']['phase'] == 'Running': 671 return JobState.up 672 time.sleep(1) 673 return JobState.destroyed 674 except: 675 return JobState.destroyed 676 677 def state(self, name): 678 """Display the given job's running state.""" 679 try: 680 return self._get_pod_state(name) 681 except KeyError: 682 return JobState.error 683 except RuntimeError: 684 return JobState.destroyed 685 686 SchedulerClient = KubeHTTPClient