github.com/rita33cool1/iot-system-gateway@v0.0.0-20200911033302-e65bde238cc5/gateway/Implementation/Algo/Replacement/replacement_algo.py (about) 1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 """ 4 """ 5 __author__ = 'YuJung Wang' 6 __date__ = '2020/04' 7 8 import os 9 import sys 10 import json 11 import copy 12 import time 13 import docker 14 import random 15 import subprocess 16 17 #overall_repo = 'yujungwang/iscc19' 18 overall_repo = os.environ['DOCKER_PROVIDER'] + '/' + os.environ['DOCKER_REPO'] 19 20 im1 = overall_repo + ":s2-audio-1" 21 im2 = overall_repo + ":s2-audio-2" 22 im3 = overall_repo + ":s2-audio-3" 23 im4 = overall_repo + ":s2-audio-4" 24 im5 = overall_repo + ":s2-audio-5" 25 im6 = overall_repo + ":s2-audio-6" 26 im7 = overall_repo + ":s2-audio-7" 27 im8 = overall_repo + ":s2-audio-8" 28 im9 = overall_repo + ":s2-audio-9" 29 im10 = overall_repo + ":s2-audio-10" 30 im11 = overall_repo + ":s2-audio-11" 31 im12 = overall_repo + ":s2-audio-12" 32 im13 = overall_repo + ":s2-yolo-1" 33 im14 = overall_repo + ":s2-yolo-2" 34 im15 = overall_repo + ":s2-yolo-3" 35 im16 = overall_repo + ":s2-yolo-4" 36 im17 = overall_repo + ":s2-yolo-5" 37 im18 = overall_repo + ":s2-yolo-6" 38 im19 = overall_repo + ":s2-yolo-7" 39 im20 = overall_repo + ":s2-yolo-8" 40 im21 = overall_repo + ":s2-yolo-9" 41 im22 = overall_repo + ":s2-yolo-10" 42 im23 = overall_repo + ":s2-yolo-11" 43 im24 = overall_repo + ":s2-yolo-12" 44 #im5 = overall_repo + ":s2-audio-5" 45 #im10 = overall_repo + ":s2-yolo-5" 46 47 # Audio: 1-5, Yolo: 6-10 48 #"im5": {'max':24} 49 #"im10": {'max':22} 50 layers = { 51 "im1": {'max':21}, "im2": {'max':22}, "im3": {'max':23}, "im4": {'max':24}, 52 "im5": {'max':19}, "im6": {'max':20}, "im7": {'max':19}, "im8": {'max':20}, 53 "im9": {'max':19}, "im10": {'max':20}, "im11": {'max':19}, "im12": {'max':20}, 54 "im13": {'max':19}, "im14": {'max':21}, "im15": {'max':21}, "im16": {'max':22}, 55 "im17": {'max':16}, "im18": {'max':16}, "im19": {'max':16}, "im20": {'max':16}, 56 "im21": {'max':16}, "im22": {'max':16}, "im23": {'max':16}, "im24": {'max':16} 57 } 58 """ 59 layers = {} 60 # Audio 61 l= {"im1": {'max':21, '1':6079, '2':59604121, '3':192622442, '4':192622442, '5':224526626, '6':224927808, '7':231758631, '8':233209203, '9':492710201, '10':493187754, '11':1223903049, '12':1235941075, '13':1246100175, '14':1253593582, '15':1259647974, '16':1303490375, '17':1320370062, '18':1643412709, '19':1766682440, '20':1808151687, '21':1937224007}} 62 layers.update(l) 63 l= {"im2": {'max':22, '1':6079, '2':58025147, '3':412963334, '4':545981655, '5':545981655, '6':551323134, '7':551724316, '8':558555139, '9':560005711, '10':823141536, '11':823610078, '12':1555692026, '13':1648989999, '14':1659149099, '15':1665212599, '16':1665212631, '17':1725163120, '18':1742042807, '19':2065085454, '20':2188355185, '21':2229824432, '22':2358896752}} 64 layers.update(l) 65 l= {"im3": {'max':23, '1':6080, '2':57119574, '3':880842540, '4':1013860861, '5':1013860861, '6':1020390492, '7':1020791949, '8':1027622772, '9':1029073344, '10':1292288532, '11':1292802374, '12':2017062763, '13':2091782818, '14':2108180085, '15':2115672200, '16':2121726804, '17':2169555153, '18':2186414163, '19':2748602029, '20':2890384908, '21':2898197294, '22':2921428855, '23':3022023085}} 66 layers.update(l) 67 l= {"im4": {'max':24, '1':6080, '2':57366901, '3':333501848, '4':1157224814, '5':1290243135, '6':1290243135, '7':1296772900, '8':1297174357, '9':1304005180, '10':1305455752, '11':1567270389, '12':1567732207, '13':2297254122, '14':2371974335, '15':2388371602, '16':2394435314, '17':2394435346, '18':2459772971, '19':2476631981, '20':3038819847, '21':3180602726, '22':3188415112, '23':3211646673, '24':3312240903}} 68 layers.update(l) 69 l= {"im5": {'max':24, '1':6080, '2':58030930, '3':881753896, '4':1514770345, '5':1647788666, '6':1647788666, '7':1654318431, '8':1654719888, '9':1661550711, '10':1663001283, '11':1926137117, '12':1926605660, '13':2658203695, '14':2733415275, '15':2749812542, '16':2755876254, '17':2755876286, '18':2819763470, '19':2836622480, '20':3398810346, '21':3540593225, '22':3548405611, '23':3571637172, '24':3672231402}} 70 layers.update(l) 71 # Yolo 72 l= {"im6": {'max':19, '1':4891, '2':710107, '3':368124564, '4':368124564, '5':604566207, '6':604566207, '7':706363170, '8':706840723, '9':1437556031, '10':1449594052, '11':1459753152, '12':1467246559, '13':1473300951, '14':1517143352, '15':1534023039, '16':1857065686, '17':1980335417, '18':2021804664, '19':2150876984}} 73 layers.update(l) 74 l= {"im7": {'max':21, '1':4915, '2':690954, '3':355629141, '4':723043598, '5':723043598, '6':983980779, '7':983980779, '8':1085771903, '9':1086233735, '10':1805565746, '11':1880288088, '12':1896685355, '13':1902749067, '14':1902749099, '15':1973156886, '16':1990140806, '17':2552328672, '18':2694111551, '19':2701923937, '20':2725155498, '21':2825749728}} 75 layers.update(l) 76 l= {"im8": {'max':21, '1':4915, '2':710152, '3':824433118, '4':1191847575, '5':1191847575, '6':1452792883, '7':1452792883, '8':1554589846, '9':1555067398, '10':2285792390, '11':2347755814, '12':2364153081, '13':2371645196, '14':2377699800, '15':2425528149, '16':2442387159, '17':3004575025, '18':3146357904, '19':3154170290, '20':3177401851, '21':3277996081}} 77 layers.update(l) 78 l= {"im9": {'max':22, '1':4915, '2':696700, '3':276831647, '4':1100554613, '5':1467969070, '6':1467969070, '7':1728904113, '8':1728904113, '9':1830694730, '10':1831156548, '11':2560678459, '12':2635398667, '13':2651795934, '14':2657859646, '15':2657859678, '16':2723197303, '17':2740056313, '18':3302244179, '19':3444027058, '20':3451839444, '21':3475071005, '22':3575665235}} 79 layers.update(l) 80 l= {"im10": {'max':22, '1':4915, '2':697186, '3':633713635, '4':1457436601, '5':1824851058, '6':1824851058, '7':2085786103, '8':2085786103, '9':2187576746, '10':2188045290, '11':2919643324, '12':2994854899, '13':3011252166, '14':3017315878, '15':3017315910, '16':3081203094, '17':3098062104, '18':3660249970, '19':3802032849, '20':3809845235, '21':3833076796, '22':3933671026}} 81 layers.update(l) 82 """ 83 84 #overall_com_layers = {} 85 #overall_layers = {} 86 87 88 def read_image_json(imgs_list, is_exist): 89 # Read the image information json file to get layers' information 90 images_dict = {} 91 for img in imgs_list: 92 #images_dict[overall_repo+':s2-'+img[:-1]+'-'+img[-1]] = {} 93 if 'yolo' in img: 94 images_dict[overall_repo+':s2-yolo-'+img.split('yolo')[-1]] = {} 95 elif 'audio' in img: 96 images_dict[overall_repo+':s2-audio-'+img.split('audio')[-1]] = {} 97 com_layers = {} 98 layers = {} 99 images_layers_list = [] 100 images_list = [] 101 for img in images_dict: 102 layers_list = [] 103 #print('img:', img) 104 img_name = img.split(':', -1)[-1] 105 #print('img_name:', img_name) 106 parts = [] 107 with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader: 108 parts = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1].split('},{') 109 for i in range(0, len(parts)): 110 com_size_str = parts[i].split('"CompressLayerSize":"', 1)[1].split('"', 1)[0] 111 size = parts[i].split('"LayerSize":"', 1)[1].split('"', 1)[0] 112 l_id = parts[i].split('"LayerID":"', 1)[1].split('"')[0] 113 layers_list.append(l_id) 114 if l_id not in com_layers: 115 # Convert unit to Byte 116 #com_size, unit = com_size_str.split(' ', 1) 117 unit = com_size_str[-2:] 118 com_size = com_size_str[:-2] 119 if unit == 'GB': 120 com_layers[l_id] = float(com_size)*1000*1000*1000 121 elif unit == 'MB': 122 com_layers[l_id] = float(com_size)*1000*1000 123 elif unit == 'KB': 124 com_layers[l_id] = float(com_size)*1000 125 else: # B 126 com_layers[l_id] = float(com_size_str[:-1]) 127 images_dict[img][l_id] = is_exist 128 # Bytes 129 layers[l_id] = float(size) 130 #images_list.append(overall_repo+':s2-'+img[:-1]+'-'+img[-1]) 131 images_list.append(img) 132 layers_list.reverse() 133 #print('layers_list:', layers_list) 134 images_layers_list.append(layers_list) 135 136 return images_dict, com_layers, layers, images_list, images_layers_list 137 138 139 def get_allimages_abbr(): 140 abbrs = [] 141 all_images = get_allimages() 142 for image in all_images: 143 abbrs.append(image.split(':', -1)[-1].split('-')[1]+image.split(':', -1)[-1].split('-')[-1]) 144 return abbrs 145 146 147 def get_unreplace_layers(exist_images_list): 148 unreplaced_images = [] 149 replaced_layer_nums = [] 150 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'r') as reader: 151 for line in reader.readlines(): 152 # If no replace images, then break 153 if line.strip() == '': break 154 rep_img, num_layers = line.rstrip().split(',') 155 rep_img = rep_img.split(':')[-1] 156 #unreplaced_images.append(rep_img[3:-2]+rep_img[-1]) 157 unreplaced_images.append(rep_img.split('-')[1]+rep_img.split('-')[-1]) 158 replaced_layer_nums.append(int(num_layers)) 159 # Remove existed layers 160 for img in unreplaced_images: 161 if img in exist_images_list: 162 del replaced_layer_nums[unreplaced_images.index(img)] 163 unreplaced_images.remove(img) 164 # Read the image information json file to get layers' information 165 images_dict = {} 166 for img in unreplaced_images: 167 images_dict[img] = {} 168 com_layers = {} 169 layers = {} 170 images_layers_list = [] 171 images_list = [] 172 for img in images_dict: 173 layers_list = [] 174 #img_name = 's2-' + img[:-1] + '-' + img[-1] 175 if 'yolo' in img: 176 img_name = 's2-yolo-' + img.split('yolo')[-1] 177 elif 'audio' in img: 178 img_name = 's2-audio-' + img.split('audio')[-1] 179 parts = [] 180 with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader: 181 parts = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1].split('},{') 182 # max number of existed layers 183 max_l_num = len(parts)-replaced_layer_nums[unreplaced_images.index(img)]-1 184 for i in range(0, len(parts)): 185 com_size_str = parts[i].split('"CompressLayerSize":"', 1)[1].split('"', 1)[0] 186 size = parts[i].split('"LayerSize":"', 1)[1].split('"', 1)[0] 187 l_id = parts[i].split('"LayerID":"', 1)[1].split('"')[0] 188 layers_list.append(l_id) 189 if l_id not in com_layers: 190 # Convert unit to Byte 191 #com_size, unit = com_size_str.split(' ', 1) 192 unit = com_size_str[-2:] 193 com_size = com_size_str[:-2] 194 if unit == 'GB': 195 com_layers[l_id] = float(com_size)*1000*1000*1000 196 elif unit == 'MB': 197 com_layers[l_id] = float(com_size)*1000*1000 198 elif unit == 'KB': 199 com_layers[l_id] = float(com_size)*1000 200 else: # B 201 com_layers[l_id] = float(com_size_str[:-1]) 202 # layers before max_l_num are existed (1) 203 images_dict[img][l_id] = 1 if i <= max_l_num else 0 204 # Bytes 205 layers[l_id] = float(size) 206 if 'yolo' in img: 207 images_list.append(overall_repo+':s2-yolo-'+img.split('yolo')[-1]) 208 elif 'audio' in img: 209 images_list.append(overall_repo+':s2-audio-'+img.split('audio')[-1]) 210 layers_list.reverse() 211 #print('layers_list:', layers_list) 212 images_layers_list.append(layers_list) 213 # Write back replacement information 214 # The new information is different from only if the unreplaced images are existed now 215 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'w') as writer: 216 for image in unreplaced_images: 217 #writer.write(overall_repo+':s2-'+image[:-1]+'-'+image[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 218 if 'yolo' in image: 219 writer.write(overall_repo+':s2-yolo-'+image.split('yolo')[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 220 elif 'audio' in image: 221 writer.write(overall_repo+':s2-audio-'+image.split('audio')[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 222 223 return images_dict, com_layers, layers, images_list, images_layers_list 224 225 226 def get_available_size(total_size): 227 #print('Before total size: ' + str(total_size)) 228 ## storage GB -> B# Here using compressed layer size to replace layer size 229 #total_size *= 1000*1000*1000 230 # Get Existed Images 231 exist_images_list = get_allimages_abbr() 232 #print (exist_images_list) 233 exist_images_dict, exist_com_layers_dict, exist_layers_dict, exist_images_list, exist_layers_list = read_image_json(exist_images_list, 1) 234 # Get Unreplaced layers 235 unreplace_images_dict, unreplace_com_layers_dict, unreplace_layers_dict, unreplace_images_list, unreplace_layers_list = get_unreplace_layers(exist_images_list) 236 # Calculate the avaliable storage size 237 # Firstly, find all existed layers from existed images and unreplace layers 238 #overall_using_layers = copy.deepcopy(exist_layers_dict) 239 # Here using compressed layer size to replace layer size 240 overall_using_layers = copy.deepcopy(exist_com_layers_dict) 241 for img in unreplace_images_dict: 242 #print('unreplace image:', img) 243 for lay in unreplace_images_dict[img]: 244 #print('unreplace layer:', lay) 245 #print('overall_using_layers:', overall_using_layers) 246 #print('unreplace_images_dict[img][lay]:', unreplace_images_dict[img][lay]) 247 if lay not in overall_using_layers.keys() and unreplace_images_dict[img][lay]: 248 #print('unsing unreplace layer:', lay) 249 # Here using compressed layer size to replace layer size 250 #overall_using_layers[lay] = unreplace_layers_dict[lay] 251 overall_using_layers[lay] = unreplace_com_layers_dict[lay] 252 # Sum the size of all the existed layers 253 sum_existed_layers_size = 0 254 for lay in overall_using_layers: 255 sum_existed_layers_size += overall_using_layers[lay] 256 print('Storage usage: ' +str(sum_existed_layers_size)) 257 # Total available size = total size - size of existed layers 258 total_size -= sum_existed_layers_size 259 #total_size /= 1000*1000*1000 260 #print ('After total_size: ' + str(total_size)) 261 ''' 262 long_exist_images_list = [] 263 for image in exist_images_list: 264 long_image = image 265 if 'yolo' in image: 266 long_image = overall_repo+':'+'s2-yolo-'+image.split('yolo')[-1] 267 elif 'audio' in image: 268 long_image = overall_repo+':'+'s2-audio-'+image.split('audio')[-1] 269 long_exist_images_list.append(long_image) 270 ''' 271 #return total_size, exist_images_dict, exist_layers_dict, exist_images_list, exist_layers_list, unreplace_images_dict, unreplace_layers_dict, unreplace_images_list, unreplace_layers_list 272 # Here using compressed layer size to replace layer size 273 return total_size, exist_images_dict, exist_com_layers_dict, exist_images_list, exist_layers_list, unreplace_images_dict, unreplace_com_layers_dict, unreplace_images_list, unreplace_layers_list 274 275 276 def imID(images_name): 277 if images_name == im1: 278 return "im1" 279 elif images_name == im2: 280 return "im2" 281 elif images_name == im3: 282 return "im3" 283 elif images_name == im3: 284 return "im3" 285 elif images_name == im4: 286 return "im4" 287 elif images_name == im5: 288 return "im5" 289 elif images_name == im6: 290 return "im6" 291 elif images_name == im7: 292 return "im7" 293 elif images_name == im8: 294 return "im8" 295 elif images_name == im9: 296 return "im9" 297 elif images_name == im10: 298 return "im10" 299 elif images_name == im11: 300 return "im11" 301 elif images_name == im12: 302 return "im12" 303 elif images_name == im13: 304 return "im13" 305 elif images_name == im14: 306 return "im14" 307 elif images_name == im15: 308 return "im15" 309 elif images_name == im16: 310 return "im16" 311 elif images_name == im17: 312 return "im17" 313 elif images_name == im18: 314 return "im18" 315 elif images_name == im19: 316 return "im19" 317 elif images_name == im20: 318 return "im20" 319 elif images_name == im21: 320 return "im21" 321 elif images_name == im22: 322 return "im22" 323 elif images_name == im23: 324 return "im23" 325 elif images_name == im24: 326 return "im24" 327 328 def get_layers(images): 329 layers = [] 330 client = docker.from_env() 331 for image in images: 332 a = client.images.get(image) 333 image_info = a.attrs 334 if 'RootFS' in image_info: 335 layer_info = image_info['RootFS'] 336 if 'Layers' in layer_info: 337 layer = layer_info['Layers'] 338 for tmp in layer: 339 if tmp not in layers: 340 layers.append(tmp) 341 342 return layers 343 344 def get_images(analytics): 345 #print('get_images') 346 images = [] 347 client = docker.from_env() 348 for container in analytics: 349 a = client.containers.get(container) 350 #print(str(a.image)) 351 #newstr = str(a.image).replace("<", "").replace(">","").replace("'","") 352 #images.append(newstr.split(' ')[1]) 353 #print(a.image) 354 newstr = str(a.image).replace("<", "").replace(">","").replace("'","") 355 img = newstr.split(' ', 1)[1] 356 #print(img) 357 if ', ' in img: 358 tmp_imgs = img.split(', ') 359 for tmp_img in tmp_imgs: 360 if tmp_img not in images: 361 images.append(tmp_img) 362 elif img not in images: 363 images.append(img) 364 return images 365 366 def get_analytics(): 367 analytics = [] 368 cmd = 'docker ps| grep k8s' 369 k8s_containers_info = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') 370 lines = k8s_containers_info.split('\n') 371 for line in lines[:-1]: 372 infos = line.split() 373 if 'k8s_audio-recognition' in infos[-1] or 'k8s_object-detection' in infos[-1]: 374 analytics.append(infos[0]) 375 return analytics 376 377 def get_allimages(): 378 exist_images = [] 379 client = docker.from_env() 380 images = client.images.list(name=overall_repo) 381 for image in images: 382 #print(image) 383 newstr = str(image).replace("<", "").replace(">","").replace("'","") 384 #exist_images.append(newstr.split(' ')[1]) 385 img = newstr.split(' ', 1)[1] 386 if ', ' in img: 387 tmp_imgs = img.split(', ') 388 for tmp_img in tmp_imgs: 389 if tmp_img not in exist_images: 390 exist_images.append(tmp_img) 391 elif img not in exist_images: 392 exist_images.append(img) 393 return exist_images 394 395 def read_log(log_file, candidate_layers): 396 397 layers = [] 398 download_times = [] 399 num_access = [] 400 401 with open(log_file) as f: 402 lines = f.readlines() 403 print('line in images_download.log:') 404 for line in lines: 405 layer_id = line.split(',')[0] 406 if layer_id in candidate_layers: 407 layers.append(layer_id) 408 print(line) 409 download_times.append(float(line.split(',')[1])) 410 num_access.append(int(line.split(',')[2])) 411 412 return layers, num_access, download_times 413 414 """ 415 def get_exist_images(): 416 client = docker.from_env() 417 exist_images = client.images.list(name=overall_repo) 418 images_list = [] 419 for image in exist_images: 420 repo = str(image).split(':')[1].split(':')[0].replace("'","").replace(" ","") 421 tag = str(image).split(':')[2].replace("'","").replace(" ","").replace(">","") 422 if overall_repo == repo: 423 #name = tag[3:-2] + tag[-1] 424 images_list.append(overall_repo+':'+tag) 425 return images_list 426 """ 427 428 def get_number_times_layers(exist_images_dict, unreplace_images_dict): 429 num_using_layers = {} 430 for img in exist_images_dict: 431 for lay in exist_images_dict[img]: 432 if lay not in num_using_layers.keys(): 433 num_using_layers[lay] = 1 434 else: 435 num_using_layers[lay] += 1 436 for img in unreplace_images_dict: 437 for lay in unreplace_images_dict[img]: 438 if lay not in num_using_layers.keys() and unreplace_images_dict[img][lay]: 439 num_using_layers[lay] = 1 440 elif lay in num_using_layers.keys(): 441 num_using_layers[lay] += unreplace_images_dict[img][lay] 442 443 return num_using_layers 444 445 446 def replacement_algo(total_size, high_level, low_level, image_layers, policy): 447 # storage GB -> B 448 total_size *= 1000*1000*1000 449 high_level_sz = high_level*total_size 450 replacement_sz = (1-low_level)*total_size 451 # Flags to indicate the image is replaced image or not 452 is_replaces = [] 453 # Survey the replaced images 454 replaced_images = [] 455 replaced_layer_nums = [] 456 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'r') as reader: 457 for line in reader.readlines(): 458 # No replace images 459 if line.strip() == '': break 460 rep_img, num_layers = line.rstrip().split(',') 461 replaced_images.append(rep_img) 462 replaced_layer_nums.append(int(num_layers)) 463 # Get the existed images now 464 exist_images = get_allimages() 465 print('Before replaced_images:', replaced_images) 466 print('Before replaced_layer_nums:', replaced_layer_nums) 467 # If the image is exited now, remove the replacemant information 468 for image in replaced_images: 469 if image in exist_images: 470 del replaced_layer_nums[replaced_images.index(image)] 471 replaced_images.remove(image) 472 print('exist_images:', exist_images) 473 print('After replaced_images:', replaced_images) 474 print('After replaced_layer_nums:', replaced_layer_nums) 475 # Get the total size and existed images, unreplaced images 476 print('Before replacement_sz: ' + str(replacement_sz)) 477 #available_size, exist_images_dict, exist_layers_dict, exist_images_list, exist_layers_list, unreplace_images_dict, unreplace_layers_dict, unreplace_images_list, unreplace_layers_list = get_available_size(total_size) 478 # Here using compressed layer size to replace layer size 479 available_size, exist_images_dict, exist_com_layers_dict, exist_images_list, exist_layers_list, unreplace_images_dict, unreplace_com_layers_dict, unreplace_images_list, unreplace_layers_list = get_available_size(total_size) 480 print('Available size: ' + str(available_size)) 481 replacement_sz = replacement_sz - available_size 482 print('After replacement_sz: ' + str(replacement_sz)) 483 484 # Check wheather occupation rate is higher than high water mark or not 485 # If occupation rate is lower than high water mark, do nothing 486 print('total_size: ' + str(total_size)) 487 print('high_level_sz: ' + str(high_level_sz)) 488 if total_size-available_size < high_level_sz: 489 return [], [], [] 490 491 # Get running image 492 running_images = get_images(get_analytics()) 493 494 # Get the times of each layer is used 495 num_times_layers = get_number_times_layers(exist_images_dict, unreplace_images_dict) 496 497 replacement_layers = [] 498 replacement_images = [] 499 layers_num = len(image_layers) 500 tmp_sz = 0 501 502 if replacement_sz > 0: 503 # Existed images include invisible replaced images 504 print('Before image_layers:', image_layers) 505 image_layers = image_layers + replaced_images 506 print('image_layers + replaced_images:', image_layers) 507 # Exclude running images 508 image_layers = [l for l in image_layers if not l in running_images] 509 print('running_images:', running_images) 510 print('image_layers exclude running_images:', image_layers) 511 # Get image using information 512 images, num_access_time, download_time = read_log("/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_download.log", image_layers) 513 print('get images from download log:', images) 514 print('get num_access_time from download log:', num_access_time) 515 print('get download_time from download log:', download_time) 516 while tmp_sz < int(replacement_sz) and len(images) > 0: 517 print('images:', images) 518 print('num_access_time:', num_access_time) 519 print('download_time: ', download_time) 520 print('policy: ' + policy) 521 # FIFO doesn't work 522 if policy == 'FIFO': 523 index = download_time.index(min(download_time)) 524 elif policy == 'LRU': 525 index = download_time.index(min(download_time)) 526 elif policy == 'MRU': 527 index = download_time.index(max(download_time)) 528 elif policy == 'LFU': 529 index = num_access_time.index(min(num_access_time)) 530 elif policy == 'MFU': 531 index = num_access_time.index(max(num_access_time)) 532 print('index: ' + str(index)) 533 replacement_images.append(images[index]) 534 app_name = imID(images[index]) 535 layer_num = layers[app_name]['max'] 536 print(images[index]) 537 # Check the image is replaced images or not 538 print('unreplace_images_list:', unreplace_images_list) 539 print('replaced_images:', replaced_images) 540 if images[index] in replaced_images: 541 is_replace = True 542 print('image in replaced_images') 543 start_num = replaced_layer_nums[replaced_images.index(images[index])] 544 unreplace_image_index = unreplace_images_list.index(images[index]) 545 print('unreplace_layers_list:', unreplace_layers_list) 546 end_num = len(unreplace_layers_list[unreplace_image_index]) 547 for i in range(start_num, end_num): 548 replace_layer = unreplace_layers_list[unreplace_image_index][i] 549 #print('replace_layer:', replace_layer) 550 #print('Before num_times_layers:', num_times_layers[replace_layer]) 551 num_times_layers[replace_layer] -= 1 552 #print('After num_times_layers:', num_times_layers[replace_layer]) 553 # Only when no images using this layer, the layer could be exactly deleted 554 if num_times_layers[replace_layer] == 0: 555 #replacement_sz -= unreplace_layers_dict[replace_layer] 556 #print('Subtract unreplace layer size:', unreplace_layers_dict[replace_layer]) 557 # Here using compressed layer size to replace layer size 558 replacement_sz -= unreplace_com_layers_dict[replace_layer] 559 print('Subtract unreplace compressed layer size:', unreplace_com_layers_dict[replace_layer]) 560 if replacement_sz <= 0: 561 replacement_layers.append(str(i+1)) 562 is_replaces.append(True) 563 break 564 # Last layer 565 elif i == end_num - 1: 566 replacement_layers.append(str(i+1)) 567 is_replaces.append(True) 568 else: 569 progressive_size = 0 570 #print('exist_images_list', exist_images_list) 571 exist_image_index = exist_images_list.index(images[index]) 572 layer_num = len(exist_layers_list[exist_image_index]) 573 for i in range(layer_num): 574 exist_layer = exist_layers_list[exist_image_index][i] 575 #print('exist_layer:', exist_layer) 576 #print('Before num_times_layers:', num_times_layers[exist_layer]) 577 num_times_layers[exist_layer] -= 1 578 #print('After num_times_layers:', num_times_layers[exist_layer]) 579 # Only when no images using this layer, the layer could be exactly deleted 580 if num_times_layers[exist_layer] == 0: 581 #replacement_sz -= exist_layers_dict[exist_layer] 582 #print('Subtract exist layer size:', exist_layers_dict[exist_layer]) 583 # Here using compressed layer size to replace layer size 584 replacement_sz -= exist_com_layers_dict[exist_layer] 585 print('Subtract exist layer size:', exist_com_layers_dict[exist_layer]) 586 if replacement_sz <= 0: 587 replacement_layers.append(str(i+1)) 588 is_replaces.append(False) 589 break 590 # Last layer 591 elif i == layer_num - 1: 592 replacement_layers.append(str(i+1)) 593 is_replaces.append(False) 594 595 del images[index] 596 del download_time[index] 597 del num_access_time[index] 598 599 600 # save the replacement information in images_replace.log 601 # If the replace image has replaced before, modify the before information 602 #print('replacement_images:', replacement_images) 603 #print('replacement_layers:', replacement_layers) 604 for image in replacement_images: 605 delete_number = replacement_layers[replacement_images.index(image)] 606 if image in replaced_images: 607 unreplace_image_index = unreplace_images_list.index(image) 608 max_num = len(unreplace_layers_list[unreplace_image_index]) 609 replace_index = replaced_images.index(image) 610 # If the delete number more than the number of layers, no need to record 611 if int(delete_number) < max_num: 612 replaced_layer_nums[replace_index] = delete_number 613 else: 614 del replaced_layer_nums[replace_index] 615 replaced_images.remove(image) 616 else: 617 app_name = imID(image) 618 layer_num = layers[app_name]['max'] 619 # If the delete number more than the number of layers, no need to record 620 if int(delete_number) < layer_num: 621 replaced_images.append(image) 622 replaced_layer_nums.append(delete_number) 623 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'w') as writer: 624 for image in replaced_images: 625 writer.write(image+','+str(replaced_layer_nums[replaced_images.index(image)])+'\n') 626 # Debug 627 #print('replacement_images:', replacement_images) 628 #print('replacement_layers:', replacement_layers) 629 #print('replaced_images:', replaced_images) 630 #print('replaced_layer_nums:', replaced_layer_nums) 631 632 return replacement_images, replacement_layers, is_replaces 633 634 if __name__ == '__main__': 635 total_sz = float(sys.argv[1]) 636 high_level = float(sys.argv[2]) 637 low_level = float(sys.argv[3]) 638 policy = sys.argv[4] 639 640 start_time = time.time() 641 642 analytics = get_analytics() 643 images = get_allimages() 644 645 #candidate_layers = get_layers(images) 646 # num, layer that can delete, policy 647 images, layers, is_replaces = replacement_algo(total_sz, high_level, low_level, images, policy) 648 #images, layers = replacement_algo(replacement_sz, images, policy) 649 #print(layers) 650 print(images, layers) 651 #for image, layer in zip(images, layers): 652 for image, layer, is_replace in zip(images, layers, is_replaces): 653 running_images = get_images(get_analytics()) 654 print('Replacing image: ' + image) 655 print('Check running_images', running_images) 656 if image in running_images: 657 print('After lrp, ' + image + 'is running now. Cannot remove it') 658 continue 659 cmd = 'docker ps -a | grep Exit | cut -d " " -f 1 | xargs docker rm' 660 subprocess.run(cmd, shell=True) 661 cmd = f'sed -i "1c {layer}" /home/minion/YC/YC-docker-engine/delete_layer_num' 662 #cmd = f'sed -i "1c {layer}" YC/YC-docker-engine/delete_layer_num' 663 subprocess.run(cmd, shell=True) 664 # Restart docker engine 665 # Stop docker engine 666 cmd = f'screen -S docker_screen -X stuff ^C' 667 subprocess.run(cmd, shell=True) 668 print('Stop docker engine') 669 time.sleep(15) 670 cmd = f'screen -S docker_screen -X stuff "exit\r"' 671 subprocess.run(cmd, shell=True) 672 print('Kill docker screen') 673 time.sleep(3) 674 # Start docker screen and docker engine 675 cmd = f'screen -S docker_screen -d -m bash /home/minion/YC/iscc19/delete_all_images.sh' 676 output = subprocess.run(cmd, shell=True) 677 print('Restart docker engine') 678 time.sleep(30) 679 # If image is the replaced image, the image should pull before delete 680 if is_replace: 681 cmd = f'docker pull {image}' 682 #output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') 683 try: 684 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 685 except subprocess.CalledProcessError as e: 686 raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) 687 print('time: ' + str(time.time()-start_time)) 688 cmd = f'docker rmi {image}' 689 #output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT).decode('utf-8') 690 try: 691 subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) 692 except subprocess.CalledProcessError as e: 693 #raise RuntimeError("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) 694 print("command '{}' return with error (code {}): {}".format(e.cmd, e.returncode, e.output)) 695 #else: 696 # print('Successfully Remove layers in image: ' + image) 697 698 print('time: ' + str(time.time()-start_time))