github.com/rita33cool1/iot-system-gateway@v0.0.0-20200911033302-e65bde238cc5/gateway/Implementation/Algo/Download/greedy_algo.py (about) 1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 """ 4 """ 5 __author__ = 'YuJung Wang' 6 __date__ = '2020/02' 7 8 import os 9 import sys 10 import time 11 import math 12 import json 13 import random 14 import docker 15 import warnings 16 import commands 17 import paho.mqtt.publish as publish 18 # list.copy only support since python 3.3 so use this instead 19 import copy 20 21 ## parameter 22 #registry_size = 16 # 16G 23 #network_bandwidth = 176000 # 176Mbps 24 25 overall_com_layers = {} 26 overall_layers = {} 27 overall_repo = 'yujungwang/iscc19' 28 29 30 def read_image_json(imgs_list, is_exist): 31 images_dict = {} 32 for img in imgs_list: 33 images_dict[img] = {} 34 35 com_layers = {} 36 layers = {} 37 for img in images_dict: 38 img_name = 's2-' + img[:-1] + '-' + img[-1] 39 with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader: 40 jf = json.loads(reader.read()) 41 #images_dict[img]['Layers'] = jf['Layers'] 42 for l in jf['Layers']: 43 if l['LayerID'] not in com_layers: 44 # Convert unit to Byte 45 com_size, unit = l['CompressLayerSize'].split(' ', 1) 46 if unit == 'GB': 47 com_layers[l['LayerID']] = float(com_size)*1000*1000*1000 48 elif unit == 'MB': 49 com_layers[l['LayerID']] = float(com_size)*1000*1000 50 elif unit == 'KB': 51 com_layers[l['LayerID']] = float(com_size)*1000 52 else: # B 53 com_layers[l['LayerID']] = float(com_size) 54 55 images_dict[img][l['LayerID']] = is_exist 56 # Bytes 57 layers[l['LayerID']] = float(l['LayerSize']) 58 return images_dict, com_layers, layers 59 60 61 def get_exist_images(): 62 client = docker.from_env() 63 exist_images = client.images.list() 64 images_list = [] 65 for image in exist_images: 66 repo = str(image).split(':')[1].split(':')[0].replace("'","").replace(" ","") 67 tag = str(image).split(':')[2].replace("'","").replace(" ","").replace(">","") 68 if overall_repo == repo: 69 name = tag[3:-2] + tag[-1] 70 images_list.append(name) 71 72 return images_list 73 74 75 def get_unreplace_layers(exist_images_list): 76 unreplaced_images = [] 77 replaced_layer_nums = [] 78 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'r') as reader: 79 for line in reader.readlines(): 80 try: 81 rep_img, num_layers = line.rstrip().split(',') 82 except: 83 warnings.warn('images_replace.log:\n' + reader.read() +'\nNo images in images_replace.log') 84 break 85 rep_img = rep_img.split(':')[1] 86 unreplaced_images.append(rep_img[3:-2]+rep_img[-1]) 87 replaced_layer_nums.append(int(num_layers)) 88 # Remove existed layers 89 for img in unreplaced_images: 90 if img in exist_images_list: 91 del replaced_layer_nums[unreplaced_images.index(img)] 92 unreplaced_images.remove(img) 93 # Read the image information json file to get layers' information 94 images_dict = {} 95 for img in unreplaced_images: 96 images_dict[img] = {} 97 com_layers = {} 98 layers = {} 99 for img in images_dict: 100 img_name = 's2-' + img[:-1] + '-' + img[-1] 101 parts = [] 102 with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader: 103 reads = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1] 104 #print('reads: ' + reads) 105 parts = reads.split('},{') 106 #parts = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1].split('},{') 107 # max number of existed layers 108 max_l_num = len(parts)-replaced_layer_nums[unreplaced_images.index(img)]-1 109 for i in range(0, len(parts)): 110 #values = parts[i].split('":"') 111 #com_size_str = values[1].split('"', 1)[0] 112 #size = values[2].split('"', 1)[0] 113 #l_id = values[3].split('"', 1)[0] 114 com_size_str = parts[i].split('"CompressLayerSize":"', 1)[1].split('"', 1)[0] 115 size = parts[i].split('"LayerSize":"', 1)[1].split('"', 1)[0] 116 l_id = parts[i].split('"LayerID":"', 1)[1].split('"')[0] 117 if l_id not in com_layers: 118 # Convert unit to Byte 119 #com_size, unit = com_size_str.split(' ', 1) 120 unit = com_size_str[-2:] 121 com_size = com_size_str[:-2] 122 if unit == 'GB': 123 com_layers[l_id] = float(com_size)*1000*1000*1000 124 elif unit == 'MB': 125 com_layers[l_id] = float(com_size)*1000*1000 126 elif unit == 'KB': 127 com_layers[l_id] = float(com_size)*1000 128 else: # B 129 com_layers[l_id] = float(com_size_str[:-1]) 130 # layers before max_l_num are existed (1) 131 images_dict[img][l_id] = 1 if i <= max_l_num else 0 132 # Bytes 133 layers[l_id] = float(size) 134 # Write back replacement information 135 # The new information is different from only if the unreplaced images are existed now 136 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'w') as writer: 137 for image in unreplaced_images: 138 writer.write(overall_repo+':s2-'+image[:-1]+'-'+image[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 139 140 # Debug Message 141 #print('Get unreplaced layers information') 142 #print('images_dict:', images_dict) 143 #print('com_layers:', com_layers) 144 #print('layers:', layers) 145 #print('keys:', images_dict.keys()) 146 return images_dict, com_layers, layers 147 148 149 def get_layer_size(container, layers): 150 layer_size = 0 151 com_layer_size = 0 152 for l in layers: 153 if not layers[l]: 154 layer_size += overall_layers[l] 155 com_layer_size += overall_com_layers[l] 156 container['LAYER'] = layer_size 157 container['COMLAYER'] = com_layer_size 158 159 160 def get_available_size(total_size): 161 #print('Before total size: ' + str(total_size)) 162 # storage GB -> B 163 total_size *= 1000*1000*1000 164 # Get Existed Images 165 exist_images_list = get_exist_images() 166 #print (exist_images_list) 167 exist_images_dict, exist_com_layers_dict, exist_layers_dict = read_image_json(exist_images_list, 1) 168 # Get Unreplaced layers 169 unreplace_images_dict, unreplace_com_layers_dict, unreplace_layers_dict = get_unreplace_layers(exist_images_list) 170 # Calculate the avallable storage size 171 # Firstly, find all existed layers from existed images and unreplace layers 172 overall_using_layers = copy.deepcopy(exist_layers_dict) 173 for img in unreplace_images_dict: 174 #print('unreplace image:', img) 175 for lay in unreplace_images_dict[img]: 176 #print('unreplace layer:', lay) 177 #print('overall_using_layers:', overall_using_layers) 178 #print('unreplace_images_dict[img][lay]:', unreplace_images_dict[img][lay]) 179 if lay not in overall_using_layers.keys() and unreplace_images_dict[img][lay]: 180 #print('unsing unreplace layer:', lay) 181 overall_using_layers[lay] = unreplace_layers_dict[lay] 182 # Sum the size of all the existed layers 183 sum_existed_layers_size = 0 184 for lay in overall_using_layers: 185 sum_existed_layers_size += overall_using_layers[lay] 186 #print('sum_existed_layers_size: ' +str(sum_existed_layers_size)) 187 # Total available size = total size - size of existed layers 188 total_size -= sum_existed_layers_size 189 total_size /= 1000*1000*1000 190 #print ('After total_size: ' + str(total_size)) 191 192 return total_size 193 194 def get_containers_info(): 195 analytics = [] 196 k8s_containers = [] 197 client = docker.from_env() 198 containers = client.containers.list() 199 for container in containers: 200 newstr = str(container).replace("<", "").replace(">","") 201 analytics.append(newstr.split(':')[1].replace(" ","")) 202 203 err, k8s_containers_info = commands.getstatusoutput('docker ps | grep k8s') 204 infos = ' '.join(k8s_containers_info.split()).split(' ') 205 for i in range(len(infos)): 206 if i % 10 == 0: 207 k8s_containers.append(infos[i]) 208 try: 209 analytics.remove(infos[i][:-2]) 210 except: 211 # print "cannot remove "+infos[i] 212 pass 213 return analytics 214 215 def get_analytics(containers): 216 analytics = [] 217 client = docker.from_env() 218 for container in containers: 219 a = client.containers.get(container) 220 image = str(a.image).split(':')[1].replace("'","").replace(" ","") 221 tag = str(a.image).split(':')[2].replace("'","").replace(">","") 222 223 analytics.append(image+":"+tag) 224 return analytics 225 226 def get_cost(analytics): 227 costs = {} 228 yolo = {"yolo1":{"CPU":105.2,"RAM":13.08,"SIZE":2.15,"BW":150}, 229 "yolo2":{"CPU":105.2,"RAM":13.08,"SIZE":2.83,"BW":150}, 230 "yolo3":{"CPU":105.2,"RAM":13.08,"SIZE":3.28,"BW":150}, 231 "yolo4":{"CPU":105.2,"RAM":13.08,"SIZE":3.58,"BW":150}, 232 "yolo5":{"CPU":105.2,"RAM":13.08,"SIZE":3.93,"BW":150} 233 } 234 audio = {"audio1":{"CPU":99.51,"RAM":7.84,"SIZE":1.94,"BW":3260}, 235 "audio2":{"CPU":99.51,"RAM":7.84,"SIZE":2.36,"BW":3260}, 236 "audio3":{"CPU":99.51,"RAM":7.84,"SIZE":3.02,"BW":3260}, 237 "audio4":{"CPU":99.51,"RAM":7.84,"SIZE":3.31,"BW":3260}, 238 "audio5":{"CPU":99.51,"RAM":7.84,"SIZE":3.67,"BW":3260}, 239 } 240 costs.update(yolo) 241 costs.update(audio) 242 243 return costs 244 245 246 def download_algo(total_size, bandwidth, analytics_cost, input_analytics): 247 da = {} 248 max_cpu = 400 # x 249 max_ram = 100 # y 250 #max_sz = total_size 251 max_sz = get_available_size(total_size) 252 253 analytics = [] 254 255 256 if input_analytics == []: 257 258 return [] 259 260 #return input_analytics[0] 261 audio_analytics = [a.split('-')[0] for a in input_analytics if 'audio' in a] 262 yolo_analytics = [a.split('-')[0] for a in input_analytics if 'yolo' in a] 263 264 while True: 265 num = len(audio_analytics) 266 if num > 0: 267 i = random.randint(0,num-1) 268 if max_sz - analytics_cost[audio_analytics[i]]['SIZE'] >= 0: 269 analytics.append(audio_analytics[i]) 270 max_sz -= analytics_cost[audio_analytics[i]]['SIZE'] 271 del audio_analytics[i] 272 else: 273 del audio_analytics[i] 274 else: 275 break 276 277 while True: 278 num = len(yolo_analytics) 279 if num > 0: 280 i = random.randint(0,num-1) 281 if max_sz - analytics_cost[yolo_analytics[i]]['SIZE'] >= 0: 282 analytics.append(yolo_analytics[i]) 283 max_sz -= analytics_cost[yolo_analytics[i]]['SIZE'] 284 del yolo_analytics[i] 285 else: 286 del yolo_analytics[i] 287 else: 288 break 289 return analytics 290 291 def is_exist(image): 292 client = docker.from_env() 293 images = client.images.list() 294 if image in images: 295 return True 296 return False 297 298 def write_log(edge_analytics): 299 name = [] 300 download_time = [] 301 access_num = [] 302 not_exist_num = 0 303 f = open("YC/iscc19/Implementation/Algo/Replacement/images_download.log","r") 304 for line in f: 305 name.append(line.split(",")[0]) 306 download_time.append(line.split(",")[1]) 307 access_num.append(int(line.split(",")[2])) 308 f.close() 309 310 for analytic in edge_analytics: 311 app = analytic[:-1] 312 version = analytic[-1] 313 analytic_name = "yujungwang/iscc19:s2-"+app+"-"+version 314 index = name.index(analytic_name) 315 if is_exist(analytic): 316 access_num[index] += 1 317 else: 318 not_exist_num += 1 319 access_num[index] += 1 320 download_time[index] = time.time() 321 content = name[index]+','+str(download_time[index])+','+str(access_num[index]) 322 os.popen('sed -i "'+str(index+1)+'c '+content+'" YC/iscc19/Implementation/Algo/Replacement/images_download.log') 323 324 return not_exist_num 325 326 def read(filename): 327 num = 0 328 with open(filename) as f: 329 analytics = f.readlines() 330 analytics = [item.rstrip().split("-")[0] for item in analytics] 331 num = len(analytics) 332 return num, analytics 333 334 if __name__ == '__main__': 335 336 #epsilon = float(sys.argv[1]) 337 registry_size = float(sys.argv[1]) 338 network_bandwidth = int(sys.argv[2]) 339 analytics_file = sys.argv[3] 340 341 start_time = time.time() 342 343 num, master_analytics = read(analytics_file) 344 345 #containers = get_containers_info() 346 #print '== continers: ==\n', containers 347 #analytics = get_analytics(containers) 348 #print '== analytics: ==\n' , analytics+master_analytics 349 analytics_cost = get_cost(master_analytics) 350 #print '== cost of each analytics: ==\n', analytics_cost 351 352 decision = download_algo(registry_size, network_bandwidth, analytics_cost, master_analytics) 353 #decision = download_algo(registry_size, network_bandwidth, analytics_cost, epsilon, master_analytics) 354 #decision = download_algo(registry_size, network_bandwidth, analytics_cost, layers, epsilon) 355 edge = decision 356 m_analytics = [a.split('-')[0] for a in master_analytics] 357 358 for a in edge: 359 m_analytics.remove(a) 360 361 cloud = m_analytics 362 #cloud = map(lambda s:s.strip(), list(set(m_analytics) - set(edge))) 363 #cloud ,edge= test_algo(master_analytics, registry_size, analytics_cost) 364 365 print 'Deploy to cloud>',';'.join(cloud),',Deploy at edge>',';'.join(edge) 366 367 if len(edge) > 0: 368 delete_num = write_log([edge[0]]) 369 370 print('time: ' + str(time.time()-start_time))