github.com/rita33cool1/iot-system-gateway@v0.0.0-20200911033302-e65bde238cc5/gateway/Implementation/Measurement/storage_usage.py (about) 1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 """ 4 """ 5 __author__ = 'YuJung Wang' 6 __date__ = '2020/05' 7 8 import os 9 import copy 10 import docker 11 12 #overall_repo = 'yujungwang/iscc19' 13 overall_repo = os.environ['DOCKER_PROVIDER'] + '/' + os.environ['DOCKER_REPO'] 14 15 16 def read_image_json(imgs_list, is_exist): 17 # Read the image information json file to get layers' information 18 images_dict = {} 19 for img in imgs_list: 20 #images_dict[overall_repo+':s2-'+img[:-1]+'-'+img[-1]] = {} 21 if 'yolo' in img: 22 images_dict[overall_repo+':s2-yolo-'+img.split('yolo')[-1]] = {} 23 elif 'audio' in img: 24 images_dict[overall_repo+':s2-audio-'+img.split('audio')[-1]] = {} 25 com_layers = {} 26 layers = {} 27 images_layers_list = [] 28 images_list = [] 29 for img in images_dict: 30 layers_list = [] 31 #print('img:', img) 32 img_name = img.split(':', -1)[-1] 33 #print('img_name:', img_name) 34 parts = [] 35 with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader: 36 parts = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1].split('},{') 37 for i in range(0, len(parts)): 38 com_size_str = parts[i].split('"CompressLayerSize":"', 1)[1].split('"', 1)[0] 39 size = parts[i].split('"LayerSize":"', 1)[1].split('"', 1)[0] 40 l_id = parts[i].split('"LayerID":"', 1)[1].split('"')[0] 41 layers_list.append(l_id) 42 if l_id not in com_layers: 43 # Convert unit to Byte 44 #com_size, unit = com_size_str.split(' ', 1) 45 unit = com_size_str[-2:] 46 com_size = com_size_str[:-2] 47 if unit == 'GB': 48 com_layers[l_id] = float(com_size)*1000*1000*1000 49 elif unit == 'MB': 50 com_layers[l_id] = float(com_size)*1000*1000 51 elif unit == 'KB': 52 com_layers[l_id] = float(com_size)*1000 53 else: # B 54 com_layers[l_id] = float(com_size_str[:-1]) 55 images_dict[img][l_id] = is_exist 56 # Bytes 57 layers[l_id] = float(size) 58 #images_list.append(overall_repo+':s2-'+img[:-1]+'-'+img[-1]) 59 images_list.append(img) 60 layers_list.reverse() 61 #print('layers_list:', layers_list) 62 images_layers_list.append(layers_list) 63 64 return images_dict, com_layers, layers, images_list, images_layers_list 65 66 67 def get_allimages_abbr(): 68 abbrs = [] 69 all_images = get_allimages() 70 for image in all_images: 71 abbrs.append(image.split(':', -1)[-1].split('-')[1]+image.split(':', -1)[-1].split('-')[-1]) 72 return abbrs 73 74 75 def get_unreplace_layers(exist_images_list): 76 unreplaced_images = [] 77 replaced_layer_nums = [] 78 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'r') as reader: 79 for line in reader.readlines(): 80 # If no replace images, then break 81 if line.strip() == '': break 82 rep_img, num_layers = line.rstrip().split(',') 83 rep_img = rep_img.split(':')[-1] 84 #unreplaced_images.append(rep_img[3:-2]+rep_img[-1]) 85 unreplaced_images.append(rep_img.split('-')[1]+rep_img.split('-')[-1]) 86 replaced_layer_nums.append(int(num_layers)) 87 # Remove existed layers 88 for img in unreplaced_images: 89 if img in exist_images_list: 90 del replaced_layer_nums[unreplaced_images.index(img)] 91 unreplaced_images.remove(img) 92 # Read the image information json file to get layers' information 93 images_dict = {} 94 for img in unreplaced_images: 95 images_dict[img] = {} 96 com_layers = {} 97 layers = {} 98 images_layers_list = [] 99 images_list = [] 100 for img in images_dict: 101 layers_list = [] 102 #img_name = 's2-' + img[:-1] + '-' + img[-1] 103 if 'yolo' in img: 104 img_name = 's2-yolo-' + img.split('yolo')[-1] 105 elif 'audio' in img: 106 img_name = 's2-audio-' + img.split('audio')[-1] 107 parts = [] 108 with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader: 109 parts = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1].split('},{') 110 # max number of existed layers 111 max_l_num = len(parts)-replaced_layer_nums[unreplaced_images.index(img)]-1 112 for i in range(0, len(parts)): 113 com_size_str = parts[i].split('"CompressLayerSize":"', 1)[1].split('"', 1)[0] 114 size = parts[i].split('"LayerSize":"', 1)[1].split('"', 1)[0] 115 l_id = parts[i].split('"LayerID":"', 1)[1].split('"')[0] 116 layers_list.append(l_id) 117 if l_id not in com_layers: 118 # Convert unit to Byte 119 #com_size, unit = com_size_str.split(' ', 1) 120 unit = com_size_str[-2:] 121 com_size = com_size_str[:-2] 122 if unit == 'GB': 123 com_layers[l_id] = float(com_size)*1000*1000*1000 124 elif unit == 'MB': 125 com_layers[l_id] = float(com_size)*1000*1000 126 elif unit == 'KB': 127 com_layers[l_id] = float(com_size)*1000 128 else: # B 129 com_layers[l_id] = float(com_size_str[:-1]) 130 # layers before max_l_num are existed (1) 131 images_dict[img][l_id] = 1 if i <= max_l_num else 0 132 # Bytes 133 layers[l_id] = float(size) 134 images_list.append(overall_repo+':s2-'+img[:-1]+'-'+img[-1]) 135 layers_list.reverse() 136 #print('layers_list:', layers_list) 137 images_layers_list.append(layers_list) 138 # Write back replacement information 139 # The new information is different from only if the unreplaced images are existed now 140 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'w') as writer: 141 for image in unreplaced_images: 142 #writer.write(overall_repo+':s2-'+image[:-1]+'-'+image[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 143 if 'yolo' in image: 144 writer.write(overall_repo+':s2-yolo-'+image.split('yolo')[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 145 elif 'audio' in image: 146 writer.write(overall_repo+':s2-audio-'+image.split('audio')[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n') 147 148 return images_dict, com_layers, layers, images_list, images_layers_list 149 150 151 #def get_available_size(total_size): 152 def get_available_size(): 153 # Get Existed Images 154 exist_images_list = get_allimages_abbr() 155 #print (exist_images_list) 156 exist_images_dict, exist_com_layers_dict, exist_layers_dict, exist_images_list, exist_layers_list = read_image_json(exist_images_list, 1) 157 # Get Unreplaced layers 158 unreplace_images_dict, unreplace_com_layers_dict, unreplace_layers_dict, unreplace_images_list, unreplace_layers_list = get_unreplace_layers(exist_images_list) 159 # Calculate the avaliable storage size 160 # Firstly, find all existed layers from existed images and unreplace layers 161 #overall_using_layers = copy.deepcopy(exist_layers_dict) 162 # Here using compressed layer size to replace layer size 163 overall_using_layers = copy.deepcopy(exist_com_layers_dict) 164 for img in unreplace_images_dict: 165 #print('unreplace image:', img) 166 for lay in unreplace_images_dict[img]: 167 #print('unreplace layer:', lay) 168 #print('overall_using_layers:', overall_using_layers) 169 #print('unreplace_images_dict[img][lay]:', unreplace_images_dict[img][lay]) 170 if lay not in overall_using_layers.keys() and unreplace_images_dict[img][lay]: 171 #print('unsing unreplace layer:', lay) 172 # Here using compressed layer size to replace layer size 173 #overall_using_layers[lay] = unreplace_layers_dict[lay] 174 overall_using_layers[lay] = unreplace_com_layers_dict[lay] 175 # Sum the size of all the existed layers 176 sum_existed_layers_size = 0 177 for lay in overall_using_layers: 178 sum_existed_layers_size += overall_using_layers[lay] 179 print('Storage usage: ' +str(sum_existed_layers_size)) 180 181 182 def get_allimages(): 183 exist_images = [] 184 client = docker.from_env() 185 images = client.images.list(name=overall_repo) 186 for image in images: 187 #print(image) 188 newstr = str(image).replace("<", "").replace(">","").replace("'","") 189 #exist_images.append(newstr.split(' ')[1]) 190 #print(newstr.split(' ')[1]) 191 img = newstr.split(' ', 1)[1] 192 if ', ' in img: 193 tmp_imgs = img.split(', ') 194 for tmp_img in tmp_imgs: 195 exist_images.append(tmp_img) 196 else: 197 exist_images.append(img) 198 return exist_images 199 200 201 def measure_storage(): 202 # Flags to indicate the image is replaced image or not 203 is_replaces = [] 204 # Survey the replaced images 205 replaced_images = [] 206 replaced_layer_nums = [] 207 with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'r') as reader: 208 for line in reader.readlines(): 209 # No replace images 210 if line.strip() == '': break 211 rep_img, num_layers = line.rstrip().split(',') 212 replaced_images.append(rep_img) 213 replaced_layer_nums.append(int(num_layers)) 214 # Get the existed images now 215 exist_images = get_allimages() 216 #print('exist_images:', exist_images) 217 # Get the total size and existed images, unreplaced images 218 # Here using compressed layer size to replace layer size 219 get_available_size() 220 221 222 223 if __name__ == '__main__': 224 measure_storage()