github.com/rita33cool1/iot-system-gateway@v0.0.0-20200911033302-e65bde238cc5/gateway/Implementation/Algo/Download/greedy_algo.py (about)

     1  #!/usr/bin/env python
     2  # -*- coding: utf-8 -*-
     3  """
     4  """
     5  __author__ = 'YuJung Wang'
     6  __date__ = '2020/02'
     7  
     8  import os
     9  import sys
    10  import time
    11  import math
    12  import json
    13  import random
    14  import docker
    15  import warnings
    16  import commands
    17  import paho.mqtt.publish as publish
    18  # list.copy only support since python 3.3 so use this instead
    19  import copy
    20  
    21  ## parameter
    22  #registry_size = 16 # 16G
    23  #network_bandwidth = 176000 # 176Mbps
    24  
    25  overall_com_layers = {}
    26  overall_layers = {}
    27  overall_repo = 'yujungwang/iscc19'
    28  
    29  
    30  def read_image_json(imgs_list, is_exist):
    31      images_dict = {}
    32      for img in imgs_list:
    33          images_dict[img] = {}
    34  
    35      com_layers = {}
    36      layers = {}
    37      for img in images_dict:
    38          img_name = 's2-' + img[:-1] + '-' + img[-1]
    39          with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader:
    40              jf = json.loads(reader.read())
    41              #images_dict[img]['Layers'] = jf['Layers']
    42              for l in jf['Layers']:
    43                  if l['LayerID'] not in com_layers:
    44                      # Convert unit to Byte
    45                      com_size, unit = l['CompressLayerSize'].split(' ', 1)
    46                      if unit == 'GB':
    47                          com_layers[l['LayerID']] = float(com_size)*1000*1000*1000
    48                      elif unit == 'MB':
    49                          com_layers[l['LayerID']] = float(com_size)*1000*1000
    50                      elif unit == 'KB':
    51                          com_layers[l['LayerID']] = float(com_size)*1000
    52                      else: # B
    53                          com_layers[l['LayerID']] = float(com_size)
    54  
    55                  images_dict[img][l['LayerID']] = is_exist
    56                  # Bytes
    57                  layers[l['LayerID']] = float(l['LayerSize'])
    58      return images_dict, com_layers, layers
    59  
    60  
    61  def get_exist_images():
    62      client = docker.from_env()
    63      exist_images = client.images.list()
    64      images_list = []
    65      for image in exist_images:
    66          repo = str(image).split(':')[1].split(':')[0].replace("'","").replace(" ","")
    67          tag = str(image).split(':')[2].replace("'","").replace(" ","").replace(">","")
    68          if overall_repo == repo:
    69              name = tag[3:-2] + tag[-1]
    70              images_list.append(name)
    71  
    72      return images_list
    73  
    74  
    75  def get_unreplace_layers(exist_images_list):
    76      unreplaced_images = []
    77      replaced_layer_nums = []
    78      with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'r') as reader:
    79          for line in reader.readlines():
    80              try: 
    81                  rep_img, num_layers = line.rstrip().split(',')
    82              except:
    83                  warnings.warn('images_replace.log:\n' + reader.read() +'\nNo images in images_replace.log')
    84                  break
    85              rep_img = rep_img.split(':')[1]
    86              unreplaced_images.append(rep_img[3:-2]+rep_img[-1])
    87              replaced_layer_nums.append(int(num_layers))
    88      # Remove existed layers
    89      for img in unreplaced_images:
    90          if img in exist_images_list:
    91              del replaced_layer_nums[unreplaced_images.index(img)]
    92              unreplaced_images.remove(img)
    93      # Read the image information json file to get layers' information
    94      images_dict = {}
    95      for img in unreplaced_images:
    96          images_dict[img] = {}
    97      com_layers = {}
    98      layers = {}
    99      for img in images_dict:
   100          img_name = 's2-' + img[:-1] + '-' + img[-1]
   101          parts = []
   102          with open('/home/minion/YC/iscc19/Implementation/Algo/Download/image_'+img_name+'.json', 'r') as reader:
   103              reads = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1]
   104              #print('reads: ' + reads)
   105              parts = reads.split('},{')
   106              #parts = reader.read().replace('\n', '').replace(' ', '').split('[{', 1)[1].split('},{')
   107          # max number of existed layers
   108          max_l_num = len(parts)-replaced_layer_nums[unreplaced_images.index(img)]-1
   109          for i in range(0, len(parts)):
   110              #values = parts[i].split('":"')
   111              #com_size_str = values[1].split('"', 1)[0]
   112              #size = values[2].split('"', 1)[0]
   113              #l_id = values[3].split('"', 1)[0]
   114              com_size_str = parts[i].split('"CompressLayerSize":"', 1)[1].split('"', 1)[0]
   115              size = parts[i].split('"LayerSize":"', 1)[1].split('"', 1)[0]
   116              l_id = parts[i].split('"LayerID":"', 1)[1].split('"')[0]
   117              if l_id not in com_layers:
   118                  # Convert unit to Byte
   119                  #com_size, unit = com_size_str.split(' ', 1)
   120                  unit = com_size_str[-2:]
   121                  com_size = com_size_str[:-2]
   122                  if unit == 'GB':
   123                      com_layers[l_id] = float(com_size)*1000*1000*1000
   124                  elif unit == 'MB':
   125                      com_layers[l_id] = float(com_size)*1000*1000
   126                  elif unit == 'KB':
   127                      com_layers[l_id] = float(com_size)*1000
   128                  else: # B
   129                      com_layers[l_id] = float(com_size_str[:-1])
   130              # layers before max_l_num are existed (1)
   131              images_dict[img][l_id] = 1 if i <= max_l_num else 0
   132              # Bytes
   133              layers[l_id] = float(size)
   134      # Write back replacement information
   135      # The new information is different from only if the unreplaced images are existed now
   136      with open('/home/minion/YC/iscc19/Implementation/Algo/Replacement/images_replace.log', 'w') as writer:
   137          for image in unreplaced_images:
   138              writer.write(overall_repo+':s2-'+image[:-1]+'-'+image[-1]+','+str(replaced_layer_nums[unreplaced_images.index(image)])+'\n')
   139  
   140      # Debug Message
   141      #print('Get unreplaced layers information')
   142      #print('images_dict:', images_dict)
   143      #print('com_layers:', com_layers)
   144      #print('layers:', layers)
   145      #print('keys:', images_dict.keys())
   146      return images_dict, com_layers, layers
   147  
   148  
   149  def get_layer_size(container, layers):
   150      layer_size = 0
   151      com_layer_size = 0
   152      for l in layers:
   153          if not layers[l]:
   154              layer_size += overall_layers[l]
   155              com_layer_size += overall_com_layers[l]
   156      container['LAYER'] = layer_size
   157      container['COMLAYER'] = com_layer_size
   158  
   159  
   160  def get_available_size(total_size):
   161      #print('Before total size: ' + str(total_size))
   162      # storage GB -> B
   163      total_size *= 1000*1000*1000
   164      # Get Existed Images
   165      exist_images_list = get_exist_images()
   166      #print (exist_images_list)
   167      exist_images_dict, exist_com_layers_dict, exist_layers_dict = read_image_json(exist_images_list, 1)
   168      # Get Unreplaced layers
   169      unreplace_images_dict, unreplace_com_layers_dict, unreplace_layers_dict = get_unreplace_layers(exist_images_list)      
   170      # Calculate the avallable storage size
   171      # Firstly, find all existed layers from existed images and unreplace layers
   172      overall_using_layers = copy.deepcopy(exist_layers_dict)
   173      for img in unreplace_images_dict:
   174          #print('unreplace image:', img)
   175          for lay in unreplace_images_dict[img]:
   176              #print('unreplace layer:', lay)
   177              #print('overall_using_layers:', overall_using_layers)
   178              #print('unreplace_images_dict[img][lay]:', unreplace_images_dict[img][lay])
   179              if lay not in overall_using_layers.keys() and unreplace_images_dict[img][lay]:
   180                  #print('unsing unreplace layer:', lay)
   181                  overall_using_layers[lay] = unreplace_layers_dict[lay]
   182      # Sum the size of all the existed layers
   183      sum_existed_layers_size = 0
   184      for lay in overall_using_layers:
   185          sum_existed_layers_size += overall_using_layers[lay]
   186      #print('sum_existed_layers_size: ' +str(sum_existed_layers_size))
   187      # Total available size = total size - size of existed layers
   188      total_size -= sum_existed_layers_size
   189      total_size /= 1000*1000*1000
   190      #print ('After total_size: ' + str(total_size))
   191      
   192      return total_size
   193      
   194  def get_containers_info():
   195      analytics = []
   196      k8s_containers = []
   197      client = docker.from_env()
   198      containers = client.containers.list()
   199      for container in containers:
   200          newstr = str(container).replace("<", "").replace(">","")
   201          analytics.append(newstr.split(':')[1].replace(" ",""))
   202  
   203      err, k8s_containers_info = commands.getstatusoutput('docker ps | grep k8s')
   204      infos = ' '.join(k8s_containers_info.split()).split(' ')
   205      for i in range(len(infos)):
   206          if i % 10 == 0:
   207             k8s_containers.append(infos[i])
   208             try:
   209                 analytics.remove(infos[i][:-2])
   210             except:
   211                # print "cannot remove "+infos[i]
   212                pass
   213      return analytics
   214  
   215  def get_analytics(containers):
   216      analytics = []
   217      client = docker.from_env()
   218      for container in containers:
   219          a = client.containers.get(container)
   220          image = str(a.image).split(':')[1].replace("'","").replace(" ","")
   221          tag = str(a.image).split(':')[2].replace("'","").replace(">","")
   222  
   223          analytics.append(image+":"+tag)
   224      return analytics
   225  
   226  def get_cost(analytics):
   227      costs = {}
   228      yolo = {"yolo1":{"CPU":105.2,"RAM":13.08,"SIZE":2.15,"BW":150},
   229      "yolo2":{"CPU":105.2,"RAM":13.08,"SIZE":2.83,"BW":150},
   230      "yolo3":{"CPU":105.2,"RAM":13.08,"SIZE":3.28,"BW":150},
   231      "yolo4":{"CPU":105.2,"RAM":13.08,"SIZE":3.58,"BW":150},
   232      "yolo5":{"CPU":105.2,"RAM":13.08,"SIZE":3.93,"BW":150}
   233  }
   234      audio = {"audio1":{"CPU":99.51,"RAM":7.84,"SIZE":1.94,"BW":3260},
   235      "audio2":{"CPU":99.51,"RAM":7.84,"SIZE":2.36,"BW":3260},
   236      "audio3":{"CPU":99.51,"RAM":7.84,"SIZE":3.02,"BW":3260},
   237      "audio4":{"CPU":99.51,"RAM":7.84,"SIZE":3.31,"BW":3260},
   238      "audio5":{"CPU":99.51,"RAM":7.84,"SIZE":3.67,"BW":3260},
   239  }
   240      costs.update(yolo)
   241      costs.update(audio)
   242  
   243      return costs
   244  
   245  
   246  def download_algo(total_size, bandwidth, analytics_cost, input_analytics):
   247      da = {}
   248      max_cpu = 400 # x
   249      max_ram = 100 # y
   250      #max_sz = total_size 
   251      max_sz = get_available_size(total_size)
   252  
   253      analytics = []
   254  
   255  
   256      if input_analytics == []:
   257  
   258         return []
   259  
   260      #return input_analytics[0]
   261      audio_analytics = [a.split('-')[0] for a in input_analytics if 'audio' in a]  
   262      yolo_analytics = [a.split('-')[0] for a in input_analytics if 'yolo' in a]
   263      
   264      while True:
   265            num = len(audio_analytics)
   266            if num > 0:
   267               i = random.randint(0,num-1)
   268               if max_sz - analytics_cost[audio_analytics[i]]['SIZE'] >= 0:
   269                  analytics.append(audio_analytics[i])
   270                  max_sz -= analytics_cost[audio_analytics[i]]['SIZE']
   271                  del audio_analytics[i]
   272               else:
   273                  del audio_analytics[i]
   274            else:
   275               break
   276  
   277      while True:
   278            num = len(yolo_analytics)
   279            if num > 0:
   280               i = random.randint(0,num-1)
   281               if max_sz - analytics_cost[yolo_analytics[i]]['SIZE'] >= 0:
   282                  analytics.append(yolo_analytics[i])
   283                  max_sz -= analytics_cost[yolo_analytics[i]]['SIZE']
   284                  del yolo_analytics[i]
   285               else:
   286                  del yolo_analytics[i]
   287            else:
   288               break
   289      return analytics
   290  
   291  def is_exist(image):
   292      client = docker.from_env()
   293      images = client.images.list()
   294      if image in images:
   295         return True
   296      return False
   297  
   298  def write_log(edge_analytics):
   299      name = []
   300      download_time = []
   301      access_num = []
   302      not_exist_num = 0
   303      f = open("YC/iscc19/Implementation/Algo/Replacement/images_download.log","r")
   304      for line in f:
   305          name.append(line.split(",")[0])
   306          download_time.append(line.split(",")[1])
   307          access_num.append(int(line.split(",")[2]))
   308      f.close()
   309  
   310      for analytic in edge_analytics:
   311          app = analytic[:-1]
   312          version = analytic[-1]
   313          analytic_name = "yujungwang/iscc19:s2-"+app+"-"+version
   314          index = name.index(analytic_name)
   315          if is_exist(analytic):
   316             access_num[index] += 1
   317          else:
   318             not_exist_num += 1
   319             access_num[index] += 1
   320             download_time[index] = time.time()
   321          content = name[index]+','+str(download_time[index])+','+str(access_num[index])
   322          os.popen('sed -i "'+str(index+1)+'c '+content+'" YC/iscc19/Implementation/Algo/Replacement/images_download.log')
   323  
   324      return not_exist_num
   325  
   326  def read(filename):
   327      num = 0
   328      with open(filename) as f:
   329           analytics = f.readlines()
   330      analytics = [item.rstrip().split("-")[0] for item in analytics]
   331      num = len(analytics)
   332      return num, analytics
   333  
   334  if __name__ == '__main__':
   335  
   336      #epsilon = float(sys.argv[1])
   337      registry_size = float(sys.argv[1])
   338      network_bandwidth = int(sys.argv[2])
   339      analytics_file = sys.argv[3]
   340  
   341      start_time = time.time()
   342  
   343      num, master_analytics = read(analytics_file)
   344       
   345      #containers = get_containers_info()
   346      #print '== continers:  ==\n', containers
   347      #analytics = get_analytics(containers)
   348      #print '== analytics:  ==\n' , analytics+master_analytics
   349      analytics_cost = get_cost(master_analytics) 
   350      #print '== cost of each analytics: ==\n', analytics_cost
   351  
   352      decision = download_algo(registry_size, network_bandwidth, analytics_cost, master_analytics)
   353      #decision = download_algo(registry_size, network_bandwidth, analytics_cost, epsilon, master_analytics)
   354      #decision = download_algo(registry_size, network_bandwidth, analytics_cost, layers, epsilon)
   355      edge =  decision
   356      m_analytics = [a.split('-')[0]  for a in master_analytics]
   357  
   358      for a in edge:
   359          m_analytics.remove(a)
   360  
   361      cloud = m_analytics
   362      #cloud = map(lambda s:s.strip(), list(set(m_analytics) - set(edge)))
   363      #cloud ,edge= test_algo(master_analytics, registry_size, analytics_cost)
   364      
   365      print 'Deploy to cloud>',';'.join(cloud),',Deploy at edge>',';'.join(edge) 
   366  
   367      if len(edge) > 0:  
   368         delete_num = write_log([edge[0]])
   369  
   370      print('time: ' + str(time.time()-start_time))