go.dedis.ch/onet/v4@v4.0.0-pre1/simul/platform/mininet/start.py (about) 1 #!/usr/bin/python 2 3 """ 4 This will run a number of hosts on the server and do all 5 the routing to being able to connect to the other mininets. 6 7 You have to give it a list of server/net/nbr for each server 8 that has mininet installed and what subnet should be run 9 on it. 10 11 It will create nbr+1 entries for each net, where the ".1" is the 12 router for the net, and ".2"..".nbr+1" will be the nodes. 13 """ 14 15 from __future__ import print_function 16 import sys, time, threading, os, datetime, contextlib, errno, platform, shutil, re 17 from mininet.topo import Topo 18 from mininet.net import Mininet 19 from mininet.cli import CLI 20 from mininet.log import lg, setLogLevel 21 from mininet.node import Node, Host, OVSController 22 from mininet.util import netParse, ipAdd, irange 23 from mininet.nodelib import NAT 24 from mininet.link import TCLink 25 from subprocess import Popen, PIPE, call 26 from mininet.node import OVSController 27 28 # What debugging-level to use 29 debugLvl = 1 30 # Debug-string for color, time and padding 31 debugStr = "" 32 # Logging-file 33 logfile = "/tmp/mininet.log" 34 logdone = "/tmp/done.log" 35 # Whether a ssh-daemon should be launched 36 runSSHD = False 37 # The port used for socat 38 socatPort = 5000 39 # Socat formats 40 # socatSend = "tcp4" 41 # socatRcv = "tcp4-listen" 42 socatSend = "udp-sendto" 43 socatRcv = "udp4-listen" 44 # Whether to redirect all socats to the main-gateway at 10.1.0.1 45 socatDirect = True 46 # If we want to end up in the CLI 47 startCLI = False 48 # 10.internalNet.x.y will be used for the ip2ip tunnels. ICCluster uses 49 # 10.0/16 for it's own internal network, and 10.90/16 for access to the machines. 50 # So we take 10.89/16 for our ip2ip tunnels, limiting the total number of 51 # servers (not conodes) to 88 52 internalNet = 89 53 54 def dbg(lvl, *str): 55 if lvl <= debugLvl: 56 print("start.py:", end=" ") 57 for s in str: 58 print(s, end=" ") 59 print() 60 sys.stdout.flush() 61 62 class BaseRouter(Node): 63 """"A Node with IP forwarding enabled.""" 64 def config( self, rootLog=None, **params ): 65 super(BaseRouter, self).config(**params) 66 dbg(3, "mynet", myNet) 67 ourIP = myNet[0][0] 68 localIndex = myNet[1] + 1 69 dbg( 2, "Starting router %s at %s, IP=%s, index %d" %( self.IP(), rootLog, ourIP, localIndex) ) 70 71 remoteIndex = 0 72 for (gw, n, i) in otherNets: 73 # First of alll we create a ip2ip tunnel using the `ip` command to 74 # create a network for all servers to communicate. This is needed so 75 # that the different nodes can communicate with each other over this 76 # network. Every server1-server2 pair gets two new IPs: 77 # 10.internalNet.localIndex.remoteIndex on the local computer 78 # and 79 # 10.internalNet.remoteIndex.localIndex on the remote computer 80 # these tunnels are then used to route the traffic from the different 81 # nodes in mininet to each other, always going through a bandwidht 82 # and latency restricted network of mininet. 83 remoteIndex += 1 84 if remoteIndex == localIndex: 85 remoteIndex += 1 86 87 dbg(3, self.cmd('hostname')) 88 tun = 'ipip%d' % remoteIndex 89 if not re.search(r'does not exist', self.cmd('ip a show dev %s', tun)): 90 dbg(3, 'removing device %s' % tun) 91 self.cmd('ip tun del %s' % tun) 92 tunLocal = '10.%d.%d.%d' %(internalNet, localIndex, remoteIndex) 93 tunRemote = '10.%d.%d.%d' %(internalNet, remoteIndex, localIndex) 94 dbg(3, "Adding tunnel %s with ips %s<->%s" % (tun, tunLocal, tunRemote)) 95 self.cmd('ip tun add %s mode ipip local %s remote %s' % (tun, ourIP, gw)) 96 self.cmd('ip address add dev %s %s peer %s/32' % (tun, tunLocal, tunRemote)) 97 self.cmd('ip link set dev %s up' % tun) 98 99 dbg( 3, "Adding route for %s through %s" % (n, tunRemote) ) 100 self.cmd( 'route add -net %s gw %s' % (n, tunRemote) ) 101 if runSSHD: 102 self.cmd('/usr/sbin/sshd -D &') 103 104 self.cmd( 'sysctl net.ipv4.ip_forward=1' ) 105 self.cmd( 'iptables -t nat -I POSTROUTING -j MASQUERADE' ) 106 socat = "socat OPEN:%s,creat,append %s:%d,reuseaddr,fork" % (logfile, socatRcv, socatPort) 107 self.cmd( '%s &' % socat ) 108 if rootLog: 109 self.cmd('tail -f %s | socat - %s:%s:%d &' % (logfile, socatSend, rootLog, socatPort)) 110 111 def terminate( self ): 112 dbg( 2, "Stopping router" ) 113 for (gw, n, i) in otherNets: 114 dbg( 3, "Deleting route for", n, gw ) 115 self.cmd( 'route del -net %s gw %s' % (n, gw) ) 116 117 self.cmd( 'sysctl net.ipv4.ip_forward=0' ) 118 self.cmd( 'killall socat' ) 119 self.cmd( 'iptables -t nat -D POSTROUTING -j MASQUERADE' ) 120 super(BaseRouter, self).terminate() 121 122 123 class Conode(Host): 124 """A conode running in a host""" 125 def config(self, gw=None, simul="", suite="Ed25519", rootLog=None, **params): 126 self.gw = gw 127 self.simul = simul 128 self.suite = suite 129 self.rootLog = rootLog 130 super(Conode, self).config(**params) 131 if runSSHD: 132 self.cmd('/usr/sbin/sshd -D &') 133 134 def startConode(self): 135 if self.rootLog and socatDirect: 136 socat="socat - %s:%s:%d" % (socatSend, self.rootLog, socatPort) 137 else: 138 socat="socat - %s:%s:%d" % (socatSend, self.gw, socatPort) 139 140 args = "-debug %s -address %s:2000 -simul %s -suite %s" % (debugLvl, self.IP(), self.simul, self.suite) 141 if True: 142 args += " -monitor %s:10000" % global_root 143 ldone = "" 144 # When the first conode on a physical server ends, tell `start.py` 145 # to go on. ".0.1" is the BaseRouter. 146 if self.IP().endswith(".0.2"): 147 ldone = "; date > " + logdone 148 dbg( 3, "Starting conode on node", self.IP(), args, ldone, socat ) 149 self.cmd('( %s ./conode %s 2>&1 %s ) | %s &' % 150 (debugStr, args, ldone, socat )) 151 152 def terminate(self): 153 dbg( 3, "Stopping conode" ) 154 self.cmd('killall socat conode') 155 super(Conode, self).terminate() 156 157 158 class InternetTopo(Topo): 159 """Create one switch with all hosts connected to it and host 160 .1 as router - all in subnet 10.x.0.0/16""" 161 def __init__(self, myNet=None, rootLog=None, **opts): 162 Topo.__init__(self, **opts) 163 server, mn, n = myNet[0] 164 switch = self.addSwitch('s0') 165 baseIp, prefix = netParse(mn) 166 gw = ipAdd(1, prefix, baseIp) 167 dbg( 2, "Gw", gw, "baseIp", baseIp, prefix, 168 "Bandwidth:", bandwidth, "- delay:", delay) 169 hostgw = self.addNode('h0', cls=BaseRouter, 170 ip='%s/%d' % (gw, prefix), 171 inNamespace=False, 172 rootLog=rootLog) 173 self.addLink(switch, hostgw) 174 175 for i in range(1, int(n) + 1): 176 ipStr = ipAdd(i + 1, prefix, baseIp) 177 host = self.addHost('h%d' % i, cls=Conode, 178 ip = '%s/%d' % (ipStr, prefix), 179 defaultRoute='via %s' % gw, 180 simul=simulation, suite=suite, gw=gw, 181 rootLog=rootLog) 182 dbg( 3, "Adding link", host, switch ) 183 self.addLink(host, switch, bw=bandwidth, delay=delay) 184 185 def RunNet(): 186 """RunNet will start the mininet and add the routes to the other 187 mininet-services""" 188 rootLog = None 189 if myNet[1] > 0: 190 i, p = netParse(otherNets[0][1]) 191 rootLog = ipAdd(1, p, i) 192 dbg( 2, "Creating network", myNet ) 193 topo = InternetTopo(myNet=myNet, rootLog=rootLog) 194 dbg( 3, "Starting on", myNet ) 195 196 net = Mininet(topo=topo, link=TCLink, controller = OVSController) 197 net.start() 198 199 for host in net.hosts[1:]: 200 host.startConode() 201 202 # Also set setLogLevel('info') if you want to use this, else 203 # there is no correct reporting on commands. 204 if startCLI: 205 CLI(net) 206 log = open(logfile, "r") 207 while not os.path.exists(logdone): 208 dbg( 4, "Waiting for conode to finish at " + platform.node() ) 209 try: 210 print(log.read(), end="") 211 sys.stdout.flush() 212 except IOError: 213 time.sleep(1) 214 215 time.sleep(1) 216 217 # Write last line of log 218 print(log.read(), end="") 219 sys.stdout.flush() 220 log.close() 221 222 dbg( 2, "conode is finished %s" % myNet ) 223 net.stop() 224 225 def GetNetworks(filename): 226 """GetServer reads the file and parses the data according to 227 server, net, count 228 It returns the first server encountered, our network if our ip is found 229 in the list and the other networks.""" 230 231 global simulation, suite, bandwidth, delay, socatDirect, debugLvl, debugStr, preScript 232 233 process = Popen(["ip", "a"], stdout=PIPE) 234 (ips, err) = process.communicate() 235 process.wait() 236 237 with open(filename) as f: 238 content = f.readlines() 239 240 # Interpret the first two lines of the file with regard to the 241 # simulation to run 242 simulation, suite, bw, d = content.pop(0).rstrip().split(' ') 243 bandwidth = int(bw) 244 delay = d + "ms" 245 dbgLvl, dbgTime, dbgColor, dbgPadding = content.pop(0).rstrip().split(' ') 246 debugLvl = int(dbgLvl) 247 if dbgTime == "true": 248 debugStr = "DEBUG_TIME=true " 249 if dbgColor == "true": 250 debugStr += "DEBUG_COLOR=true" 251 if dbgPadding == "false": 252 debugStr += "DEBUG_PADDING=false" 253 preScript = content.pop(0).rstrip().split(' ')[0] 254 255 list = [] 256 for line in content: 257 list.append(line.rstrip().split(' ')) 258 259 otherNets = [] 260 myNet = None 261 pos = 0 262 totalHosts = 0 263 for (server, net, count) in list: 264 totalHosts += int(count) 265 t = [server, net, count] 266 if ips.find('inet %s/' % server) >= 0: 267 myNet = [t, pos] 268 else: 269 otherNets.append(t) 270 pos += 1 271 272 if totalHosts > 2000: 273 dbg(0, "Redirection output through local gateway") 274 socatDirect = False 275 276 if preScript != "": 277 dbg(0, "Running PreScript " + preScript) 278 call("./%s mininet" % preScript, shell=True) 279 280 return list[0][0], myNet, otherNets 281 282 283 def rm_file(file): 284 try: 285 os.remove(file) 286 except OSError: 287 pass 288 289 290 def call_other(server, list_file): 291 dbg( 3, "Calling remote server with", server, list_file ) 292 call("ssh -q %s sudo python -u start.py %s" % (server, list_file), shell=True) 293 dbg( 3, "Done with start.py" ) 294 295 # The only argument given to the script is the server-list. Everything 296 # else will be read from that and searched in the computer-configuration. 297 if __name__ == '__main__': 298 if startCLI: 299 setLogLevel('info') 300 else: 301 # With this loglevel CLI(net) does not report correctly. 302 lg.setLogLevel( 'critical') 303 if len(sys.argv) < 2: 304 dbg(0, "please give list-name") 305 sys.exit(-1) 306 307 list_file = sys.argv[1] 308 global global_root, myNet, otherNets 309 global_root, myNet, otherNets = GetNetworks(list_file) 310 311 if myNet: 312 dbg( 2, "Cleaning up mininet and logfiles" ) 313 # rm_file(logfile) 314 rm_file(logdone) 315 call("mn -c > /dev/null 2>&1", shell=True) 316 317 threads = [] 318 if len(sys.argv) > 2: 319 if len(otherNets) > 0: 320 if len(otherNets) +1 >= internalNet: 321 dbg(0, "Cannot have more than %d servers!" % internalNet - 1) 322 sys.exit(-1) 323 324 dbg( 2, "Starting remotely on nets", otherNets) 325 for (server, mn, nbr) in otherNets: 326 dbg( 3, "Cleaning up", server ) 327 call("ssh -q %s 'mn -c; pkill -9 -f start.py' > /dev/null 2>&1" % server, shell=True) 328 dbg( 3, "Going to copy things %s to %s and run %s hosts in net %s" % \ 329 (list_file, server, nbr, mn) ) 330 shutil.rmtree('config', ignore_errors=True) 331 call("scp -q * %s %s:" % (list_file, server), shell=True) 332 threads.append(threading.Thread(target=call_other, args=[server, list_file])) 333 334 time.sleep(1) 335 for thr in threads: 336 dbg( 3, "Starting thread", thr ) 337 thr.start() 338 339 if myNet: 340 dbg( 1, "Starting mininet for %s" % myNet ) 341 t1 = threading.Thread(target=RunNet) 342 t1.start() 343 time.sleep(1) 344 t1.join() 345 346 for thr in threads: 347 thr.join() 348 349 dbg( 2, "Done with main in %s" % platform.node())