github.com/dustinrc/deis@v1.10.1-0.20150917223407-0894a5fb979e/controller/api/tests/test_scheduler.py (about) 1 """ 2 Unit tests for the Deis api app. 3 4 Run the tests with "./manage.py test api" 5 """ 6 7 from __future__ import unicode_literals 8 9 import json 10 11 from django.conf import settings 12 from django.contrib.auth.models import User 13 from django.test import TransactionTestCase 14 from rest_framework.authtoken.models import Token 15 16 from scheduler import chaos 17 18 19 class SchedulerTest(TransactionTestCase): 20 """Tests creation of containers on nodes""" 21 22 fixtures = ['tests.json'] 23 24 def setUp(self): 25 self.user = User.objects.get(username='autotest') 26 self.token = Token.objects.get(user=self.user).key 27 # start without any chaos 28 chaos.CREATE_ERROR_RATE = 0 29 chaos.DESTROY_ERROR_RATE = 0 30 chaos.START_ERROR_RATE = 0 31 chaos.STOP_ERROR_RATE = 0 32 # use chaos scheduler 33 settings.SCHEDULER_MODULE = 'scheduler.chaos' 34 # provide mock authentication used for run commands 35 settings.SSH_PRIVATE_KEY = '<some-ssh-private-key>' 36 37 def tearDown(self): 38 # reset for subsequent tests 39 settings.SCHEDULER_MODULE = 'scheduler.mock' 40 settings.SSH_PRIVATE_KEY = '' 41 42 def test_create_chaos(self): 43 url = '/v1/apps' 44 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 45 self.assertEqual(response.status_code, 201) 46 app_id = response.data['id'] 47 # post a new build 48 url = "/v1/apps/{app_id}/builds".format(**locals()) 49 body = {'image': 'autotest/example', 'sha': 'a'*40, 50 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 51 response = self.client.post(url, json.dumps(body), content_type='application/json', 52 HTTP_AUTHORIZATION='token {}'.format(self.token)) 53 self.assertEqual(response.status_code, 201) 54 url = "/v1/apps/{app_id}/containers".format(**locals()) 55 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 56 self.assertEqual(response.status_code, 200) 57 self.assertEqual(len(response.data['results']), 1) 58 # scale to zero for consistency 59 url = "/v1/apps/{app_id}/scale".format(**locals()) 60 body = {'web': 0} 61 response = self.client.post(url, json.dumps(body), content_type='application/json', 62 HTTP_AUTHORIZATION='token {}'.format(self.token)) 63 self.assertEqual(response.status_code, 204) 64 # let's get chaotic 65 chaos.CREATE_ERROR_RATE = 0.5 66 # scale up but expect a 503 67 url = "/v1/apps/{app_id}/scale".format(**locals()) 68 body = {'web': 20} 69 response = self.client.post(url, json.dumps(body), content_type='application/json', 70 HTTP_AUTHORIZATION='token {}'.format(self.token)) 71 self.assertEqual(response.status_code, 503) 72 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 73 self.assertEqual(response.get('content-type'), 'application/json') 74 # inspect broken containers 75 url = "/v1/apps/{app_id}/containers".format(**locals()) 76 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 77 self.assertEqual(response.status_code, 200) 78 self.assertEqual(len(response.data['results']), 0) 79 80 def test_start_chaos(self): 81 url = '/v1/apps' 82 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 83 self.assertEqual(response.status_code, 201) 84 app_id = response.data['id'] 85 # post a new build 86 url = "/v1/apps/{app_id}/builds".format(**locals()) 87 body = {'image': 'autotest/example', 'sha': 'a'*40, 88 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 89 response = self.client.post(url, json.dumps(body), content_type='application/json', 90 HTTP_AUTHORIZATION='token {}'.format(self.token)) 91 self.assertEqual(response.status_code, 201) 92 url = "/v1/apps/{app_id}/containers".format(**locals()) 93 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 94 self.assertEqual(response.status_code, 200) 95 self.assertEqual(len(response.data['results']), 1) 96 # scale to zero for consistency 97 url = "/v1/apps/{app_id}/scale".format(**locals()) 98 body = {'web': 0} 99 response = self.client.post(url, json.dumps(body), content_type='application/json', 100 HTTP_AUTHORIZATION='token {}'.format(self.token)) 101 self.assertEqual(response.status_code, 204) 102 # let's get chaotic 103 chaos.START_ERROR_RATE = 0.5 104 # scale up, which will allow some crashed containers 105 url = "/v1/apps/{app_id}/scale".format(**locals()) 106 body = {'web': 20} 107 response = self.client.post(url, json.dumps(body), content_type='application/json', 108 HTTP_AUTHORIZATION='token {}'.format(self.token)) 109 self.assertEqual(response.status_code, 204) 110 # inspect broken containers 111 url = "/v1/apps/{app_id}/containers".format(**locals()) 112 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 113 self.assertEqual(response.status_code, 200) 114 self.assertEqual(len(response.data['results']), 20) 115 # make sure some failed 116 states = set([c['state'] for c in response.data['results']]) 117 self.assertEqual(states, set(['crashed', 'up'])) 118 119 def test_restart_chaos(self): 120 url = '/v1/apps' 121 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 122 self.assertEqual(response.status_code, 201) 123 app_id = response.data['id'] 124 # post a new build 125 url = "/v1/apps/{app_id}/builds".format(**locals()) 126 body = {'image': 'autotest/example', 'sha': 'a'*40, 127 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 128 response = self.client.post(url, json.dumps(body), content_type='application/json', 129 HTTP_AUTHORIZATION='token {}'.format(self.token)) 130 self.assertEqual(response.status_code, 201) 131 url = "/v1/apps/{app_id}/containers".format(**locals()) 132 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 133 self.assertEqual(response.status_code, 200) 134 self.assertEqual(len(response.data['results']), 1) 135 # scale up, which will allow some crashed containers 136 url = "/v1/apps/{app_id}/scale".format(**locals()) 137 body = {'web': 20, 'worker': 20} 138 response = self.client.post(url, json.dumps(body), content_type='application/json', 139 HTTP_AUTHORIZATION='token {}'.format(self.token)) 140 self.assertEqual(response.status_code, 204) 141 # let's get chaotic 142 chaos.STOP_ERROR_RATE = 0.5 143 chaos.START_ERROR_RATE = 0.5 144 # reboot the web processes 145 url = "/v1/apps/{app_id}/containers/web/restart".format(**locals()) 146 response = self.client.post(url, 147 content_type='application/json', 148 HTTP_AUTHORIZATION='token {}'.format(self.token)) 149 self.assertEqual(response.status_code, 200, response.data) 150 # inspect broken containers 151 url = "/v1/apps/{app_id}/containers".format(**locals()) 152 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 153 self.assertEqual(response.status_code, 200) 154 self.assertEqual(response.data['count'], 40) 155 # make sure some failed 156 states = set([c['state'] for c in response.data['results']]) 157 self.assertEqual(states, set(['crashed', 'up'])) 158 # make sure that we only rebooted the web processes 159 types = set([c['type'] for c in response.data['results'] if c['state'] == 'crashed']) 160 self.assertEqual(types, set(['web'])) 161 # start fresh 162 chaos.STOP_ERROR_RATE = 0.0 163 chaos.START_ERROR_RATE = 0.0 164 url = "/v1/apps/{app_id}/containers/web/restart".format(**locals()) 165 response = self.client.post(url, 166 content_type='application/json', 167 HTTP_AUTHORIZATION='token {}'.format(self.token)) 168 # let the carnage continue 169 chaos.STOP_ERROR_RATE = 0.5 170 chaos.START_ERROR_RATE = 0.5 171 # reboot ALL the containers! 172 url = "/v1/apps/{app_id}/containers/restart".format(**locals()) 173 response = self.client.post(url, 174 content_type='application/json', 175 HTTP_AUTHORIZATION='token {}'.format(self.token)) 176 self.assertEqual(response.status_code, 200) 177 # inspect broken containers 178 url = "/v1/apps/{app_id}/containers".format(**locals()) 179 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 180 self.assertEqual(response.status_code, 200) 181 self.assertEqual(len(response.data['results']), 40) 182 # make sure some failed 183 states = set([c['state'] for c in response.data['results']]) 184 self.assertEqual(states, set(['crashed', 'up'])) 185 types = set([c['type'] for c in response.data['results']]) 186 self.assertEqual(types, set(['web', 'worker'])) 187 188 def test_destroy_chaos(self): 189 url = '/v1/apps' 190 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 191 self.assertEqual(response.status_code, 201) 192 app_id = response.data['id'] 193 # post a new build 194 url = "/v1/apps/{app_id}/builds".format(**locals()) 195 body = {'image': 'autotest/example', 'sha': 'a'*40, 196 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 197 response = self.client.post(url, json.dumps(body), content_type='application/json', 198 HTTP_AUTHORIZATION='token {}'.format(self.token)) 199 self.assertEqual(response.status_code, 201) 200 url = "/v1/apps/{app_id}/containers".format(**locals()) 201 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 202 self.assertEqual(response.status_code, 200) 203 self.assertEqual(len(response.data['results']), 1) 204 # scale up 205 url = "/v1/apps/{app_id}/scale".format(**locals()) 206 body = {'web': 20} 207 response = self.client.post(url, json.dumps(body), content_type='application/json', 208 HTTP_AUTHORIZATION='token {}'.format(self.token)) 209 self.assertEqual(response.status_code, 204) 210 url = "/v1/apps/{app_id}/containers".format(**locals()) 211 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 212 self.assertEqual(response.status_code, 200) 213 self.assertEqual(len(response.data['results']), 20) 214 # let's get chaotic 215 chaos.DESTROY_ERROR_RATE = 0.5 216 # scale to zero but expect a 503 217 url = "/v1/apps/{app_id}/scale".format(**locals()) 218 body = {'web': 0} 219 response = self.client.post(url, json.dumps(body), content_type='application/json', 220 HTTP_AUTHORIZATION='token {}'.format(self.token)) 221 self.assertEqual(response.status_code, 503) 222 self.assertEqual(response.data, {'detail': 'aborting, failed to destroy some containers'}) 223 self.assertEqual(response.get('content-type'), 'application/json') 224 # inspect broken containers 225 url = "/v1/apps/{app_id}/containers".format(**locals()) 226 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 227 self.assertEqual(response.status_code, 200) 228 states = set([c['state'] for c in response.data['results']]) 229 self.assertEqual(states, set(['error'])) 230 # make sure we can cleanup after enough tries 231 containers = 20 232 for _ in xrange(100): 233 url = "/v1/apps/{app_id}/scale".format(**locals()) 234 body = {'web': 0} 235 response = self.client.post(url, json.dumps(body), content_type='application/json', 236 HTTP_AUTHORIZATION='token {}'.format(self.token)) 237 # break if we destroyed successfully 238 if response.status_code == 204: 239 break 240 self.assertEqual(response.status_code, 503) 241 self.assertEqual(response.data, {'detail': 'aborting, failed to ' 242 'destroy some containers'}) 243 self.assertEqual(response.get('content-type'), 'application/json') 244 # inspect broken containers 245 url = "/v1/apps/{app_id}/containers".format(**locals()) 246 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 247 self.assertEqual(response.status_code, 200) 248 containers = len(response.data['results']) 249 250 def test_build_chaos(self): 251 url = '/v1/apps' 252 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 253 self.assertEqual(response.status_code, 201) 254 app_id = response.data['id'] 255 # post a new build 256 url = "/v1/apps/{app_id}/builds".format(**locals()) 257 body = {'image': 'autotest/example', 'sha': 'a'*40, 258 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 259 response = self.client.post(url, json.dumps(body), content_type='application/json', 260 HTTP_AUTHORIZATION='token {}'.format(self.token)) 261 self.assertEqual(response.status_code, 201) 262 # inspect builds 263 url = "/v1/apps/{app_id}/builds".format(**locals()) 264 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 265 self.assertEqual(response.status_code, 200) 266 self.assertEqual(len(response.data['results']), 1) 267 # inspect releases 268 url = "/v1/apps/{app_id}/releases".format(**locals()) 269 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 270 self.assertEqual(response.status_code, 200) 271 self.assertEqual(len(response.data['results']), 2) 272 url = "/v1/apps/{app_id}/containers".format(**locals()) 273 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 274 self.assertEqual(response.status_code, 200) 275 self.assertEqual(len(response.data['results']), 1) 276 # scale up 277 url = "/v1/apps/{app_id}/scale".format(**locals()) 278 body = {'web': 20} 279 response = self.client.post(url, json.dumps(body), content_type='application/json', 280 HTTP_AUTHORIZATION='token {}'.format(self.token)) 281 self.assertEqual(response.status_code, 204) 282 # simulate failing to create containers 283 chaos.CREATE_ERROR_RATE = 0.5 284 chaos.START_ERROR_RATE = 0.5 285 # post a new build 286 url = "/v1/apps/{app_id}/builds".format(**locals()) 287 body = {'image': 'autotest/example', 'sha': 'b'*40, 288 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 289 response = self.client.post(url, json.dumps(body), content_type='application/json', 290 HTTP_AUTHORIZATION='token {}'.format(self.token)) 291 self.assertEqual(response.status_code, 503) 292 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 293 self.assertEqual(response.get('content-type'), 'application/json') 294 # inspect releases 295 url = "/v1/apps/{app_id}/releases".format(**locals()) 296 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 297 self.assertEqual(response.status_code, 200) 298 self.assertEqual(len(response.data['results']), 2) 299 # inspect containers 300 url = "/v1/apps/{app_id}/containers".format(**locals()) 301 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 302 self.assertEqual(response.status_code, 200) 303 self.assertEqual(len(response.data['results']), 20) 304 # make sure all old containers are still up 305 states = set([c['state'] for c in response.data['results']]) 306 self.assertEqual(states, set(['up'])) 307 308 def test_config_chaos(self): 309 url = '/v1/apps' 310 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 311 self.assertEqual(response.status_code, 201) 312 app_id = response.data['id'] 313 # post a new build 314 url = "/v1/apps/{app_id}/builds".format(**locals()) 315 body = {'image': 'autotest/example', 'sha': 'a'*40, 316 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 317 response = self.client.post(url, json.dumps(body), content_type='application/json', 318 HTTP_AUTHORIZATION='token {}'.format(self.token)) 319 self.assertEqual(response.status_code, 201) 320 # inspect releases 321 url = "/v1/apps/{app_id}/releases".format(**locals()) 322 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 323 self.assertEqual(response.status_code, 200) 324 self.assertEqual(len(response.data['results']), 2) 325 url = "/v1/apps/{app_id}/containers".format(**locals()) 326 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 327 self.assertEqual(response.status_code, 200) 328 self.assertEqual(len(response.data['results']), 1) 329 # scale up 330 url = "/v1/apps/{app_id}/scale".format(**locals()) 331 body = {'web': 20} 332 response = self.client.post(url, json.dumps(body), content_type='application/json', 333 HTTP_AUTHORIZATION='token {}'.format(self.token)) 334 self.assertEqual(response.status_code, 204) 335 # simulate failing to create or start containers 336 chaos.CREATE_ERROR_RATE = 0.5 337 chaos.START_ERROR_RATE = 0.5 338 # post a new config 339 url = "/v1/apps/{app_id}/config".format(**locals()) 340 body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})} 341 response = self.client.post(url, json.dumps(body), content_type='application/json', 342 HTTP_AUTHORIZATION='token {}'.format(self.token)) 343 self.assertEqual(response.status_code, 503) 344 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 345 self.assertEqual(response.get('content-type'), 'application/json') 346 # inspect releases 347 url = "/v1/apps/{app_id}/releases".format(**locals()) 348 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 349 self.assertEqual(response.status_code, 200) 350 self.assertEqual(len(response.data['results']), 2) 351 # inspect containers 352 url = "/v1/apps/{app_id}/containers".format(**locals()) 353 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 354 self.assertEqual(response.status_code, 200) 355 self.assertEqual(len(response.data['results']), 20) 356 # make sure all old containers are still up 357 states = set([c['state'] for c in response.data['results']]) 358 self.assertEqual(states, set(['up'])) 359 360 def test_run_chaos(self): 361 url = '/v1/apps' 362 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 363 self.assertEqual(response.status_code, 201) 364 app_id = response.data['id'] 365 # post a new build 366 url = "/v1/apps/{app_id}/builds".format(**locals()) 367 body = {'image': 'autotest/example', 'sha': 'a'*40, 368 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 369 response = self.client.post(url, json.dumps(body), content_type='application/json', 370 HTTP_AUTHORIZATION='token {}'.format(self.token)) 371 self.assertEqual(response.status_code, 201) 372 # inspect builds 373 url = "/v1/apps/{app_id}/builds".format(**locals()) 374 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 375 self.assertEqual(response.status_code, 200) 376 self.assertEqual(len(response.data['results']), 1) 377 # inspect releases 378 url = "/v1/apps/{app_id}/releases".format(**locals()) 379 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 380 self.assertEqual(response.status_code, 200) 381 self.assertEqual(len(response.data['results']), 2) 382 url = "/v1/apps/{app_id}/containers".format(**locals()) 383 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 384 self.assertEqual(response.status_code, 200) 385 self.assertEqual(len(response.data['results']), 1) 386 # block all create operations 387 chaos.CREATE_ERROR_RATE = 1 388 # make sure the run fails with a 503 389 url = '/v1/apps/{app_id}/run'.format(**locals()) 390 body = {'command': 'ls -al'} 391 response = self.client.post(url, json.dumps(body), content_type='application/json', 392 HTTP_AUTHORIZATION='token {}'.format(self.token)) 393 self.assertEqual(response.status_code, 503) 394 self.assertEqual(response.data, {'detail': 'exit code 1'}) 395 self.assertEqual(response.get('content-type'), 'application/json')