github.com/spg/deis@v1.7.3/controller/api/tests/test_scheduler.py (about) 1 """ 2 Unit tests for the Deis api app. 3 4 Run the tests with "./manage.py test api" 5 """ 6 7 from __future__ import unicode_literals 8 9 import json 10 11 from django.conf import settings 12 from django.contrib.auth.models import User 13 from django.test import TransactionTestCase 14 from rest_framework.authtoken.models import Token 15 16 from scheduler import chaos 17 18 19 class SchedulerTest(TransactionTestCase): 20 """Tests creation of containers on nodes""" 21 22 fixtures = ['tests.json'] 23 24 def setUp(self): 25 self.user = User.objects.get(username='autotest') 26 self.token = Token.objects.get(user=self.user).key 27 # start without any chaos 28 chaos.CREATE_ERROR_RATE = 0 29 chaos.DESTROY_ERROR_RATE = 0 30 chaos.START_ERROR_RATE = 0 31 chaos.STOP_ERROR_RATE = 0 32 # use chaos scheduler 33 settings.SCHEDULER_MODULE = 'scheduler.chaos' 34 # provide mock authentication used for run commands 35 settings.SSH_PRIVATE_KEY = '<some-ssh-private-key>' 36 37 def tearDown(self): 38 # reset for subsequent tests 39 settings.SCHEDULER_MODULE = 'scheduler.mock' 40 settings.SSH_PRIVATE_KEY = '' 41 42 def test_create_chaos(self): 43 url = '/v1/apps' 44 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 45 self.assertEqual(response.status_code, 201) 46 app_id = response.data['id'] 47 # post a new build 48 url = "/v1/apps/{app_id}/builds".format(**locals()) 49 body = {'image': 'autotest/example', 'sha': 'a'*40, 50 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 51 response = self.client.post(url, json.dumps(body), content_type='application/json', 52 HTTP_AUTHORIZATION='token {}'.format(self.token)) 53 self.assertEqual(response.status_code, 201) 54 url = "/v1/apps/{app_id}/containers".format(**locals()) 55 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 56 self.assertEqual(response.status_code, 200) 57 self.assertEqual(len(response.data['results']), 1) 58 # scale to zero for consistency 59 url = "/v1/apps/{app_id}/scale".format(**locals()) 60 body = {'web': 0} 61 response = self.client.post(url, json.dumps(body), content_type='application/json', 62 HTTP_AUTHORIZATION='token {}'.format(self.token)) 63 self.assertEqual(response.status_code, 204) 64 # let's get chaotic 65 chaos.CREATE_ERROR_RATE = 0.5 66 # scale up but expect a 503 67 url = "/v1/apps/{app_id}/scale".format(**locals()) 68 body = {'web': 20} 69 response = self.client.post(url, json.dumps(body), content_type='application/json', 70 HTTP_AUTHORIZATION='token {}'.format(self.token)) 71 self.assertEqual(response.status_code, 503) 72 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 73 self.assertEqual(response.get('content-type'), 'application/json') 74 # inspect broken containers 75 url = "/v1/apps/{app_id}/containers".format(**locals()) 76 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 77 self.assertEqual(response.status_code, 200) 78 self.assertEqual(len(response.data['results']), 20) 79 # make sure some failed 80 states = set([c['state'] for c in response.data['results']]) 81 self.assertEqual(states, set(['error', 'created'])) 82 83 def test_start_chaos(self): 84 url = '/v1/apps' 85 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 86 self.assertEqual(response.status_code, 201) 87 app_id = response.data['id'] 88 # post a new build 89 url = "/v1/apps/{app_id}/builds".format(**locals()) 90 body = {'image': 'autotest/example', 'sha': 'a'*40, 91 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 92 response = self.client.post(url, json.dumps(body), content_type='application/json', 93 HTTP_AUTHORIZATION='token {}'.format(self.token)) 94 self.assertEqual(response.status_code, 201) 95 url = "/v1/apps/{app_id}/containers".format(**locals()) 96 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 97 self.assertEqual(response.status_code, 200) 98 self.assertEqual(len(response.data['results']), 1) 99 # scale to zero for consistency 100 url = "/v1/apps/{app_id}/scale".format(**locals()) 101 body = {'web': 0} 102 response = self.client.post(url, json.dumps(body), content_type='application/json', 103 HTTP_AUTHORIZATION='token {}'.format(self.token)) 104 self.assertEqual(response.status_code, 204) 105 # let's get chaotic 106 chaos.START_ERROR_RATE = 0.5 107 # scale up, which will allow some crashed containers 108 url = "/v1/apps/{app_id}/scale".format(**locals()) 109 body = {'web': 20} 110 response = self.client.post(url, json.dumps(body), content_type='application/json', 111 HTTP_AUTHORIZATION='token {}'.format(self.token)) 112 self.assertEqual(response.status_code, 204) 113 # inspect broken containers 114 url = "/v1/apps/{app_id}/containers".format(**locals()) 115 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 116 self.assertEqual(response.status_code, 200) 117 self.assertEqual(len(response.data['results']), 20) 118 # make sure some failed 119 states = set([c['state'] for c in response.data['results']]) 120 self.assertEqual(states, set(['crashed', 'up'])) 121 122 def test_restart_chaos(self): 123 url = '/v1/apps' 124 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 125 self.assertEqual(response.status_code, 201) 126 app_id = response.data['id'] 127 # post a new build 128 url = "/v1/apps/{app_id}/builds".format(**locals()) 129 body = {'image': 'autotest/example', 'sha': 'a'*40, 130 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 131 response = self.client.post(url, json.dumps(body), content_type='application/json', 132 HTTP_AUTHORIZATION='token {}'.format(self.token)) 133 self.assertEqual(response.status_code, 201) 134 url = "/v1/apps/{app_id}/containers".format(**locals()) 135 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 136 self.assertEqual(response.status_code, 200) 137 self.assertEqual(len(response.data['results']), 1) 138 # scale up, which will allow some crashed containers 139 url = "/v1/apps/{app_id}/scale".format(**locals()) 140 body = {'web': 20, 'worker': 20} 141 response = self.client.post(url, json.dumps(body), content_type='application/json', 142 HTTP_AUTHORIZATION='token {}'.format(self.token)) 143 self.assertEqual(response.status_code, 204) 144 # let's get chaotic 145 chaos.STOP_ERROR_RATE = 0.5 146 chaos.START_ERROR_RATE = 0.5 147 # reboot the web processes 148 url = "/v1/apps/{app_id}/containers/web/restart".format(**locals()) 149 response = self.client.post(url, 150 content_type='application/json', 151 HTTP_AUTHORIZATION='token {}'.format(self.token)) 152 self.assertEqual(response.status_code, 200, response.data) 153 # inspect broken containers 154 url = "/v1/apps/{app_id}/containers".format(**locals()) 155 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 156 self.assertEqual(response.status_code, 200) 157 self.assertEqual(response.data['count'], 40) 158 # make sure some failed 159 states = set([c['state'] for c in response.data['results']]) 160 self.assertEqual(states, set(['crashed', 'up'])) 161 # make sure that we only rebooted the web processes 162 types = set([c['type'] for c in response.data['results'] if c['state'] == 'crashed']) 163 self.assertEqual(types, set(['web'])) 164 # start fresh 165 chaos.STOP_ERROR_RATE = 0.0 166 chaos.START_ERROR_RATE = 0.0 167 url = "/v1/apps/{app_id}/containers/web/restart".format(**locals()) 168 response = self.client.post(url, 169 content_type='application/json', 170 HTTP_AUTHORIZATION='token {}'.format(self.token)) 171 # let the carnage continue 172 chaos.STOP_ERROR_RATE = 0.5 173 chaos.START_ERROR_RATE = 0.5 174 # reboot ALL the containers! 175 url = "/v1/apps/{app_id}/containers/restart".format(**locals()) 176 response = self.client.post(url, 177 content_type='application/json', 178 HTTP_AUTHORIZATION='token {}'.format(self.token)) 179 self.assertEqual(response.status_code, 200) 180 # inspect broken containers 181 url = "/v1/apps/{app_id}/containers".format(**locals()) 182 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 183 self.assertEqual(response.status_code, 200) 184 self.assertEqual(len(response.data['results']), 40) 185 # make sure some failed 186 states = set([c['state'] for c in response.data['results']]) 187 self.assertEqual(states, set(['crashed', 'up'])) 188 types = set([c['type'] for c in response.data['results']]) 189 self.assertEqual(types, set(['web', 'worker'])) 190 191 def test_destroy_chaos(self): 192 url = '/v1/apps' 193 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 194 self.assertEqual(response.status_code, 201) 195 app_id = response.data['id'] 196 # post a new build 197 url = "/v1/apps/{app_id}/builds".format(**locals()) 198 body = {'image': 'autotest/example', 'sha': 'a'*40, 199 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 200 response = self.client.post(url, json.dumps(body), content_type='application/json', 201 HTTP_AUTHORIZATION='token {}'.format(self.token)) 202 self.assertEqual(response.status_code, 201) 203 url = "/v1/apps/{app_id}/containers".format(**locals()) 204 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 205 self.assertEqual(response.status_code, 200) 206 self.assertEqual(len(response.data['results']), 1) 207 # scale up 208 url = "/v1/apps/{app_id}/scale".format(**locals()) 209 body = {'web': 20} 210 response = self.client.post(url, json.dumps(body), content_type='application/json', 211 HTTP_AUTHORIZATION='token {}'.format(self.token)) 212 self.assertEqual(response.status_code, 204) 213 url = "/v1/apps/{app_id}/containers".format(**locals()) 214 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 215 self.assertEqual(response.status_code, 200) 216 self.assertEqual(len(response.data['results']), 20) 217 # let's get chaotic 218 chaos.DESTROY_ERROR_RATE = 0.5 219 # scale to zero but expect a 503 220 url = "/v1/apps/{app_id}/scale".format(**locals()) 221 body = {'web': 0} 222 response = self.client.post(url, json.dumps(body), content_type='application/json', 223 HTTP_AUTHORIZATION='token {}'.format(self.token)) 224 self.assertEqual(response.status_code, 503) 225 self.assertEqual(response.data, {'detail': 'aborting, failed to destroy some containers'}) 226 self.assertEqual(response.get('content-type'), 'application/json') 227 # inspect broken containers 228 url = "/v1/apps/{app_id}/containers".format(**locals()) 229 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 230 self.assertEqual(response.status_code, 200) 231 states = set([c['state'] for c in response.data['results']]) 232 self.assertEqual(states, set(['error'])) 233 # make sure we can cleanup after enough tries 234 containers = 20 235 for _ in xrange(100): 236 url = "/v1/apps/{app_id}/scale".format(**locals()) 237 body = {'web': 0} 238 response = self.client.post(url, json.dumps(body), content_type='application/json', 239 HTTP_AUTHORIZATION='token {}'.format(self.token)) 240 # break if we destroyed successfully 241 if response.status_code == 204: 242 break 243 self.assertEqual(response.status_code, 503) 244 self.assertEqual(response.data, {'detail': 'aborting, failed to ' 245 'destroy some containers'}) 246 self.assertEqual(response.get('content-type'), 'application/json') 247 # inspect broken containers 248 url = "/v1/apps/{app_id}/containers".format(**locals()) 249 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 250 self.assertEqual(response.status_code, 200) 251 containers = len(response.data['results']) 252 253 def test_build_chaos(self): 254 url = '/v1/apps' 255 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 256 self.assertEqual(response.status_code, 201) 257 app_id = response.data['id'] 258 # post a new build 259 url = "/v1/apps/{app_id}/builds".format(**locals()) 260 body = {'image': 'autotest/example', 'sha': 'a'*40, 261 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 262 response = self.client.post(url, json.dumps(body), content_type='application/json', 263 HTTP_AUTHORIZATION='token {}'.format(self.token)) 264 self.assertEqual(response.status_code, 201) 265 # inspect builds 266 url = "/v1/apps/{app_id}/builds".format(**locals()) 267 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 268 self.assertEqual(response.status_code, 200) 269 self.assertEqual(len(response.data['results']), 1) 270 # inspect releases 271 url = "/v1/apps/{app_id}/releases".format(**locals()) 272 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 273 self.assertEqual(response.status_code, 200) 274 self.assertEqual(len(response.data['results']), 2) 275 url = "/v1/apps/{app_id}/containers".format(**locals()) 276 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 277 self.assertEqual(response.status_code, 200) 278 self.assertEqual(len(response.data['results']), 1) 279 # scale up 280 url = "/v1/apps/{app_id}/scale".format(**locals()) 281 body = {'web': 20} 282 response = self.client.post(url, json.dumps(body), content_type='application/json', 283 HTTP_AUTHORIZATION='token {}'.format(self.token)) 284 self.assertEqual(response.status_code, 204) 285 # simulate failing to create containers 286 chaos.CREATE_ERROR_RATE = 0.5 287 chaos.START_ERROR_RATE = 0.5 288 # post a new build 289 url = "/v1/apps/{app_id}/builds".format(**locals()) 290 body = {'image': 'autotest/example', 'sha': 'b'*40, 291 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 292 response = self.client.post(url, json.dumps(body), content_type='application/json', 293 HTTP_AUTHORIZATION='token {}'.format(self.token)) 294 self.assertEqual(response.status_code, 503) 295 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 296 self.assertEqual(response.get('content-type'), 'application/json') 297 # inspect releases 298 url = "/v1/apps/{app_id}/releases".format(**locals()) 299 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 300 self.assertEqual(response.status_code, 200) 301 self.assertEqual(len(response.data['results']), 2) 302 # inspect containers 303 url = "/v1/apps/{app_id}/containers".format(**locals()) 304 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 305 self.assertEqual(response.status_code, 200) 306 self.assertEqual(len(response.data['results']), 20) 307 308 # make sure all old containers are still up 309 states = set([c['state'] for c in response.data['results']]) 310 self.assertEqual(states, set(['up'])) 311 312 def test_config_chaos(self): 313 url = '/v1/apps' 314 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 315 self.assertEqual(response.status_code, 201) 316 app_id = response.data['id'] 317 # post a new build 318 url = "/v1/apps/{app_id}/builds".format(**locals()) 319 body = {'image': 'autotest/example', 'sha': 'a'*40, 320 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 321 response = self.client.post(url, json.dumps(body), content_type='application/json', 322 HTTP_AUTHORIZATION='token {}'.format(self.token)) 323 self.assertEqual(response.status_code, 201) 324 # inspect releases 325 url = "/v1/apps/{app_id}/releases".format(**locals()) 326 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 327 self.assertEqual(response.status_code, 200) 328 self.assertEqual(len(response.data['results']), 2) 329 url = "/v1/apps/{app_id}/containers".format(**locals()) 330 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 331 self.assertEqual(response.status_code, 200) 332 self.assertEqual(len(response.data['results']), 1) 333 # scale up 334 url = "/v1/apps/{app_id}/scale".format(**locals()) 335 body = {'web': 20} 336 response = self.client.post(url, json.dumps(body), content_type='application/json', 337 HTTP_AUTHORIZATION='token {}'.format(self.token)) 338 self.assertEqual(response.status_code, 204) 339 # simulate failing to create or start containers 340 chaos.CREATE_ERROR_RATE = 0.5 341 chaos.START_ERROR_RATE = 0.5 342 # post a new config 343 url = "/v1/apps/{app_id}/config".format(**locals()) 344 body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})} 345 response = self.client.post(url, json.dumps(body), content_type='application/json', 346 HTTP_AUTHORIZATION='token {}'.format(self.token)) 347 self.assertEqual(response.status_code, 503) 348 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 349 self.assertEqual(response.get('content-type'), 'application/json') 350 # inspect releases 351 url = "/v1/apps/{app_id}/releases".format(**locals()) 352 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 353 self.assertEqual(response.status_code, 200) 354 self.assertEqual(len(response.data['results']), 2) 355 # inspect containers 356 url = "/v1/apps/{app_id}/containers".format(**locals()) 357 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 358 self.assertEqual(response.status_code, 200) 359 self.assertEqual(len(response.data['results']), 20) 360 # make sure all old containers are still up 361 states = set([c['state'] for c in response.data['results']]) 362 self.assertEqual(states, set(['up'])) 363 364 def test_run_chaos(self): 365 url = '/v1/apps' 366 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 367 self.assertEqual(response.status_code, 201) 368 app_id = response.data['id'] 369 # post a new build 370 url = "/v1/apps/{app_id}/builds".format(**locals()) 371 body = {'image': 'autotest/example', 'sha': 'a'*40, 372 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 373 response = self.client.post(url, json.dumps(body), content_type='application/json', 374 HTTP_AUTHORIZATION='token {}'.format(self.token)) 375 self.assertEqual(response.status_code, 201) 376 # inspect builds 377 url = "/v1/apps/{app_id}/builds".format(**locals()) 378 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 379 self.assertEqual(response.status_code, 200) 380 self.assertEqual(len(response.data['results']), 1) 381 # inspect releases 382 url = "/v1/apps/{app_id}/releases".format(**locals()) 383 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 384 self.assertEqual(response.status_code, 200) 385 self.assertEqual(len(response.data['results']), 2) 386 url = "/v1/apps/{app_id}/containers".format(**locals()) 387 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 388 self.assertEqual(response.status_code, 200) 389 self.assertEqual(len(response.data['results']), 1) 390 # block all create operations 391 chaos.CREATE_ERROR_RATE = 1 392 # make sure the run fails with a 503 393 url = '/v1/apps/{app_id}/run'.format(**locals()) 394 body = {'command': 'ls -al'} 395 response = self.client.post(url, json.dumps(body), content_type='application/json', 396 HTTP_AUTHORIZATION='token {}'.format(self.token)) 397 self.assertEqual(response.status_code, 503) 398 self.assertEqual(response.data, {'detail': 'exit code 1'}) 399 self.assertEqual(response.get('content-type'), 'application/json')