github.com/jiasir/deis@v1.12.2/controller/api/tests/test_scheduler.py (about) 1 """ 2 Unit tests for the Deis api app. 3 4 Run the tests with "./manage.py test api" 5 """ 6 7 from __future__ import unicode_literals 8 9 import json 10 11 from django.conf import settings 12 from django.contrib.auth.models import User 13 from django.test import TransactionTestCase 14 import mock 15 from rest_framework.authtoken.models import Token 16 17 from scheduler import chaos 18 19 20 @mock.patch('api.models.publish_release', lambda *args: None) 21 class SchedulerTest(TransactionTestCase): 22 """Tests creation of containers on nodes""" 23 24 fixtures = ['tests.json'] 25 26 def setUp(self): 27 self.user = User.objects.get(username='autotest') 28 self.token = Token.objects.get(user=self.user).key 29 # start without any chaos 30 chaos.CREATE_ERROR_RATE = 0 31 chaos.DESTROY_ERROR_RATE = 0 32 chaos.START_ERROR_RATE = 0 33 chaos.STOP_ERROR_RATE = 0 34 # use chaos scheduler 35 settings.SCHEDULER_MODULE = 'scheduler.chaos' 36 # provide mock authentication used for run commands 37 settings.SSH_PRIVATE_KEY = '<some-ssh-private-key>' 38 39 def tearDown(self): 40 # reset for subsequent tests 41 settings.SCHEDULER_MODULE = 'scheduler.mock' 42 settings.SSH_PRIVATE_KEY = '' 43 44 def test_create_chaos(self): 45 url = '/v1/apps' 46 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 47 self.assertEqual(response.status_code, 201) 48 app_id = response.data['id'] 49 # post a new build 50 url = "/v1/apps/{app_id}/builds".format(**locals()) 51 body = {'image': 'autotest/example', 'sha': 'a'*40, 52 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 53 response = self.client.post(url, json.dumps(body), content_type='application/json', 54 HTTP_AUTHORIZATION='token {}'.format(self.token)) 55 self.assertEqual(response.status_code, 201) 56 url = "/v1/apps/{app_id}/containers".format(**locals()) 57 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 58 self.assertEqual(response.status_code, 200) 59 self.assertEqual(len(response.data['results']), 1) 60 # scale to zero for consistency 61 url = "/v1/apps/{app_id}/scale".format(**locals()) 62 body = {'web': 0} 63 response = self.client.post(url, json.dumps(body), content_type='application/json', 64 HTTP_AUTHORIZATION='token {}'.format(self.token)) 65 self.assertEqual(response.status_code, 204) 66 # let's get chaotic 67 chaos.CREATE_ERROR_RATE = 0.5 68 # scale up but expect a 503 69 url = "/v1/apps/{app_id}/scale".format(**locals()) 70 body = {'web': 20} 71 response = self.client.post(url, json.dumps(body), content_type='application/json', 72 HTTP_AUTHORIZATION='token {}'.format(self.token)) 73 self.assertEqual(response.status_code, 503) 74 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 75 self.assertEqual(response.get('content-type'), 'application/json') 76 # inspect broken containers 77 url = "/v1/apps/{app_id}/containers".format(**locals()) 78 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 79 self.assertEqual(response.status_code, 200) 80 self.assertEqual(len(response.data['results']), 0) 81 82 def test_start_chaos(self): 83 url = '/v1/apps' 84 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 85 self.assertEqual(response.status_code, 201) 86 app_id = response.data['id'] 87 # post a new build 88 url = "/v1/apps/{app_id}/builds".format(**locals()) 89 body = {'image': 'autotest/example', 'sha': 'a'*40, 90 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 91 response = self.client.post(url, json.dumps(body), content_type='application/json', 92 HTTP_AUTHORIZATION='token {}'.format(self.token)) 93 self.assertEqual(response.status_code, 201) 94 url = "/v1/apps/{app_id}/containers".format(**locals()) 95 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 96 self.assertEqual(response.status_code, 200) 97 self.assertEqual(len(response.data['results']), 1) 98 # scale to zero for consistency 99 url = "/v1/apps/{app_id}/scale".format(**locals()) 100 body = {'web': 0} 101 response = self.client.post(url, json.dumps(body), content_type='application/json', 102 HTTP_AUTHORIZATION='token {}'.format(self.token)) 103 self.assertEqual(response.status_code, 204) 104 # let's get chaotic 105 chaos.START_ERROR_RATE = 0.5 106 # scale up, which will allow some crashed containers 107 url = "/v1/apps/{app_id}/scale".format(**locals()) 108 body = {'web': 20} 109 response = self.client.post(url, json.dumps(body), content_type='application/json', 110 HTTP_AUTHORIZATION='token {}'.format(self.token)) 111 self.assertEqual(response.status_code, 204) 112 # inspect broken containers 113 url = "/v1/apps/{app_id}/containers".format(**locals()) 114 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 115 self.assertEqual(response.status_code, 200) 116 self.assertEqual(len(response.data['results']), 20) 117 # make sure some failed 118 states = set([c['state'] for c in response.data['results']]) 119 self.assertEqual(states, set(['crashed', 'up'])) 120 121 def test_restart_chaos(self): 122 url = '/v1/apps' 123 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 124 self.assertEqual(response.status_code, 201) 125 app_id = response.data['id'] 126 # post a new build 127 url = "/v1/apps/{app_id}/builds".format(**locals()) 128 body = {'image': 'autotest/example', 'sha': 'a'*40, 129 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 130 response = self.client.post(url, json.dumps(body), content_type='application/json', 131 HTTP_AUTHORIZATION='token {}'.format(self.token)) 132 self.assertEqual(response.status_code, 201) 133 url = "/v1/apps/{app_id}/containers".format(**locals()) 134 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 135 self.assertEqual(response.status_code, 200) 136 self.assertEqual(len(response.data['results']), 1) 137 # scale up, which will allow some crashed containers 138 url = "/v1/apps/{app_id}/scale".format(**locals()) 139 body = {'web': 20, 'worker': 20} 140 response = self.client.post(url, json.dumps(body), content_type='application/json', 141 HTTP_AUTHORIZATION='token {}'.format(self.token)) 142 self.assertEqual(response.status_code, 204) 143 # let's get chaotic 144 chaos.STOP_ERROR_RATE = 0.5 145 chaos.START_ERROR_RATE = 0.5 146 # reboot the web processes 147 url = "/v1/apps/{app_id}/containers/web/restart".format(**locals()) 148 response = self.client.post(url, 149 content_type='application/json', 150 HTTP_AUTHORIZATION='token {}'.format(self.token)) 151 self.assertEqual(response.status_code, 200, response.data) 152 # inspect broken containers 153 url = "/v1/apps/{app_id}/containers".format(**locals()) 154 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 155 self.assertEqual(response.status_code, 200) 156 self.assertEqual(response.data['count'], 40) 157 # make sure some failed 158 states = set([c['state'] for c in response.data['results']]) 159 self.assertEqual(states, set(['crashed', 'up'])) 160 # make sure that we only rebooted the web processes 161 types = set([c['type'] for c in response.data['results'] if c['state'] == 'crashed']) 162 self.assertEqual(types, set(['web'])) 163 # start fresh 164 chaos.STOP_ERROR_RATE = 0.0 165 chaos.START_ERROR_RATE = 0.0 166 url = "/v1/apps/{app_id}/containers/web/restart".format(**locals()) 167 response = self.client.post(url, 168 content_type='application/json', 169 HTTP_AUTHORIZATION='token {}'.format(self.token)) 170 # let the carnage continue 171 chaos.STOP_ERROR_RATE = 0.5 172 chaos.START_ERROR_RATE = 0.5 173 # reboot ALL the containers! 174 url = "/v1/apps/{app_id}/containers/restart".format(**locals()) 175 response = self.client.post(url, 176 content_type='application/json', 177 HTTP_AUTHORIZATION='token {}'.format(self.token)) 178 self.assertEqual(response.status_code, 200) 179 # inspect broken containers 180 url = "/v1/apps/{app_id}/containers".format(**locals()) 181 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 182 self.assertEqual(response.status_code, 200) 183 self.assertEqual(len(response.data['results']), 40) 184 # make sure some failed 185 states = set([c['state'] for c in response.data['results']]) 186 self.assertEqual(states, set(['crashed', 'up'])) 187 types = set([c['type'] for c in response.data['results']]) 188 self.assertEqual(types, set(['web', 'worker'])) 189 190 def test_destroy_chaos(self): 191 url = '/v1/apps' 192 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 193 self.assertEqual(response.status_code, 201) 194 app_id = response.data['id'] 195 # post a new build 196 url = "/v1/apps/{app_id}/builds".format(**locals()) 197 body = {'image': 'autotest/example', 'sha': 'a'*40, 198 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 199 response = self.client.post(url, json.dumps(body), content_type='application/json', 200 HTTP_AUTHORIZATION='token {}'.format(self.token)) 201 self.assertEqual(response.status_code, 201) 202 url = "/v1/apps/{app_id}/containers".format(**locals()) 203 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 204 self.assertEqual(response.status_code, 200) 205 self.assertEqual(len(response.data['results']), 1) 206 # scale up 207 url = "/v1/apps/{app_id}/scale".format(**locals()) 208 body = {'web': 20} 209 response = self.client.post(url, json.dumps(body), content_type='application/json', 210 HTTP_AUTHORIZATION='token {}'.format(self.token)) 211 self.assertEqual(response.status_code, 204) 212 url = "/v1/apps/{app_id}/containers".format(**locals()) 213 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 214 self.assertEqual(response.status_code, 200) 215 self.assertEqual(len(response.data['results']), 20) 216 # let's get chaotic 217 chaos.DESTROY_ERROR_RATE = 0.5 218 # scale to zero but expect a 503 219 url = "/v1/apps/{app_id}/scale".format(**locals()) 220 body = {'web': 0} 221 response = self.client.post(url, json.dumps(body), content_type='application/json', 222 HTTP_AUTHORIZATION='token {}'.format(self.token)) 223 self.assertEqual(response.status_code, 503) 224 self.assertEqual(response.data, {'detail': 'aborting, failed to destroy some containers'}) 225 self.assertEqual(response.get('content-type'), 'application/json') 226 # inspect broken containers 227 url = "/v1/apps/{app_id}/containers".format(**locals()) 228 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 229 self.assertEqual(response.status_code, 200) 230 states = set([c['state'] for c in response.data['results']]) 231 self.assertEqual(states, set(['error'])) 232 # make sure we can cleanup after enough tries 233 containers = 20 234 for _ in xrange(100): 235 url = "/v1/apps/{app_id}/scale".format(**locals()) 236 body = {'web': 0} 237 response = self.client.post(url, json.dumps(body), content_type='application/json', 238 HTTP_AUTHORIZATION='token {}'.format(self.token)) 239 # break if we destroyed successfully 240 if response.status_code == 204: 241 break 242 self.assertEqual(response.status_code, 503) 243 self.assertEqual(response.data, {'detail': 'aborting, failed to ' 244 'destroy some containers'}) 245 self.assertEqual(response.get('content-type'), 'application/json') 246 # inspect broken containers 247 url = "/v1/apps/{app_id}/containers".format(**locals()) 248 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 249 self.assertEqual(response.status_code, 200) 250 containers = len(response.data['results']) 251 252 def test_build_chaos(self): 253 url = '/v1/apps' 254 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 255 self.assertEqual(response.status_code, 201) 256 app_id = response.data['id'] 257 # post a new build 258 url = "/v1/apps/{app_id}/builds".format(**locals()) 259 body = {'image': 'autotest/example', 'sha': 'a'*40, 260 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 261 response = self.client.post(url, json.dumps(body), content_type='application/json', 262 HTTP_AUTHORIZATION='token {}'.format(self.token)) 263 self.assertEqual(response.status_code, 201) 264 # inspect builds 265 url = "/v1/apps/{app_id}/builds".format(**locals()) 266 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 267 self.assertEqual(response.status_code, 200) 268 self.assertEqual(len(response.data['results']), 1) 269 # inspect releases 270 url = "/v1/apps/{app_id}/releases".format(**locals()) 271 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 272 self.assertEqual(response.status_code, 200) 273 self.assertEqual(len(response.data['results']), 2) 274 url = "/v1/apps/{app_id}/containers".format(**locals()) 275 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 276 self.assertEqual(response.status_code, 200) 277 self.assertEqual(len(response.data['results']), 1) 278 # scale up 279 url = "/v1/apps/{app_id}/scale".format(**locals()) 280 body = {'web': 20} 281 response = self.client.post(url, json.dumps(body), content_type='application/json', 282 HTTP_AUTHORIZATION='token {}'.format(self.token)) 283 self.assertEqual(response.status_code, 204) 284 # simulate failing to create containers 285 chaos.CREATE_ERROR_RATE = 0.5 286 chaos.START_ERROR_RATE = 0.5 287 # post a new build 288 url = "/v1/apps/{app_id}/builds".format(**locals()) 289 body = {'image': 'autotest/example', 'sha': 'b'*40, 290 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 291 response = self.client.post(url, json.dumps(body), content_type='application/json', 292 HTTP_AUTHORIZATION='token {}'.format(self.token)) 293 self.assertEqual(response.status_code, 503) 294 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 295 self.assertEqual(response.get('content-type'), 'application/json') 296 # inspect releases 297 url = "/v1/apps/{app_id}/releases".format(**locals()) 298 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 299 self.assertEqual(response.status_code, 200) 300 self.assertEqual(len(response.data['results']), 2) 301 # inspect containers 302 url = "/v1/apps/{app_id}/containers".format(**locals()) 303 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 304 self.assertEqual(response.status_code, 200) 305 self.assertEqual(len(response.data['results']), 20) 306 # make sure all old containers are still up 307 states = set([c['state'] for c in response.data['results']]) 308 self.assertEqual(states, set(['up'])) 309 310 def test_config_chaos(self): 311 url = '/v1/apps' 312 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 313 self.assertEqual(response.status_code, 201) 314 app_id = response.data['id'] 315 # post a new build 316 url = "/v1/apps/{app_id}/builds".format(**locals()) 317 body = {'image': 'autotest/example', 'sha': 'a'*40, 318 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 319 response = self.client.post(url, json.dumps(body), content_type='application/json', 320 HTTP_AUTHORIZATION='token {}'.format(self.token)) 321 self.assertEqual(response.status_code, 201) 322 # inspect releases 323 url = "/v1/apps/{app_id}/releases".format(**locals()) 324 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 325 self.assertEqual(response.status_code, 200) 326 self.assertEqual(len(response.data['results']), 2) 327 url = "/v1/apps/{app_id}/containers".format(**locals()) 328 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 329 self.assertEqual(response.status_code, 200) 330 self.assertEqual(len(response.data['results']), 1) 331 # scale up 332 url = "/v1/apps/{app_id}/scale".format(**locals()) 333 body = {'web': 20} 334 response = self.client.post(url, json.dumps(body), content_type='application/json', 335 HTTP_AUTHORIZATION='token {}'.format(self.token)) 336 self.assertEqual(response.status_code, 204) 337 # simulate failing to create or start containers 338 chaos.CREATE_ERROR_RATE = 0.5 339 chaos.START_ERROR_RATE = 0.5 340 # post a new config 341 url = "/v1/apps/{app_id}/config".format(**locals()) 342 body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})} 343 response = self.client.post(url, json.dumps(body), content_type='application/json', 344 HTTP_AUTHORIZATION='token {}'.format(self.token)) 345 self.assertEqual(response.status_code, 503) 346 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 347 self.assertEqual(response.get('content-type'), 'application/json') 348 # inspect releases 349 url = "/v1/apps/{app_id}/releases".format(**locals()) 350 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 351 self.assertEqual(response.status_code, 200) 352 self.assertEqual(len(response.data['results']), 2) 353 # inspect containers 354 url = "/v1/apps/{app_id}/containers".format(**locals()) 355 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 356 self.assertEqual(response.status_code, 200) 357 self.assertEqual(len(response.data['results']), 20) 358 # make sure all old containers are still up 359 states = set([c['state'] for c in response.data['results']]) 360 self.assertEqual(states, set(['up'])) 361 362 def test_run_chaos(self): 363 url = '/v1/apps' 364 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 365 self.assertEqual(response.status_code, 201) 366 app_id = response.data['id'] 367 # post a new build 368 url = "/v1/apps/{app_id}/builds".format(**locals()) 369 body = {'image': 'autotest/example', 'sha': 'a'*40, 370 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 371 response = self.client.post(url, json.dumps(body), content_type='application/json', 372 HTTP_AUTHORIZATION='token {}'.format(self.token)) 373 self.assertEqual(response.status_code, 201) 374 # inspect builds 375 url = "/v1/apps/{app_id}/builds".format(**locals()) 376 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 377 self.assertEqual(response.status_code, 200) 378 self.assertEqual(len(response.data['results']), 1) 379 # inspect releases 380 url = "/v1/apps/{app_id}/releases".format(**locals()) 381 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 382 self.assertEqual(response.status_code, 200) 383 self.assertEqual(len(response.data['results']), 2) 384 url = "/v1/apps/{app_id}/containers".format(**locals()) 385 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 386 self.assertEqual(response.status_code, 200) 387 self.assertEqual(len(response.data['results']), 1) 388 # block all create operations 389 chaos.CREATE_ERROR_RATE = 1 390 # make sure the run fails with a 503 391 url = '/v1/apps/{app_id}/run'.format(**locals()) 392 body = {'command': 'ls -al'} 393 response = self.client.post(url, json.dumps(body), content_type='application/json', 394 HTTP_AUTHORIZATION='token {}'.format(self.token)) 395 self.assertEqual(response.status_code, 503) 396 self.assertEqual(response.data, {'detail': 'exit code 1'}) 397 self.assertEqual(response.get('content-type'), 'application/json')