github.com/amrnt/deis@v1.3.1/controller/api/tests/test_scheduler.py (about) 1 """ 2 Unit tests for the Deis api app. 3 4 Run the tests with "./manage.py test api" 5 """ 6 7 from __future__ import unicode_literals 8 9 import json 10 11 from django.conf import settings 12 from django.contrib.auth.models import User 13 from django.test import TransactionTestCase 14 from rest_framework.authtoken.models import Token 15 16 from scheduler import chaos 17 18 19 class SchedulerTest(TransactionTestCase): 20 """Tests creation of containers on nodes""" 21 22 fixtures = ['tests.json'] 23 24 def setUp(self): 25 self.user = User.objects.get(username='autotest') 26 self.token = Token.objects.get(user=self.user).key 27 # start without any chaos 28 chaos.CREATE_ERROR_RATE = 0 29 chaos.DESTROY_ERROR_RATE = 0 30 chaos.START_ERROR_RATE = 0 31 chaos.STOP_ERROR_RATE = 0 32 # use chaos scheduler 33 settings.SCHEDULER_MODULE = 'chaos' 34 # provide mock authentication used for run commands 35 settings.SSH_PRIVATE_KEY = '<some-ssh-private-key>' 36 37 def tearDown(self): 38 # reset for subsequent tests 39 settings.SCHEDULER_MODULE = 'mock' 40 settings.SSH_PRIVATE_KEY = '' 41 42 def test_create_chaos(self): 43 url = '/v1/apps' 44 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 45 self.assertEqual(response.status_code, 201) 46 app_id = response.data['id'] 47 # post a new build 48 url = "/v1/apps/{app_id}/builds".format(**locals()) 49 body = {'image': 'autotest/example', 'sha': 'a'*40, 50 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 51 response = self.client.post(url, json.dumps(body), content_type='application/json', 52 HTTP_AUTHORIZATION='token {}'.format(self.token)) 53 self.assertEqual(response.status_code, 201) 54 url = "/v1/apps/{app_id}/containers".format(**locals()) 55 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 56 self.assertEqual(response.status_code, 200) 57 self.assertEqual(len(response.data['results']), 1) 58 # scale to zero for consistency 59 url = "/v1/apps/{app_id}/scale".format(**locals()) 60 body = {'web': 0} 61 response = self.client.post(url, json.dumps(body), content_type='application/json', 62 HTTP_AUTHORIZATION='token {}'.format(self.token)) 63 self.assertEqual(response.status_code, 204) 64 # let's get chaotic 65 chaos.CREATE_ERROR_RATE = 0.5 66 # scale up but expect a 503 67 url = "/v1/apps/{app_id}/scale".format(**locals()) 68 body = {'web': 20} 69 response = self.client.post(url, json.dumps(body), content_type='application/json', 70 HTTP_AUTHORIZATION='token {}'.format(self.token)) 71 self.assertEqual(response.status_code, 503) 72 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 73 self.assertEqual(response.get('content-type'), 'application/json') 74 # inspect broken containers 75 url = "/v1/apps/{app_id}/containers".format(**locals()) 76 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 77 self.assertEqual(response.status_code, 200) 78 self.assertEqual(len(response.data['results']), 20) 79 # make sure some failed 80 states = set([c['state'] for c in response.data['results']]) 81 self.assertEqual(states, set(['error', 'created'])) 82 83 def test_start_chaos(self): 84 url = '/v1/apps' 85 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 86 self.assertEqual(response.status_code, 201) 87 app_id = response.data['id'] 88 # post a new build 89 url = "/v1/apps/{app_id}/builds".format(**locals()) 90 body = {'image': 'autotest/example', 'sha': 'a'*40, 91 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 92 response = self.client.post(url, json.dumps(body), content_type='application/json', 93 HTTP_AUTHORIZATION='token {}'.format(self.token)) 94 self.assertEqual(response.status_code, 201) 95 url = "/v1/apps/{app_id}/containers".format(**locals()) 96 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 97 self.assertEqual(response.status_code, 200) 98 self.assertEqual(len(response.data['results']), 1) 99 # scale to zero for consistency 100 url = "/v1/apps/{app_id}/scale".format(**locals()) 101 body = {'web': 0} 102 response = self.client.post(url, json.dumps(body), content_type='application/json', 103 HTTP_AUTHORIZATION='token {}'.format(self.token)) 104 self.assertEqual(response.status_code, 204) 105 # let's get chaotic 106 chaos.START_ERROR_RATE = 0.5 107 # scale up, which will allow some crashed containers 108 url = "/v1/apps/{app_id}/scale".format(**locals()) 109 body = {'web': 20} 110 response = self.client.post(url, json.dumps(body), content_type='application/json', 111 HTTP_AUTHORIZATION='token {}'.format(self.token)) 112 self.assertEqual(response.status_code, 204) 113 # inspect broken containers 114 url = "/v1/apps/{app_id}/containers".format(**locals()) 115 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 116 self.assertEqual(response.status_code, 200) 117 self.assertEqual(len(response.data['results']), 20) 118 # make sure some failed 119 states = set([c['state'] for c in response.data['results']]) 120 self.assertEqual(states, set(['crashed', 'up'])) 121 122 def test_destroy_chaos(self): 123 url = '/v1/apps' 124 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 125 self.assertEqual(response.status_code, 201) 126 app_id = response.data['id'] 127 # post a new build 128 url = "/v1/apps/{app_id}/builds".format(**locals()) 129 body = {'image': 'autotest/example', 'sha': 'a'*40, 130 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 131 response = self.client.post(url, json.dumps(body), content_type='application/json', 132 HTTP_AUTHORIZATION='token {}'.format(self.token)) 133 self.assertEqual(response.status_code, 201) 134 url = "/v1/apps/{app_id}/containers".format(**locals()) 135 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 136 self.assertEqual(response.status_code, 200) 137 self.assertEqual(len(response.data['results']), 1) 138 # scale up 139 url = "/v1/apps/{app_id}/scale".format(**locals()) 140 body = {'web': 20} 141 response = self.client.post(url, json.dumps(body), content_type='application/json', 142 HTTP_AUTHORIZATION='token {}'.format(self.token)) 143 self.assertEqual(response.status_code, 204) 144 url = "/v1/apps/{app_id}/containers".format(**locals()) 145 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 146 self.assertEqual(response.status_code, 200) 147 self.assertEqual(len(response.data['results']), 20) 148 # let's get chaotic 149 chaos.DESTROY_ERROR_RATE = 0.5 150 # scale to zero but expect a 503 151 url = "/v1/apps/{app_id}/scale".format(**locals()) 152 body = {'web': 0} 153 response = self.client.post(url, json.dumps(body), content_type='application/json', 154 HTTP_AUTHORIZATION='token {}'.format(self.token)) 155 self.assertEqual(response.status_code, 503) 156 self.assertEqual(response.data, {'detail': 'aborting, failed to destroy some containers'}) 157 self.assertEqual(response.get('content-type'), 'application/json') 158 # inspect broken containers 159 url = "/v1/apps/{app_id}/containers".format(**locals()) 160 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 161 self.assertEqual(response.status_code, 200) 162 states = set([c['state'] for c in response.data['results']]) 163 self.assertEqual(states, set(['error'])) 164 # make sure we can cleanup after enough tries 165 containers = 20 166 for _ in range(100): 167 url = "/v1/apps/{app_id}/scale".format(**locals()) 168 body = {'web': 0} 169 response = self.client.post(url, json.dumps(body), content_type='application/json', 170 HTTP_AUTHORIZATION='token {}'.format(self.token)) 171 # break if we destroyed successfully 172 if response.status_code == 204: 173 break 174 self.assertEqual(response.status_code, 503) 175 self.assertEqual(response.data, {'detail': 'aborting, failed to ' 176 'destroy some containers'}) 177 self.assertEqual(response.get('content-type'), 'application/json') 178 # inspect broken containers 179 url = "/v1/apps/{app_id}/containers".format(**locals()) 180 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 181 self.assertEqual(response.status_code, 200) 182 containers = len(response.data['results']) 183 184 def test_build_chaos(self): 185 url = '/v1/apps' 186 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 187 self.assertEqual(response.status_code, 201) 188 app_id = response.data['id'] 189 # post a new build 190 url = "/v1/apps/{app_id}/builds".format(**locals()) 191 body = {'image': 'autotest/example', 'sha': 'a'*40, 192 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 193 response = self.client.post(url, json.dumps(body), content_type='application/json', 194 HTTP_AUTHORIZATION='token {}'.format(self.token)) 195 self.assertEqual(response.status_code, 201) 196 # inspect builds 197 url = "/v1/apps/{app_id}/builds".format(**locals()) 198 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 199 self.assertEqual(response.status_code, 200) 200 self.assertEqual(len(response.data['results']), 1) 201 # inspect releases 202 url = "/v1/apps/{app_id}/releases".format(**locals()) 203 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 204 self.assertEqual(response.status_code, 200) 205 self.assertEqual(len(response.data['results']), 2) 206 url = "/v1/apps/{app_id}/containers".format(**locals()) 207 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 208 self.assertEqual(response.status_code, 200) 209 self.assertEqual(len(response.data['results']), 1) 210 # scale up 211 url = "/v1/apps/{app_id}/scale".format(**locals()) 212 body = {'web': 20} 213 response = self.client.post(url, json.dumps(body), content_type='application/json', 214 HTTP_AUTHORIZATION='token {}'.format(self.token)) 215 self.assertEqual(response.status_code, 204) 216 # simulate failing to create containers 217 chaos.CREATE_ERROR_RATE = 0.5 218 chaos.START_ERROR_RATE = 0.5 219 # post a new build 220 url = "/v1/apps/{app_id}/builds".format(**locals()) 221 body = {'image': 'autotest/example', 'sha': 'b'*40, 222 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 223 response = self.client.post(url, json.dumps(body), content_type='application/json', 224 HTTP_AUTHORIZATION='token {}'.format(self.token)) 225 self.assertEqual(response.status_code, 503) 226 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 227 self.assertEqual(response.get('content-type'), 'application/json') 228 # inspect releases 229 url = "/v1/apps/{app_id}/releases".format(**locals()) 230 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 231 self.assertEqual(response.status_code, 200) 232 self.assertEqual(len(response.data['results']), 2) 233 # inspect containers 234 url = "/v1/apps/{app_id}/containers".format(**locals()) 235 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 236 self.assertEqual(response.status_code, 200) 237 self.assertEqual(len(response.data['results']), 20) 238 239 # make sure all old containers are still up 240 states = set([c['state'] for c in response.data['results']]) 241 self.assertEqual(states, set(['up'])) 242 243 def test_config_chaos(self): 244 url = '/v1/apps' 245 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 246 self.assertEqual(response.status_code, 201) 247 app_id = response.data['id'] 248 # post a new build 249 url = "/v1/apps/{app_id}/builds".format(**locals()) 250 body = {'image': 'autotest/example', 'sha': 'a'*40, 251 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 252 response = self.client.post(url, json.dumps(body), content_type='application/json', 253 HTTP_AUTHORIZATION='token {}'.format(self.token)) 254 self.assertEqual(response.status_code, 201) 255 # inspect releases 256 url = "/v1/apps/{app_id}/releases".format(**locals()) 257 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 258 self.assertEqual(response.status_code, 200) 259 self.assertEqual(len(response.data['results']), 2) 260 url = "/v1/apps/{app_id}/containers".format(**locals()) 261 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 262 self.assertEqual(response.status_code, 200) 263 self.assertEqual(len(response.data['results']), 1) 264 # scale up 265 url = "/v1/apps/{app_id}/scale".format(**locals()) 266 body = {'web': 20} 267 response = self.client.post(url, json.dumps(body), content_type='application/json', 268 HTTP_AUTHORIZATION='token {}'.format(self.token)) 269 self.assertEqual(response.status_code, 204) 270 # simulate failing to create or start containers 271 chaos.CREATE_ERROR_RATE = 0.5 272 chaos.START_ERROR_RATE = 0.5 273 # post a new config 274 url = "/v1/apps/{app_id}/config".format(**locals()) 275 body = {'values': json.dumps({'NEW_URL1': 'http://localhost:8080/'})} 276 response = self.client.post(url, json.dumps(body), content_type='application/json', 277 HTTP_AUTHORIZATION='token {}'.format(self.token)) 278 self.assertEqual(response.status_code, 503) 279 self.assertEqual(response.data, {'detail': 'aborting, failed to create some containers'}) 280 self.assertEqual(response.get('content-type'), 'application/json') 281 # inspect releases 282 url = "/v1/apps/{app_id}/releases".format(**locals()) 283 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 284 self.assertEqual(response.status_code, 200) 285 self.assertEqual(len(response.data['results']), 2) 286 # inspect containers 287 url = "/v1/apps/{app_id}/containers".format(**locals()) 288 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 289 self.assertEqual(response.status_code, 200) 290 self.assertEqual(len(response.data['results']), 20) 291 # make sure all old containers are still up 292 states = set([c['state'] for c in response.data['results']]) 293 self.assertEqual(states, set(['up'])) 294 295 def test_run_chaos(self): 296 url = '/v1/apps' 297 response = self.client.post(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 298 self.assertEqual(response.status_code, 201) 299 app_id = response.data['id'] 300 # post a new build 301 url = "/v1/apps/{app_id}/builds".format(**locals()) 302 body = {'image': 'autotest/example', 'sha': 'a'*40, 303 'procfile': json.dumps({'web': 'node server.js', 'worker': 'node worker.js'})} 304 response = self.client.post(url, json.dumps(body), content_type='application/json', 305 HTTP_AUTHORIZATION='token {}'.format(self.token)) 306 self.assertEqual(response.status_code, 201) 307 # inspect builds 308 url = "/v1/apps/{app_id}/builds".format(**locals()) 309 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 310 self.assertEqual(response.status_code, 200) 311 self.assertEqual(len(response.data['results']), 1) 312 # inspect releases 313 url = "/v1/apps/{app_id}/releases".format(**locals()) 314 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 315 self.assertEqual(response.status_code, 200) 316 self.assertEqual(len(response.data['results']), 2) 317 url = "/v1/apps/{app_id}/containers".format(**locals()) 318 response = self.client.get(url, HTTP_AUTHORIZATION='token {}'.format(self.token)) 319 self.assertEqual(response.status_code, 200) 320 self.assertEqual(len(response.data['results']), 1) 321 # block all create operations 322 chaos.CREATE_ERROR_RATE = 1 323 # make sure the run fails with a 503 324 url = '/v1/apps/{app_id}/run'.format(**locals()) 325 body = {'command': 'ls -al'} 326 response = self.client.post(url, json.dumps(body), content_type='application/json', 327 HTTP_AUTHORIZATION='token {}'.format(self.token)) 328 self.assertEqual(response.status_code, 503) 329 self.assertEqual(response.data, {'detail': 'exit code 1'}) 330 self.assertEqual(response.get('content-type'), 'application/json')