github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/pfs_middleware/tests/test_bimodal_checker.py (about) 1 # Copyright (c) 2015-2021, NVIDIA CORPORATION. 2 # SPDX-License-Identifier: Apache-2.0 3 4 5 import errno 6 import mock 7 import os 8 import socket 9 import unittest 10 11 from swift.common import swob 12 from . import helpers 13 14 import pfs_middleware.bimodal_checker as bimodal_checker 15 import pfs_middleware.utils as utils 16 17 18 class FakeJsonRpcWithErrors(helpers.FakeJsonRpc): 19 def __init__(self, *a, **kw): 20 super(FakeJsonRpcWithErrors, self).__init__(*a, **kw) 21 self._errors = [] 22 23 def add_call_error(self, ex): 24 self._errors.append(ex) 25 26 def call(self, *a, **kw): 27 # Call super() so we get call tracking. 28 retval = super(FakeJsonRpcWithErrors, self).call(*a, **kw) 29 if self._errors: 30 raise self._errors.pop(0) 31 return retval 32 33 34 class BimodalHeaderinator(object): 35 def __init__(self, app): 36 self.app = app 37 38 def __call__(self, environ, start_response): 39 is_bimodal = bool(environ.get(utils.ENV_IS_BIMODAL)) 40 41 def my_sr(status, headers, exc_info=None): 42 my_headers = list(headers) 43 my_headers.append(("Is-Bimodal", ('yes' if is_bimodal else 'no'))) 44 return start_response(status, my_headers, exc_info) 45 46 return self.app(environ, my_sr) 47 48 49 class TestDecision(unittest.TestCase): 50 def setUp(self): 51 self.app = helpers.FakeProxy() 52 bh = BimodalHeaderinator(self.app) 53 self.bc = bimodal_checker.BimodalChecker(bh, { 54 'bimodal_recheck_interval': '5.0', 55 }) 56 self.fake_rpc = helpers.FakeJsonRpc() 57 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 58 lambda *_: self.fake_rpc) 59 patcher.start() 60 self.addCleanup(patcher.stop) 61 62 self.app.register('HEAD', '/v1/alice', 204, {}, '') 63 self.app.register('HEAD', '/v1/bob', 204, 64 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 65 '') 66 self.app.register('HEAD', '/v1/carol', 204, 67 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 68 '') 69 70 def fake_RpcIsAccountBimodal(request): 71 return { 72 "error": None, 73 "result": { 74 "IsBimodal": True, 75 "ActivePeerPrivateIPAddr": 76 "fc00:a377:29bc:fc90:9808:ba1f:e94b:1215"}} 77 78 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 79 fake_RpcIsAccountBimodal) 80 81 def test_not_bimodal(self): 82 req = swob.Request.blank("/v1/alice", 83 environ={"REQUEST_METHOD": "HEAD"}) 84 resp = req.get_response(self.bc) 85 86 self.assertEqual("no", resp.headers["Is-Bimodal"]) 87 # Swift said no, so we didn't bother with an RPC. 88 self.assertEqual(tuple(), self.fake_rpc.calls) 89 90 def test_bimodal(self): 91 req = swob.Request.blank("/v1/bob", 92 environ={"REQUEST_METHOD": "HEAD"}) 93 resp = req.get_response(self.bc) 94 95 self.assertEqual("yes", resp.headers["Is-Bimodal"]) 96 self.assertEqual(self.fake_rpc.calls, ( 97 ('Server.RpcIsAccountBimodal', ({'AccountName': 'bob'},)),)) 98 99 def test_bad_path(self): 100 self.app.register('GET', '//v1/alice', 404, {}, '') 101 req = swob.Request.blank("//v1/alice") 102 # (in)sanity check 103 self.assertNotEqual(req.environ['PATH_INFO'], '//v1/alice') 104 # fix it 105 req.environ['PATH_INFO'] = '//v1/alice' 106 resp = req.get_response(self.bc) 107 108 self.assertEqual("no", resp.headers["Is-Bimodal"]) 109 # We just passed through -- didn't bother with an RPC. 110 self.assertEqual(tuple(), self.fake_rpc.calls) 111 112 def test_disagreement(self): 113 def fake_RpcIsAccountBimodal(request): 114 return { 115 "error": None, 116 "result": {"IsBimodal": False, 117 "ActivePeerPrivateIPAddr": ""}} 118 119 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 120 fake_RpcIsAccountBimodal) 121 122 req = swob.Request.blank("/v1/bob", 123 environ={"REQUEST_METHOD": "HEAD"}) 124 resp = req.get_response(self.bc) 125 126 self.assertEqual(resp.status_int, 503) 127 128 129 class TestBimodalCaching(unittest.TestCase): 130 def setUp(self): 131 self.app = helpers.FakeProxy() 132 self.bc = bimodal_checker.BimodalChecker(self.app, { 133 'bimodal_recheck_interval': '5.0', 134 }) 135 self.fake_rpc = helpers.FakeJsonRpc() 136 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 137 lambda *_: self.fake_rpc) 138 patcher.start() 139 self.addCleanup(patcher.stop) 140 141 self.app.register('HEAD', '/v1/alice', 204, {}, '') 142 self.app.register('HEAD', '/v1/bob', 204, 143 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 144 '') 145 self.app.register('HEAD', '/v1/carol', 204, 146 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 147 '') 148 self.app.register('HEAD', '/v1/dave', 204, 149 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 150 '') 151 152 def test_caching(self): 153 the_time = [12345.6] 154 rpc_iab_calls = [] 155 156 def fake_time_function(): 157 now = the_time[0] 158 the_time[0] += 0.001 159 return now 160 161 fake_time_module = mock.Mock(time=fake_time_function) 162 163 def fake_RpcIsAccountBimodal(request): 164 acc = request["AccountName"] 165 rpc_iab_calls.append(acc) 166 167 if acc == 'alice': 168 # Not bimodal 169 return { 170 "error": None, 171 "result": { 172 "IsBimodal": False, 173 "ActivePeerPrivateIPAddr": ""}} 174 elif acc == 'bob': 175 # Normal, happy bimodal account 176 return { 177 "error": None, 178 "result": { 179 "IsBimodal": True, 180 "ActivePeerPrivateIPAddr": "10.221.76.210"}} 181 elif acc == 'carol': 182 # Temporarily in limbo as it's being moved from one proxyfsd 183 # to another 184 return { 185 "error": None, 186 "result": { 187 "IsBimodal": True, 188 "ActivePeerPrivateIPAddr": ""}} 189 elif acc == 'dave': 190 # Another bimodal account 191 return { 192 "error": None, 193 "result": { 194 "IsBimodal": True, 195 "ActivePeerPrivateIPAddr": "10.221.76.210"}} 196 else: 197 raise ValueError("test helper can't handle %r" % (acc,)) 198 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 199 fake_RpcIsAccountBimodal) 200 201 status = [None] 202 203 def start_response(s, h, ei=None): 204 status[0] = s 205 206 with mock.patch('pfs_middleware.bimodal_checker.time', 207 fake_time_module): 208 a_req = swob.Request.blank("/v1/alice", 209 environ={"REQUEST_METHOD": "HEAD"}) 210 b_req = swob.Request.blank("/v1/bob", 211 environ={"REQUEST_METHOD": "HEAD"}) 212 c_req = swob.Request.blank("/v1/carol", 213 environ={"REQUEST_METHOD": "HEAD"}) 214 d_req = swob.Request.blank("/v1/dave", 215 environ={"REQUEST_METHOD": "HEAD"}) 216 217 # Negative results are served without any RPCs at all 218 list(self.bc(a_req.environ, start_response)) 219 self.assertEqual(status[0], '204 No Content') # sanity check 220 self.assertEqual(rpc_iab_calls, []) 221 222 # First time, we have a completely empty cache, so an RPC is made 223 list(self.bc(d_req.environ, start_response)) 224 self.assertEqual(status[0], '204 No Content') # sanity check 225 self.assertEqual(rpc_iab_calls, ['dave']) 226 227 # A couple seconds later, a second request for the same account 228 # comes in, and is handled from cache 229 the_time[0] += 2 230 del rpc_iab_calls[:] 231 list(self.bc(d_req.environ, start_response)) 232 self.assertEqual(status[0], '204 No Content') # sanity check 233 self.assertEqual(rpc_iab_calls, []) 234 235 # If a request for another account comes in, it is cached 236 # separately. 237 del rpc_iab_calls[:] 238 list(self.bc(b_req.environ, start_response)) 239 self.assertEqual(status[0], '204 No Content') # sanity check 240 self.assertEqual(rpc_iab_calls, ["bob"]) 241 242 # Each account has its own cache time 243 the_time[0] += 3 # "dave" is now invalid, "bob" remains valid 244 del rpc_iab_calls[:] 245 list(self.bc(d_req.environ, start_response)) 246 list(self.bc(b_req.environ, start_response)) 247 self.assertEqual(rpc_iab_calls, ["dave"]) 248 249 # In-transit accounts don't get cached 250 del rpc_iab_calls[:] 251 list(self.bc(c_req.environ, start_response)) 252 list(self.bc(c_req.environ, start_response)) 253 self.assertEqual(rpc_iab_calls, ["carol", "carol"]) 254 255 256 class TestRetry(unittest.TestCase): 257 def setUp(self): 258 self.app = helpers.FakeProxy() 259 self.bc = bimodal_checker.BimodalChecker(self.app, { 260 'bimodal_recheck_interval': '5.0', 261 'proxyfsd_host': '10.1.1.1, 10.2.2.2', 262 }) 263 self.fake_rpc = FakeJsonRpcWithErrors() 264 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 265 lambda *_: self.fake_rpc) 266 patcher.start() 267 self.addCleanup(patcher.stop) 268 self.app.register('HEAD', '/v1/AUTH_test', 204, 269 {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, 270 '') 271 272 def fake_RpcIsAccountBimodal(request): 273 return { 274 "error": None, 275 "result": { 276 "IsBimodal": True, 277 "ActivePeerPrivateIPAddr": "10.9.8.7", 278 }} 279 280 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 281 fake_RpcIsAccountBimodal) 282 283 def test_retry_socketerror(self): 284 self.fake_rpc.add_call_error( 285 socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED))) 286 287 req = swob.Request.blank( 288 "/v1/AUTH_test", 289 environ={'REQUEST_METHOD': 'HEAD'}) 290 resp = req.get_response(self.bc) 291 self.assertEqual(resp.status_int, 204) 292 293 self.assertEqual(self.fake_rpc.calls, ( 294 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)), 295 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)))) 296 297 def test_no_retry_timeout(self): 298 err = utils.RpcTimeout() 299 self.fake_rpc.add_call_error(err) 300 301 req = swob.Request.blank( 302 "/v1/AUTH_test", 303 environ={'REQUEST_METHOD': 'HEAD'}) 304 resp = req.get_response(self.bc) 305 self.assertEqual(resp.status_int, 503) 306 307 self.assertEqual(self.fake_rpc.calls, ( 308 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)),)) 309 310 def test_no_catch_other_error(self): 311 self.fake_rpc.add_call_error(ZeroDivisionError) 312 313 req = swob.Request.blank( 314 "/v1/AUTH_test", 315 environ={'REQUEST_METHOD': 'HEAD'}) 316 317 self.assertRaises(ZeroDivisionError, req.get_response, self.bc) 318 self.assertEqual(self.fake_rpc.calls, ( 319 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)),))