github.com/swiftstack/proxyfs@v0.0.0-20201223034610-5434d919416e/pfs_middleware/tests/test_bimodal_checker.py (about) 1 # Copyright (c) 2017 SwiftStack, Inc. 2 # 3 # Licensed under the Apache License, Version 2.0 (the "License"); 4 # you may not use this file except in compliance with the License. 5 # You may obtain a copy of the License at 6 # 7 # http://www.apache.org/licenses/LICENSE-2.0 8 # 9 # Unless required by applicable law or agreed to in writing, software 10 # distributed under the License is distributed on an "AS IS" BASIS, 11 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 12 # implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 import errno 17 import mock 18 import os 19 import socket 20 import unittest 21 22 from swift.common import swob 23 from . import helpers 24 25 import pfs_middleware.bimodal_checker as bimodal_checker 26 import pfs_middleware.utils as utils 27 28 29 class FakeJsonRpcWithErrors(helpers.FakeJsonRpc): 30 def __init__(self, *a, **kw): 31 super(FakeJsonRpcWithErrors, self).__init__(*a, **kw) 32 self._errors = [] 33 34 def add_call_error(self, ex): 35 self._errors.append(ex) 36 37 def call(self, *a, **kw): 38 # Call super() so we get call tracking. 39 retval = super(FakeJsonRpcWithErrors, self).call(*a, **kw) 40 if self._errors: 41 raise self._errors.pop(0) 42 return retval 43 44 45 class BimodalHeaderinator(object): 46 def __init__(self, app): 47 self.app = app 48 49 def __call__(self, environ, start_response): 50 is_bimodal = bool(environ.get(utils.ENV_IS_BIMODAL)) 51 52 def my_sr(status, headers, exc_info=None): 53 my_headers = list(headers) 54 my_headers.append(("Is-Bimodal", ('yes' if is_bimodal else 'no'))) 55 return start_response(status, my_headers, exc_info) 56 57 return self.app(environ, my_sr) 58 59 60 class TestDecision(unittest.TestCase): 61 def setUp(self): 62 self.app = helpers.FakeProxy() 63 bh = BimodalHeaderinator(self.app) 64 self.bc = bimodal_checker.BimodalChecker(bh, { 65 'bimodal_recheck_interval': '5.0', 66 }) 67 self.fake_rpc = helpers.FakeJsonRpc() 68 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 69 lambda *_: self.fake_rpc) 70 patcher.start() 71 self.addCleanup(patcher.stop) 72 73 self.app.register('HEAD', '/v1/alice', 204, {}, '') 74 self.app.register('HEAD', '/v1/bob', 204, 75 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 76 '') 77 self.app.register('HEAD', '/v1/carol', 204, 78 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 79 '') 80 81 def fake_RpcIsAccountBimodal(request): 82 return { 83 "error": None, 84 "result": { 85 "IsBimodal": True, 86 "ActivePeerPrivateIPAddr": 87 "fc00:a377:29bc:fc90:9808:ba1f:e94b:1215"}} 88 89 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 90 fake_RpcIsAccountBimodal) 91 92 def test_not_bimodal(self): 93 req = swob.Request.blank("/v1/alice", 94 environ={"REQUEST_METHOD": "HEAD"}) 95 resp = req.get_response(self.bc) 96 97 self.assertEqual("no", resp.headers["Is-Bimodal"]) 98 # Swift said no, so we didn't bother with an RPC. 99 self.assertEqual(tuple(), self.fake_rpc.calls) 100 101 def test_bimodal(self): 102 req = swob.Request.blank("/v1/bob", 103 environ={"REQUEST_METHOD": "HEAD"}) 104 resp = req.get_response(self.bc) 105 106 self.assertEqual("yes", resp.headers["Is-Bimodal"]) 107 self.assertEqual(self.fake_rpc.calls, ( 108 ('Server.RpcIsAccountBimodal', ({'AccountName': 'bob'},)),)) 109 110 def test_bad_path(self): 111 self.app.register('GET', '//v1/alice', 404, {}, '') 112 req = swob.Request.blank("//v1/alice") 113 # (in)sanity check 114 self.assertNotEqual(req.environ['PATH_INFO'], '//v1/alice') 115 # fix it 116 req.environ['PATH_INFO'] = '//v1/alice' 117 resp = req.get_response(self.bc) 118 119 self.assertEqual("no", resp.headers["Is-Bimodal"]) 120 # We just passed through -- didn't bother with an RPC. 121 self.assertEqual(tuple(), self.fake_rpc.calls) 122 123 def test_disagreement(self): 124 def fake_RpcIsAccountBimodal(request): 125 return { 126 "error": None, 127 "result": {"IsBimodal": False, 128 "ActivePeerPrivateIPAddr": ""}} 129 130 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 131 fake_RpcIsAccountBimodal) 132 133 req = swob.Request.blank("/v1/bob", 134 environ={"REQUEST_METHOD": "HEAD"}) 135 resp = req.get_response(self.bc) 136 137 self.assertEqual(resp.status_int, 503) 138 139 140 class TestBimodalCaching(unittest.TestCase): 141 def setUp(self): 142 self.app = helpers.FakeProxy() 143 self.bc = bimodal_checker.BimodalChecker(self.app, { 144 'bimodal_recheck_interval': '5.0', 145 }) 146 self.fake_rpc = helpers.FakeJsonRpc() 147 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 148 lambda *_: self.fake_rpc) 149 patcher.start() 150 self.addCleanup(patcher.stop) 151 152 self.app.register('HEAD', '/v1/alice', 204, {}, '') 153 self.app.register('HEAD', '/v1/bob', 204, 154 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 155 '') 156 self.app.register('HEAD', '/v1/carol', 204, 157 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 158 '') 159 self.app.register('HEAD', '/v1/dave', 204, 160 {"X-Account-Sysmeta-ProxyFS-Bimodal": "true"}, 161 '') 162 163 def test_caching(self): 164 the_time = [12345.6] 165 rpc_iab_calls = [] 166 167 def fake_time_function(): 168 now = the_time[0] 169 the_time[0] += 0.001 170 return now 171 172 fake_time_module = mock.Mock(time=fake_time_function) 173 174 def fake_RpcIsAccountBimodal(request): 175 acc = request["AccountName"] 176 rpc_iab_calls.append(acc) 177 178 if acc == 'alice': 179 # Not bimodal 180 return { 181 "error": None, 182 "result": { 183 "IsBimodal": False, 184 "ActivePeerPrivateIPAddr": ""}} 185 elif acc == 'bob': 186 # Normal, happy bimodal account 187 return { 188 "error": None, 189 "result": { 190 "IsBimodal": True, 191 "ActivePeerPrivateIPAddr": "10.221.76.210"}} 192 elif acc == 'carol': 193 # Temporarily in limbo as it's being moved from one proxyfsd 194 # to another 195 return { 196 "error": None, 197 "result": { 198 "IsBimodal": True, 199 "ActivePeerPrivateIPAddr": ""}} 200 elif acc == 'dave': 201 # Another bimodal account 202 return { 203 "error": None, 204 "result": { 205 "IsBimodal": True, 206 "ActivePeerPrivateIPAddr": "10.221.76.210"}} 207 else: 208 raise ValueError("test helper can't handle %r" % (acc,)) 209 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 210 fake_RpcIsAccountBimodal) 211 212 status = [None] 213 214 def start_response(s, h, ei=None): 215 status[0] = s 216 217 with mock.patch('pfs_middleware.bimodal_checker.time', 218 fake_time_module): 219 a_req = swob.Request.blank("/v1/alice", 220 environ={"REQUEST_METHOD": "HEAD"}) 221 b_req = swob.Request.blank("/v1/bob", 222 environ={"REQUEST_METHOD": "HEAD"}) 223 c_req = swob.Request.blank("/v1/carol", 224 environ={"REQUEST_METHOD": "HEAD"}) 225 d_req = swob.Request.blank("/v1/dave", 226 environ={"REQUEST_METHOD": "HEAD"}) 227 228 # Negative results are served without any RPCs at all 229 list(self.bc(a_req.environ, start_response)) 230 self.assertEqual(status[0], '204 No Content') # sanity check 231 self.assertEqual(rpc_iab_calls, []) 232 233 # First time, we have a completely empty cache, so an RPC is made 234 list(self.bc(d_req.environ, start_response)) 235 self.assertEqual(status[0], '204 No Content') # sanity check 236 self.assertEqual(rpc_iab_calls, ['dave']) 237 238 # A couple seconds later, a second request for the same account 239 # comes in, and is handled from cache 240 the_time[0] += 2 241 del rpc_iab_calls[:] 242 list(self.bc(d_req.environ, start_response)) 243 self.assertEqual(status[0], '204 No Content') # sanity check 244 self.assertEqual(rpc_iab_calls, []) 245 246 # If a request for another account comes in, it is cached 247 # separately. 248 del rpc_iab_calls[:] 249 list(self.bc(b_req.environ, start_response)) 250 self.assertEqual(status[0], '204 No Content') # sanity check 251 self.assertEqual(rpc_iab_calls, ["bob"]) 252 253 # Each account has its own cache time 254 the_time[0] += 3 # "dave" is now invalid, "bob" remains valid 255 del rpc_iab_calls[:] 256 list(self.bc(d_req.environ, start_response)) 257 list(self.bc(b_req.environ, start_response)) 258 self.assertEqual(rpc_iab_calls, ["dave"]) 259 260 # In-transit accounts don't get cached 261 del rpc_iab_calls[:] 262 list(self.bc(c_req.environ, start_response)) 263 list(self.bc(c_req.environ, start_response)) 264 self.assertEqual(rpc_iab_calls, ["carol", "carol"]) 265 266 267 class TestRetry(unittest.TestCase): 268 def setUp(self): 269 self.app = helpers.FakeProxy() 270 self.bc = bimodal_checker.BimodalChecker(self.app, { 271 'bimodal_recheck_interval': '5.0', 272 'proxyfsd_host': '10.1.1.1, 10.2.2.2', 273 }) 274 self.fake_rpc = FakeJsonRpcWithErrors() 275 patcher = mock.patch('pfs_middleware.utils.JsonRpcClient', 276 lambda *_: self.fake_rpc) 277 patcher.start() 278 self.addCleanup(patcher.stop) 279 self.app.register('HEAD', '/v1/AUTH_test', 204, 280 {'X-Account-Sysmeta-ProxyFS-Bimodal': 'true'}, 281 '') 282 283 def fake_RpcIsAccountBimodal(request): 284 return { 285 "error": None, 286 "result": { 287 "IsBimodal": True, 288 "ActivePeerPrivateIPAddr": "10.9.8.7", 289 }} 290 291 self.fake_rpc.register_handler("Server.RpcIsAccountBimodal", 292 fake_RpcIsAccountBimodal) 293 294 def test_retry_socketerror(self): 295 self.fake_rpc.add_call_error( 296 socket.error(errno.ECONNREFUSED, os.strerror(errno.ECONNREFUSED))) 297 298 req = swob.Request.blank( 299 "/v1/AUTH_test", 300 environ={'REQUEST_METHOD': 'HEAD'}) 301 resp = req.get_response(self.bc) 302 self.assertEqual(resp.status_int, 204) 303 304 self.assertEqual(self.fake_rpc.calls, ( 305 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)), 306 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)))) 307 308 def test_no_retry_timeout(self): 309 err = utils.RpcTimeout() 310 self.fake_rpc.add_call_error(err) 311 312 req = swob.Request.blank( 313 "/v1/AUTH_test", 314 environ={'REQUEST_METHOD': 'HEAD'}) 315 resp = req.get_response(self.bc) 316 self.assertEqual(resp.status_int, 503) 317 318 self.assertEqual(self.fake_rpc.calls, ( 319 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)),)) 320 321 def test_no_catch_other_error(self): 322 self.fake_rpc.add_call_error(ZeroDivisionError) 323 324 req = swob.Request.blank( 325 "/v1/AUTH_test", 326 environ={'REQUEST_METHOD': 'HEAD'}) 327 328 self.assertRaises(ZeroDivisionError, req.get_response, self.bc) 329 self.assertEqual(self.fake_rpc.calls, ( 330 ('Server.RpcIsAccountBimodal', ({'AccountName': 'AUTH_test'},)),))