github.com/rohankumardubey/proxyfs@v0.0.0-20210108201508-653efa9ab00e/regression_test.py (about) 1 #!/usr/bin/env python 2 3 """ 4 A script to lint and test ProxyFS project code. 5 """ 6 7 from __future__ import print_function, unicode_literals 8 from threading import Timer 9 10 import os 11 import argparse 12 import functools 13 import logging 14 import platform 15 import contextlib 16 import subprocess 17 import shutil 18 import sys 19 import tempfile 20 import time 21 22 23 PACKAGES = ["blunder", 24 "cleanproxyfs", 25 "conf", 26 "dlm", 27 "fs", 28 "fsworkout", 29 "fuse", 30 "headhunter", 31 "httpserver", 32 "inode", 33 "inodeworkout", 34 "jrpcfs", 35 "logger", 36 "mkproxyfs", "mkproxyfs/mkproxyfs", 37 "pfs-stress", 38 "pfsconfjson", "pfsconfjsonpacked", 39 "pfsworkout", 40 "platform", 41 "proxyfsd", "proxyfsd/proxyfsd", 42 "ramswift", "ramswift/ramswift", 43 "stats", 44 "statslogger", 45 "swiftclient", 46 "utils"] 47 48 49 COLORS = {"bright red": '1;31', "bright green": '1;32'} 50 51 52 @contextlib.contextmanager 53 def return_to_wd(): 54 curdir = os.getcwd() 55 try: 56 yield 57 finally: 58 os.chdir(curdir) 59 60 61 @contextlib.contextmanager 62 def self_cleaning_tempdir(*args, **kwargs): 63 our_tempdir = tempfile.mkdtemp(*args, **kwargs) 64 try: 65 yield our_tempdir 66 finally: 67 shutil.rmtree(our_tempdir, ignore_errors=True) 68 69 70 def proxyfs_binary_path(binary): 71 try: 72 gopath = os.environ["GOPATH"] 73 except KeyError: 74 color_print("$GOPATH must be set", 'bright red') 75 os.exit(1) 76 return os.path.join(gopath, "bin", binary) 77 78 79 def color_print(content, color=None): 80 print("\x1b[{color}m{content}\x1b[0m".format(content=content, 81 color=COLORS[color])) 82 83 84 def proxyfs_package_path(package): 85 try: 86 gopath = os.environ["GOPATH"] 87 except KeyError: 88 color_print("$GOPATH must be set", 'bright red') 89 os.exit(1) 90 return os.path.join(gopath, "src/github.com/swiftstack/ProxyFS", package) 91 92 93 def color_print(content, color=None): 94 print("\x1b[{color}m{content}\x1b[0m".format(content=content, 95 color=COLORS[color])) 96 97 98 def report(task, success=False): 99 printer = color_print if sys.stdout.isatty() else lambda *a, **kw: print(*a) 100 if success: 101 printer("{} {}".format(task, "succeeded!"), color="bright green") 102 else: 103 printer("{} {}".format(task, "failed!"), color="bright red") 104 105 106 class GoCommand(object): 107 def __init__(self, command, project_path, 108 options=None, report_as=None, skip=False): 109 self.command = command 110 self.project_path = project_path 111 self.options = options or [] 112 self.report_as = report_as 113 self.skip = skip 114 115 def execute(self, package): 116 if self.skip: 117 return None 118 package_path = "{}{}".format(self.project_path, package) 119 command_line = ["go", self.command] + self.options + [package_path] 120 logging.info(' '.join("'{}'".format(s) if ' ' in s else s 121 for s in command_line)) 122 success = not bool(subprocess.call(command_line)) 123 return success 124 125 126 def get_go_commands(options, project_path, skip_tests=False): 127 commands = [ 128 GoCommand('fmt', project_path), 129 GoCommand('get', project_path, ['-t', '-u'], skip=not options.get), 130 # Source code has to be `go install`ed before `stringer` can run on 131 # it, so we install before generate, which mysteriously does work? 132 # see https://github.com/golang/go/issues/10249 133 GoCommand('install', project_path, ['-gcflags', '-N -l'], report_as="`go install`"), 134 GoCommand('generate', project_path, report_as="`go generate`"), 135 ] 136 if not options.deb_builder: 137 commands.append( 138 GoCommand('test', project_path, 139 filter(lambda x: x, [options.bench, options.cover]), 140 report_as="`go test`", 141 skip=skip_tests) 142 ) 143 commands.append(GoCommand('vet', project_path, report_as="`go vet`")) 144 return commands 145 146 147 def execute_go_commands(commands, packages): 148 reports = [] 149 failures = 0 150 for command in commands: 151 successes = filter(lambda success: success is not None, 152 [command.execute(package) for package in packages]) 153 success = all(successes) 154 failures += not success 155 if command.report_as is not None: 156 reports.append(functools.partial(report, command.report_as, 157 success=success)) 158 for reporter in reports: 159 reporter() 160 return failures 161 162 163 def test_pfs_middleware(options): 164 failures = 0 165 with return_to_wd(): 166 proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) 167 pfs_middleware_dir = os.path.join(proxyfs_dir, "pfs_middleware") 168 os.chdir(pfs_middleware_dir) 169 failures += bool(subprocess.call((['python', 'setup.py', 'test']))) 170 report("test_pfs_middleware()", not failures) 171 return failures 172 173 174 def build_proxyfs(options): 175 commands = get_go_commands(options, "github.com/swiftstack/ProxyFS/") 176 if options.packages is None: 177 selected_packages = PACKAGES 178 else: 179 selected_packages = options.packages 180 return execute_go_commands(commands, selected_packages) 181 182 183 def build_dependencies(options): 184 failures = 0 185 proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) 186 stringer_path = 'golang.org/x/tools/cmd/stringer' 187 full_stringer_path = os.path.join(proxyfs_dir, "vendor", stringer_path) 188 print("Building " + stringer_path) 189 install_cmd = ("go", "install") 190 with return_to_wd(): 191 os.chdir(full_stringer_path) 192 success = not(bool(subprocess.call(install_cmd))) 193 failures += not success 194 report("build_dependencies()", not failures) 195 return failures 196 197 198 def build_jrpcclient(options): 199 proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) 200 jrpcclient_dir = os.path.join(proxyfs_dir, "jrpcclient") 201 command = ['./regression_test.py'] 202 if options.no_install: 203 command.append('--no-install') 204 if options.deb_builder: 205 command.append('--deb-builder') 206 if options.just_build_libs: 207 command.append('--just-build-libs') 208 return bool(subprocess.call(command, cwd=jrpcclient_dir)) 209 210 211 def build_vfs(options): 212 proxyfs_dir = os.path.dirname(os.path.abspath(__file__)) 213 vfs_dir = os.path.join(proxyfs_dir, "vfs") 214 failures = 0 215 clean_cmd = ('make', 'clean') 216 failures += bool(subprocess.call(clean_cmd, cwd=vfs_dir)) 217 if failures: 218 return failures 219 failures += bool(subprocess.call('make', cwd=vfs_dir)) 220 if failures: 221 return failures 222 distro = platform.linux_distribution()[0] 223 if not options.no_install: 224 if options.deb_builder: 225 if 'centos' in distro.lower(): 226 install_cmd = ('make', 'installcentos') 227 else: 228 install_cmd = ('make', 'install') 229 else: 230 if 'centos' in distro.lower(): 231 install_cmd = ('sudo', '-E', 'make', 'installcentos') 232 else: 233 install_cmd = ('sudo', '-E', 'make', 'install') 234 failures += bool(subprocess.call(install_cmd, cwd=vfs_dir)) 235 return failures 236 237 238 def main(options): 239 failures = 0 240 241 if not options.just_libs and not options.just_build_libs: 242 243 go_version = subprocess.check_output((['go', 'version'])) 244 color_print(go_version[:-1], "bright green") 245 246 if not options.quiet: 247 logging.basicConfig(format="%(message)s", level=logging.INFO) 248 249 failures += build_dependencies(options) 250 if failures: 251 return failures 252 253 failures += build_proxyfs(options) 254 if failures: 255 return failures 256 257 if platform.system() != "Darwin": 258 259 if not options.no_libs: 260 261 failures += build_jrpcclient(options) 262 if failures: 263 return failures 264 265 failures += build_vfs(options) 266 if failures: 267 return failures 268 269 return failures 270 271 272 if __name__ == "__main__": 273 arg_parser = argparse.ArgumentParser(description=__doc__) 274 arg_parser.add_argument('--bench', '-bench', 275 action='store_const', const='-bench=.', 276 help="include benchmark measurements in test output") 277 arg_parser.add_argument('--cover', '-cover', 278 action='store_const', const='-cover', 279 help="include coverage statistics in test output") 280 arg_parser.add_argument('--get', '-get', action='store_true', 281 help="invoke `go get` to retrieve new dependencies") 282 arg_parser.add_argument('--packages', '-p', action='store', nargs='*', 283 help="specific packages to process") 284 arg_parser.add_argument('--no-install', action='store_true', 285 help="When building C libraries, do not attempt " 286 "to install resulting objects") 287 arg_parser.add_argument('--deb-builder', action='store_true', 288 help="Modify commands to run inside " 289 "swift-deb-builder") 290 arg_parser.add_argument('--quiet', '-q', action='store_true', 291 help="suppress printing of what commands are being run") 292 libs_group = arg_parser.add_mutually_exclusive_group() 293 libs_group.add_argument('--no-libs', action='store_true', 294 help="don't build C libraries or run C tests") 295 libs_group.add_argument('--just-build-libs', action='store_true', 296 help="only build C libraries") 297 libs_group.add_argument('--just-libs', action='store_true', 298 help="only build C libraries and run C tests") 299 options = arg_parser.parse_args() 300 301 exit(main(options))