github.com/nextlinux/gosbom@v0.81.1-0.20230627115839-1ff50c281391/gosbom/pkg/cataloger/python/test-fixtures/setup/dynamic-setup.py (about) 1 # source: https://android.googlesource.com/platform/prebuilts/fullsdk-darwin/platform-tools/+/a7b08828aaa5d139fb3f1d6c8f5119212c94de70/systrace/catapult/telemetry/third_party/modulegraph/setup.py#808 2 """ 3 Shared setup file for simple python packages. Uses a setup.cfg that 4 is the same as the distutils2 project, unless noted otherwise. 5 It exists for two reasons: 6 1) This makes it easier to reuse setup.py code between my own 7 projects 8 2) Easier migration to distutils2 when that catches on. 9 Additional functionality: 10 * Section metadata: 11 requires-test: Same as 'tests_require' option for setuptools. 12 """ 13 import sys 14 import os 15 import re 16 import platform 17 from fnmatch import fnmatch 18 import os 19 import sys 20 import time 21 import tempfile 22 import tarfile 23 try: 24 import urllib.request as urllib 25 except ImportError: 26 import urllib 27 from distutils import log 28 try: 29 from hashlib import md5 30 except ImportError: 31 from md5 import md5 32 if sys.version_info[0] == 2: 33 from ConfigParser import RawConfigParser, NoOptionError, NoSectionError 34 else: 35 from configparser import RawConfigParser, NoOptionError, NoSectionError 36 ROOTDIR = os.path.dirname(os.path.abspath(__file__)) 37 # 38 # 39 # 40 # Parsing the setup.cfg and converting it to something that can be 41 # used by setuptools.setup() 42 # 43 # 44 # 45 def eval_marker(value): 46 """ 47 Evaluate an distutils2 environment marker. 48 This code is unsafe when used with hostile setup.cfg files, 49 but that's not a problem for our own files. 50 """ 51 value = value.strip() 52 class M: 53 def __init__(self, **kwds): 54 for k, v in kwds.items(): 55 setattr(self, k, v) 56 variables = { 57 'python_version': '%d.%d'%(sys.version_info[0], sys.version_info[1]), 58 'python_full_version': sys.version.split()[0], 59 'os': M( 60 name=os.name, 61 ), 62 'sys': M( 63 platform=sys.platform, 64 ), 65 'platform': M( 66 version=platform.version(), 67 machine=platform.machine(), 68 ), 69 } 70 return bool(eval(value, variables, variables)) 71 return True 72 def _opt_value(cfg, into, section, key, transform = None): 73 try: 74 v = cfg.get(section, key) 75 if transform != _as_lines and ';' in v: 76 v, marker = v.rsplit(';', 1) 77 if not eval_marker(marker): 78 return 79 v = v.strip() 80 if v: 81 if transform: 82 into[key] = transform(v.strip()) 83 else: 84 into[key] = v.strip() 85 except (NoOptionError, NoSectionError): 86 pass 87 def _as_bool(value): 88 if value.lower() in ('y', 'yes', 'on'): 89 return True 90 elif value.lower() in ('n', 'no', 'off'): 91 return False 92 elif value.isdigit(): 93 return bool(int(value)) 94 else: 95 raise ValueError(value) 96 def _as_list(value): 97 return value.split() 98 def _as_lines(value): 99 result = [] 100 for v in value.splitlines(): 101 if ';' in v: 102 v, marker = v.rsplit(';', 1) 103 if not eval_marker(marker): 104 continue 105 v = v.strip() 106 if v: 107 result.append(v) 108 else: 109 result.append(v) 110 return result 111 def _map_requirement(value): 112 m = re.search(r'(\S+)\s*(?:\((.*)\))?', value) 113 name = m.group(1) 114 version = m.group(2) 115 if version is None: 116 return name 117 else: 118 mapped = [] 119 for v in version.split(','): 120 v = v.strip() 121 if v[0].isdigit(): 122 # Checks for a specific version prefix 123 m = v.rsplit('.', 1) 124 mapped.append('>=%s,<%s.%s'%( 125 v, m[0], int(m[1])+1)) 126 else: 127 mapped.append(v) 128 return '%s %s'%(name, ','.join(mapped),) 129 def _as_requires(value): 130 requires = [] 131 for req in value.splitlines(): 132 if ';' in req: 133 req, marker = v.rsplit(';', 1) 134 if not eval_marker(marker): 135 continue 136 req = req.strip() 137 if not req: 138 continue 139 requires.append(_map_requirement(req)) 140 return requires 141 def parse_setup_cfg(): 142 cfg = RawConfigParser() 143 r = cfg.read([os.path.join(ROOTDIR, 'setup.cfg')]) 144 if len(r) != 1: 145 print("Cannot read 'setup.cfg'") 146 sys.exit(1) 147 metadata = dict( 148 name = cfg.get('metadata', 'name'), 149 version = cfg.get('metadata', 'version'), 150 description = cfg.get('metadata', 'description'), 151 ) 152 _opt_value(cfg, metadata, 'metadata', 'license') 153 _opt_value(cfg, metadata, 'metadata', 'maintainer') 154 _opt_value(cfg, metadata, 'metadata', 'maintainer_email') 155 _opt_value(cfg, metadata, 'metadata', 'author') 156 _opt_value(cfg, metadata, 'metadata', 'author_email') 157 _opt_value(cfg, metadata, 'metadata', 'url') 158 _opt_value(cfg, metadata, 'metadata', 'download_url') 159 _opt_value(cfg, metadata, 'metadata', 'classifiers', _as_lines) 160 _opt_value(cfg, metadata, 'metadata', 'platforms', _as_list) 161 _opt_value(cfg, metadata, 'metadata', 'packages', _as_list) 162 _opt_value(cfg, metadata, 'metadata', 'keywords', _as_list) 163 try: 164 v = cfg.get('metadata', 'requires-dist') 165 except (NoOptionError, NoSectionError): 166 pass 167 else: 168 requires = _as_requires(v) 169 if requires: 170 metadata['install_requires'] = requires 171 try: 172 v = cfg.get('metadata', 'requires-test') 173 except (NoOptionError, NoSectionError): 174 pass 175 else: 176 requires = _as_requires(v) 177 if requires: 178 metadata['tests_require'] = requires 179 try: 180 v = cfg.get('metadata', 'long_description_file') 181 except (NoOptionError, NoSectionError): 182 pass 183 else: 184 parts = [] 185 for nm in v.split(): 186 fp = open(nm, 'rU') 187 parts.append(fp.read()) 188 fp.close() 189 metadata['long_description'] = '\n\n'.join(parts) 190 try: 191 v = cfg.get('metadata', 'zip-safe') 192 except (NoOptionError, NoSectionError): 193 pass 194 else: 195 metadata['zip_safe'] = _as_bool(v) 196 try: 197 v = cfg.get('metadata', 'console_scripts') 198 except (NoOptionError, NoSectionError): 199 pass 200 else: 201 if 'entry_points' not in metadata: 202 metadata['entry_points'] = {} 203 metadata['entry_points']['console_scripts'] = v.splitlines() 204 if sys.version_info[:2] <= (2,6): 205 try: 206 metadata['tests_require'] += ", unittest2" 207 except KeyError: 208 metadata['tests_require'] = "unittest2" 209 return metadata 210 # 211 # 212 # 213 # Bootstrapping setuptools/distribute, based on 214 # a heavily modified version of distribute_setup.py 215 # 216 # 217 # 218 SETUPTOOLS_PACKAGE='setuptools' 219 try: 220 import subprocess 221 def _python_cmd(*args): 222 args = (sys.executable,) + args 223 return subprocess.call(args) == 0 224 except ImportError: 225 def _python_cmd(*args): 226 args = (sys.executable,) + args 227 new_args = [] 228 for a in args: 229 new_args.append(a.replace("'", "'\"'\"'")) 230 os.system(' '.join(new_args)) == 0 231 try: 232 import json 233 def get_pypi_src_download(package): 234 url = 'https://pypi.python.org/pypi/%s/json'%(package,) 235 fp = urllib.urlopen(url) 236 try: 237 try: 238 data = fp.read() 239 finally: 240 fp.close() 241 except urllib.error: 242 raise RuntimeError("Cannot determine download link for %s"%(package,)) 243 pkgdata = json.loads(data.decode('utf-8')) 244 if 'urls' not in pkgdata: 245 raise RuntimeError("Cannot determine download link for %s"%(package,)) 246 for info in pkgdata['urls']: 247 if info['packagetype'] == 'sdist' and info['url'].endswith('tar.gz'): 248 return (info.get('md5_digest'), info['url']) 249 raise RuntimeError("Cannot determine downlink link for %s"%(package,)) 250 except ImportError: 251 # Python 2.5 compatibility, no JSON in stdlib but luckily JSON syntax is 252 # simular enough to Python's syntax to be able to abuse the Python compiler 253 import _ast as ast 254 def get_pypi_src_download(package): 255 url = 'https://pypi.python.org/pypi/%s/json'%(package,) 256 fp = urllib.urlopen(url) 257 try: 258 try: 259 data = fp.read() 260 finally: 261 fp.close() 262 except urllib.error: 263 raise RuntimeError("Cannot determine download link for %s"%(package,)) 264 a = compile(data, '-', 'eval', ast.PyCF_ONLY_AST) 265 if not isinstance(a, ast.Expression): 266 raise RuntimeError("Cannot determine download link for %s"%(package,)) 267 a = a.body 268 if not isinstance(a, ast.Dict): 269 raise RuntimeError("Cannot determine download link for %s"%(package,)) 270 for k, v in zip(a.keys, a.values): 271 if not isinstance(k, ast.Str): 272 raise RuntimeError("Cannot determine download link for %s"%(package,)) 273 k = k.s 274 if k == 'urls': 275 a = v 276 break 277 else: 278 raise RuntimeError("PyPI JSON for %s doesn't contain URLs section"%(package,)) 279 if not isinstance(a, ast.List): 280 raise RuntimeError("Cannot determine download link for %s"%(package,)) 281 for info in v.elts: 282 if not isinstance(info, ast.Dict): 283 raise RuntimeError("Cannot determine download link for %s"%(package,)) 284 url = None 285 packagetype = None 286 chksum = None 287 for k, v in zip(info.keys, info.values): 288 if not isinstance(k, ast.Str): 289 raise RuntimeError("Cannot determine download link for %s"%(package,)) 290 if k.s == 'url': 291 if not isinstance(v, ast.Str): 292 raise RuntimeError("Cannot determine download link for %s"%(package,)) 293 url = v.s 294 elif k.s == 'packagetype': 295 if not isinstance(v, ast.Str): 296 raise RuntimeError("Cannot determine download link for %s"%(package,)) 297 packagetype = v.s 298 elif k.s == 'md5_digest': 299 if not isinstance(v, ast.Str): 300 raise RuntimeError("Cannot determine download link for %s"%(package,)) 301 chksum = v.s 302 if url is not None and packagetype == 'sdist' and url.endswith('.tar.gz'): 303 return (chksum, url) 304 raise RuntimeError("Cannot determine download link for %s"%(package,)) 305 def _build_egg(egg, tarball, to_dir): 306 # extracting the tarball 307 tmpdir = tempfile.mkdtemp() 308 log.warn('Extracting in %s', tmpdir) 309 old_wd = os.getcwd() 310 try: 311 os.chdir(tmpdir) 312 tar = tarfile.open(tarball) 313 _extractall(tar) 314 tar.close() 315 # going in the directory 316 subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) 317 os.chdir(subdir) 318 log.warn('Now working in %s', subdir) 319 # building an egg 320 log.warn('Building a %s egg in %s', egg, to_dir) 321 _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) 322 finally: 323 os.chdir(old_wd) 324 # returning the result 325 log.warn(egg) 326 if not os.path.exists(egg): 327 raise IOError('Could not build the egg.') 328 def _do_download(to_dir, packagename=SETUPTOOLS_PACKAGE): 329 tarball = download_setuptools(packagename, to_dir) 330 version = tarball.split('-')[-1][:-7] 331 egg = os.path.join(to_dir, '%s-%s-py%d.%d.egg' 332 % (packagename, version, sys.version_info[0], sys.version_info[1])) 333 if not os.path.exists(egg): 334 _build_egg(egg, tarball, to_dir) 335 sys.path.insert(0, egg) 336 import setuptools 337 setuptools.bootstrap_install_from = egg 338 def use_setuptools(): 339 # making sure we use the absolute path 340 return _do_download(os.path.abspath(os.curdir)) 341 def download_setuptools(packagename, to_dir): 342 # making sure we use the absolute path 343 to_dir = os.path.abspath(to_dir) 344 try: 345 from urllib.request import urlopen 346 except ImportError: 347 from urllib2 import urlopen 348 chksum, url = get_pypi_src_download(packagename) 349 tgz_name = os.path.basename(url) 350 saveto = os.path.join(to_dir, tgz_name) 351 src = dst = None 352 if not os.path.exists(saveto): # Avoid repeated downloads 353 try: 354 log.warn("Downloading %s", url) 355 src = urlopen(url) 356 # Read/write all in one block, so we don't create a corrupt file 357 # if the download is interrupted. 358 data = src.read() 359 if chksum is not None: 360 data_sum = md5(data).hexdigest() 361 if data_sum != chksum: 362 raise RuntimeError("Downloading %s failed: corrupt checksum"%(url,)) 363 dst = open(saveto, "wb") 364 dst.write(data) 365 finally: 366 if src: 367 src.close() 368 if dst: 369 dst.close() 370 return os.path.realpath(saveto) 371 def _extractall(self, path=".", members=None): 372 """Extract all members from the archive to the current working 373 directory and set owner, modification time and permissions on 374 directories afterwards. `path' specifies a different directory 375 to extract to. `members' is optional and must be a subset of the 376 list returned by getmembers(). 377 """ 378 import copy 379 import operator 380 from tarfile import ExtractError 381 directories = [] 382 if members is None: 383 members = self 384 for tarinfo in members: 385 if tarinfo.isdir(): 386 # Extract directories with a safe mode. 387 directories.append(tarinfo) 388 tarinfo = copy.copy(tarinfo) 389 tarinfo.mode = 448 # decimal for oct 0700 390 self.extract(tarinfo, path) 391 # Reverse sort directories. 392 if sys.version_info < (2, 4): 393 def sorter(dir1, dir2): 394 return cmp(dir1.name, dir2.name) 395 directories.sort(sorter) 396 directories.reverse() 397 else: 398 directories.sort(key=operator.attrgetter('name'), reverse=True) 399 # Set correct owner, mtime and filemode on directories. 400 for tarinfo in directories: 401 dirpath = os.path.join(path, tarinfo.name) 402 try: 403 self.chown(tarinfo, dirpath) 404 self.utime(tarinfo, dirpath) 405 self.chmod(tarinfo, dirpath) 406 except ExtractError: 407 e = sys.exc_info()[1] 408 if self.errorlevel > 1: 409 raise 410 else: 411 self._dbg(1, "tarfile: %s" % e) 412 # 413 # 414 # 415 # Definitions of custom commands 416 # 417 # 418 # 419 try: 420 import setuptools 421 except ImportError: 422 use_setuptools() 423 from setuptools import setup 424 try: 425 from distutils.core import PyPIRCCommand 426 except ImportError: 427 PyPIRCCommand = None # Ancient python version 428 from distutils.core import Command 429 from distutils.errors import DistutilsError 430 from distutils import log 431 if PyPIRCCommand is None: 432 class upload_docs (Command): 433 description = "upload sphinx documentation" 434 user_options = [] 435 def initialize_options(self): 436 pass 437 def finalize_options(self): 438 pass 439 def run(self): 440 raise DistutilsError("not supported on this version of python") 441 else: 442 class upload_docs (PyPIRCCommand): 443 description = "upload sphinx documentation" 444 user_options = PyPIRCCommand.user_options 445 def initialize_options(self): 446 PyPIRCCommand.initialize_options(self) 447 self.username = '' 448 self.password = '' 449 def finalize_options(self): 450 PyPIRCCommand.finalize_options(self) 451 config = self._read_pypirc() 452 if config != {}: 453 self.username = config['username'] 454 self.password = config['password'] 455 def run(self): 456 import subprocess 457 import shutil 458 import zipfile 459 import os 460 import urllib 461 import StringIO 462 from base64 import standard_b64encode 463 import httplib 464 import urlparse 465 # Extract the package name from distutils metadata 466 meta = self.distribution.metadata 467 name = meta.get_name() 468 # Run sphinx 469 if os.path.exists('doc/_build'): 470 shutil.rmtree('doc/_build') 471 os.mkdir('doc/_build') 472 p = subprocess.Popen(['make', 'html'], 473 cwd='doc') 474 exit = p.wait() 475 if exit != 0: 476 raise DistutilsError("sphinx-build failed") 477 # Collect sphinx output 478 if not os.path.exists('dist'): 479 os.mkdir('dist') 480 zf = zipfile.ZipFile('dist/%s-docs.zip'%(name,), 'w', 481 compression=zipfile.ZIP_DEFLATED) 482 for toplevel, dirs, files in os.walk('doc/_build/html'): 483 for fn in files: 484 fullname = os.path.join(toplevel, fn) 485 relname = os.path.relpath(fullname, 'doc/_build/html') 486 print ("%s -> %s"%(fullname, relname)) 487 zf.write(fullname, relname) 488 zf.close() 489 # Upload the results, this code is based on the distutils 490 # 'upload' command. 491 content = open('dist/%s-docs.zip'%(name,), 'rb').read() 492 data = { 493 ':action': 'doc_upload', 494 'name': name, 495 'content': ('%s-docs.zip'%(name,), content), 496 } 497 auth = "Basic " + standard_b64encode(self.username + ":" + 498 self.password) 499 boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' 500 sep_boundary = '\n--' + boundary 501 end_boundary = sep_boundary + '--' 502 body = StringIO.StringIO() 503 for key, value in data.items(): 504 if not isinstance(value, list): 505 value = [value] 506 for value in value: 507 if isinstance(value, tuple): 508 fn = ';filename="%s"'%(value[0]) 509 value = value[1] 510 else: 511 fn = '' 512 body.write(sep_boundary) 513 body.write('\nContent-Disposition: form-data; name="%s"'%key) 514 body.write(fn) 515 body.write("\n\n") 516 body.write(value) 517 body.write(end_boundary) 518 body.write('\n') 519 body = body.getvalue() 520 self.announce("Uploading documentation to %s"%(self.repository,), log.INFO) 521 schema, netloc, url, params, query, fragments = \ 522 urlparse.urlparse(self.repository) 523 if schema == 'http': 524 http = httplib.HTTPConnection(netloc) 525 elif schema == 'https': 526 http = httplib.HTTPSConnection(netloc) 527 else: 528 raise AssertionError("unsupported schema "+schema) 529 data = '' 530 loglevel = log.INFO 531 try: 532 http.connect() 533 http.putrequest("POST", url) 534 http.putheader('Content-type', 535 'multipart/form-data; boundary=%s'%boundary) 536 http.putheader('Content-length', str(len(body))) 537 http.putheader('Authorization', auth) 538 http.endheaders() 539 http.send(body) 540 except socket.error: 541 e = socket.exc_info()[1] 542 self.announce(str(e), log.ERROR) 543 return 544 r = http.getresponse() 545 if r.status in (200, 301): 546 self.announce('Upload succeeded (%s): %s' % (r.status, r.reason), 547 log.INFO) 548 else: 549 self.announce('Upload failed (%s): %s' % (r.status, r.reason), 550 log.ERROR) 551 print ('-'*75) 552 print (r.read()) 553 print ('-'*75) 554 def recursiveGlob(root, pathPattern): 555 """ 556 Recursively look for files matching 'pathPattern'. Return a list 557 of matching files/directories. 558 """ 559 result = [] 560 for rootpath, dirnames, filenames in os.walk(root): 561 for fn in filenames: 562 if fnmatch(fn, pathPattern): 563 result.append(os.path.join(rootpath, fn)) 564 return result 565 def importExternalTestCases(unittest, 566 pathPattern="test_*.py", root=".", package=None): 567 """ 568 Import all unittests in the PyObjC tree starting at 'root' 569 """ 570 testFiles = recursiveGlob(root, pathPattern) 571 testModules = map(lambda x:x[len(root)+1:-3].replace('/', '.'), testFiles) 572 if package is not None: 573 testModules = [(package + '.' + m) for m in testModules] 574 suites = [] 575 for modName in testModules: 576 try: 577 module = __import__(modName) 578 except ImportError: 579 print("SKIP %s: %s"%(modName, sys.exc_info()[1])) 580 continue 581 if '.' in modName: 582 for elem in modName.split('.')[1:]: 583 module = getattr(module, elem) 584 s = unittest.defaultTestLoader.loadTestsFromModule(module) 585 suites.append(s) 586 return unittest.TestSuite(suites) 587 class test (Command): 588 description = "run test suite" 589 user_options = [ 590 ('verbosity=', None, "print what tests are run"), 591 ] 592 def initialize_options(self): 593 self.verbosity='1' 594 def finalize_options(self): 595 if isinstance(self.verbosity, str): 596 self.verbosity = int(self.verbosity) 597 def cleanup_environment(self): 598 ei_cmd = self.get_finalized_command('egg_info') 599 egg_name = ei_cmd.egg_name.replace('-', '_') 600 to_remove = [] 601 for dirname in sys.path: 602 bn = os.path.basename(dirname) 603 if bn.startswith(egg_name + "-"): 604 to_remove.append(dirname) 605 for dirname in to_remove: 606 log.info("removing installed %r from sys.path before testing"%( 607 dirname,)) 608 sys.path.remove(dirname) 609 def add_project_to_sys_path(self): 610 from pkg_resources import normalize_path, add_activation_listener 611 from pkg_resources import working_set, require 612 self.reinitialize_command('egg_info') 613 self.run_command('egg_info') 614 self.reinitialize_command('build_ext', inplace=1) 615 self.run_command('build_ext') 616 # Check if this distribution is already on sys.path 617 # and remove that version, this ensures that the right 618 # copy of the package gets tested. 619 self.__old_path = sys.path[:] 620 self.__old_modules = sys.modules.copy() 621 ei_cmd = self.get_finalized_command('egg_info') 622 sys.path.insert(0, normalize_path(ei_cmd.egg_base)) 623 sys.path.insert(1, os.path.dirname(__file__)) 624 # Strip the namespace packages defined in this distribution 625 # from sys.modules, needed to reset the search path for 626 # those modules. 627 nspkgs = getattr(self.distribution, 'namespace_packages') 628 if nspkgs is not None: 629 for nm in nspkgs: 630 del sys.modules[nm] 631 # Reset pkg_resources state: 632 add_activation_listener(lambda dist: dist.activate()) 633 working_set.__init__() 634 require('%s==%s'%(ei_cmd.egg_name, ei_cmd.egg_version)) 635 def remove_from_sys_path(self): 636 from pkg_resources import working_set 637 sys.path[:] = self.__old_path 638 sys.modules.clear() 639 sys.modules.update(self.__old_modules) 640 working_set.__init__() 641 def run(self): 642 import unittest 643 # Ensure that build directory is on sys.path (py3k) 644 self.cleanup_environment() 645 self.add_project_to_sys_path() 646 try: 647 meta = self.distribution.metadata 648 name = meta.get_name() 649 test_pkg = name + "_tests" 650 suite = importExternalTestCases(unittest, 651 "test_*.py", test_pkg, test_pkg) 652 runner = unittest.TextTestRunner(verbosity=self.verbosity) 653 result = runner.run(suite) 654 # Print out summary. This is a structured format that 655 # should make it easy to use this information in scripts. 656 summary = dict( 657 count=result.testsRun, 658 fails=len(result.failures), 659 errors=len(result.errors), 660 xfails=len(getattr(result, 'expectedFailures', [])), 661 xpass=len(getattr(result, 'expectedSuccesses', [])), 662 skip=len(getattr(result, 'skipped', [])), 663 ) 664 print("SUMMARY: %s"%(summary,)) 665 finally: 666 self.remove_from_sys_path() 667 # 668 # 669 # 670 # And finally run the setuptools main entry point. 671 # 672 # 673 # 674 metadata = parse_setup_cfg() 675 setup( 676 cmdclass=dict( 677 upload_docs=upload_docs, 678 test=test, 679 ), 680 **metadata 681 )