github.com/johnnyeven/libtools@v0.0.0-20191126065708-61829c1adf46/third_party/py/python_configure.bzl (about) 1 """Repository rule for Python autoconfiguration. 2 3 `python_configure` depends on the following environment variables: 4 5 * `PYTHON_BIN_PATH`: location of python binary. 6 * `PYTHON_LIB_PATH`: Location of python libraries. 7 """ 8 9 _BAZEL_SH = "BAZEL_SH" 10 _PYTHON_BIN_PATH = "PYTHON_BIN_PATH" 11 _PYTHON_LIB_PATH = "PYTHON_LIB_PATH" 12 _TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO" 13 14 def _tpl(repository_ctx, tpl, substitutions = {}, out = None): 15 if not out: 16 out = tpl 17 repository_ctx.template( 18 out, 19 Label("//third_party/py:%s.tpl" % tpl), 20 substitutions, 21 ) 22 23 def _fail(msg): 24 """Output failure message when auto configuration fails.""" 25 red = "\033[0;31m" 26 no_color = "\033[0m" 27 fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) 28 29 def _is_windows(repository_ctx): 30 """Returns true if the host operating system is windows.""" 31 os_name = repository_ctx.os.name.lower() 32 if os_name.find("windows") != -1: 33 return True 34 return False 35 36 def _execute( 37 repository_ctx, 38 cmdline, 39 error_msg = None, 40 error_details = None, 41 empty_stdout_fine = False): 42 """Executes an arbitrary shell command. 43 44 Args: 45 repository_ctx: the repository_ctx object 46 cmdline: list of strings, the command to execute 47 error_msg: string, a summary of the error if the command fails 48 error_details: string, details about the error or steps to fix it 49 empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise 50 it's an error 51 Return: 52 the result of repository_ctx.execute(cmdline) 53 """ 54 result = repository_ctx.execute(cmdline) 55 if result.stderr or not (empty_stdout_fine or result.stdout): 56 _fail("\n".join([ 57 error_msg.strip() if error_msg else "Repository command failed", 58 result.stderr.strip(), 59 error_details if error_details else "", 60 ])) 61 return result 62 63 def _read_dir(repository_ctx, src_dir): 64 """Returns a string with all files in a directory. 65 66 Finds all files inside a directory, traversing subfolders and following 67 symlinks. The returned string contains the full path of all files 68 separated by line breaks. 69 """ 70 if _is_windows(repository_ctx): 71 src_dir = src_dir.replace("/", "\\") 72 find_result = _execute( 73 repository_ctx, 74 ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], 75 empty_stdout_fine = True, 76 ) 77 78 # src_files will be used in genrule.outs where the paths must 79 # use forward slashes. 80 result = find_result.stdout.replace("\\", "/") 81 else: 82 find_result = _execute( 83 repository_ctx, 84 ["find", src_dir, "-follow", "-type", "f"], 85 empty_stdout_fine = True, 86 ) 87 result = find_result.stdout 88 return result 89 90 def _genrule(src_dir, genrule_name, command, outs): 91 """Returns a string with a genrule. 92 93 Genrule executes the given command and produces the given outputs. 94 """ 95 return ( 96 "genrule(\n" + 97 ' name = "' + 98 genrule_name + '",\n' + 99 " outs = [\n" + 100 outs + 101 "\n ],\n" + 102 ' cmd = """\n' + 103 command + 104 '\n """,\n' + 105 ")\n" 106 ) 107 108 def _norm_path(path): 109 """Returns a path with '/' and remove the trailing slash.""" 110 path = path.replace("\\", "/") 111 if path[-1] == "/": 112 path = path[:-1] 113 return path 114 115 def _symlink_genrule_for_dir( 116 repository_ctx, 117 src_dir, 118 dest_dir, 119 genrule_name, 120 src_files = [], 121 dest_files = []): 122 """Returns a genrule to symlink(or copy if on Windows) a set of files. 123 124 If src_dir is passed, files will be read from the given directory; otherwise 125 we assume files are in src_files and dest_files 126 """ 127 if src_dir != None: 128 src_dir = _norm_path(src_dir) 129 dest_dir = _norm_path(dest_dir) 130 files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines())) 131 132 # Create a list with the src_dir stripped to use for outputs. 133 dest_files = files.replace(src_dir, "").splitlines() 134 src_files = files.splitlines() 135 command = [] 136 outs = [] 137 for i in range(len(dest_files)): 138 if dest_files[i] != "": 139 # If we have only one file to link we do not want to use the dest_dir, as 140 # $(@D) will include the full path to the file. 141 dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i] 142 143 # Copy the headers to create a sandboxable setup. 144 cmd = "cp -f" 145 command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) 146 outs.append(' "' + dest_dir + dest_files[i] + '",') 147 genrule = _genrule( 148 src_dir, 149 genrule_name, 150 " && ".join(command), 151 "\n".join(outs), 152 ) 153 return genrule 154 155 def _get_python_bin(repository_ctx): 156 """Gets the python bin path.""" 157 python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH) 158 if python_bin != None: 159 return python_bin 160 python_bin_path = repository_ctx.which("python") 161 if python_bin_path != None: 162 return str(python_bin_path) 163 _fail("Cannot find python in PATH, please make sure " + 164 "python is installed and add its directory in PATH, or --define " + 165 "%s='/something/else'.\nPATH=%s" % ( 166 _PYTHON_BIN_PATH, 167 repository_ctx.os.environ.get("PATH", ""), 168 )) 169 170 def _get_bash_bin(repository_ctx): 171 """Gets the bash bin path.""" 172 bash_bin = repository_ctx.os.environ.get(_BAZEL_SH) 173 if bash_bin != None: 174 return bash_bin 175 else: 176 bash_bin_path = repository_ctx.which("bash") 177 if bash_bin_path != None: 178 return str(bash_bin_path) 179 else: 180 _fail("Cannot find bash in PATH, please make sure " + 181 "bash is installed and add its directory in PATH, or --define " + 182 "%s='/path/to/bash'.\nPATH=%s" % ( 183 _BAZEL_SH, 184 repository_ctx.os.environ.get("PATH", ""), 185 )) 186 187 def _get_python_lib(repository_ctx, python_bin): 188 """Gets the python lib path.""" 189 python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH) 190 if python_lib != None: 191 return python_lib 192 print_lib = ("<<END\n" + 193 "from __future__ import print_function\n" + 194 "import site\n" + 195 "import os\n" + 196 "\n" + 197 "try:\n" + 198 " input = raw_input\n" + 199 "except NameError:\n" + 200 " pass\n" + 201 "\n" + 202 "python_paths = []\n" + 203 "if os.getenv('PYTHONPATH') is not None:\n" + 204 " python_paths = os.getenv('PYTHONPATH').split(':')\n" + 205 "try:\n" + 206 " library_paths = site.getsitepackages()\n" + 207 "except AttributeError:\n" + 208 " from distutils.sysconfig import get_python_lib\n" + 209 " library_paths = [get_python_lib()]\n" + 210 "all_paths = set(python_paths + library_paths)\n" + 211 "paths = []\n" + 212 "for path in all_paths:\n" + 213 " if os.path.isdir(path):\n" + 214 " paths.append(path)\n" + 215 "if len(paths) >=1:\n" + 216 " print(paths[0])\n" + 217 "END") 218 cmd = "%s - %s" % (python_bin, print_lib) 219 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 220 return result.stdout.strip("\n") 221 222 def _check_python_lib(repository_ctx, python_lib): 223 """Checks the python lib path.""" 224 cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib) 225 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 226 if result.return_code == 1: 227 _fail("Invalid python library path: %s" % python_lib) 228 229 def _check_python_bin(repository_ctx, python_bin): 230 """Checks the python bin path.""" 231 cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin) 232 result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd]) 233 if result.return_code == 1: 234 _fail("--define %s='%s' is not executable. Is it the python binary?" % ( 235 _PYTHON_BIN_PATH, 236 python_bin, 237 )) 238 239 def _get_python_include(repository_ctx, python_bin): 240 """Gets the python include path.""" 241 result = _execute( 242 repository_ctx, 243 [ 244 python_bin, 245 "-c", 246 "from __future__ import print_function;" + 247 "from distutils import sysconfig;" + 248 "print(sysconfig.get_python_inc())", 249 ], 250 error_msg = "Problem getting python include path.", 251 error_details = ("Is the Python binary path set up right? " + 252 "(See ./configure or " + _PYTHON_BIN_PATH + ".) " + 253 "Is distutils installed?"), 254 ) 255 return result.stdout.splitlines()[0] 256 257 def _get_python_import_lib_name(repository_ctx, python_bin): 258 """Get Python import library name (pythonXY.lib) on Windows.""" 259 result = _execute( 260 repository_ctx, 261 [ 262 python_bin, 263 "-c", 264 "import sys;" + 265 'print("python" + str(sys.version_info[0]) + ' + 266 ' str(sys.version_info[1]) + ".lib")', 267 ], 268 error_msg = "Problem getting python import library.", 269 error_details = ("Is the Python binary path set up right? " + 270 "(See ./configure or " + _PYTHON_BIN_PATH + ".) "), 271 ) 272 return result.stdout.splitlines()[0] 273 274 def _get_numpy_include(repository_ctx, python_bin): 275 """Gets the numpy include path.""" 276 return _execute( 277 repository_ctx, 278 [ 279 python_bin, 280 "-c", 281 "from __future__ import print_function;" + 282 "import numpy;" + 283 " print(numpy.get_include());", 284 ], 285 error_msg = "Problem getting numpy include path.", 286 error_details = "Is numpy installed?", 287 ).stdout.splitlines()[0] 288 289 def _create_local_python_repository(repository_ctx): 290 """Creates the repository containing files set up to build with Python.""" 291 python_bin = _get_python_bin(repository_ctx) 292 _check_python_bin(repository_ctx, python_bin) 293 python_lib = _get_python_lib(repository_ctx, python_bin) 294 _check_python_lib(repository_ctx, python_lib) 295 python_include = _get_python_include(repository_ctx, python_bin) 296 numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy" 297 python_include_rule = _symlink_genrule_for_dir( 298 repository_ctx, 299 python_include, 300 "python_include", 301 "python_include", 302 ) 303 python_import_lib_genrule = "" 304 305 # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib 306 # See https://docs.python.org/3/extending/windows.html 307 if _is_windows(repository_ctx): 308 python_include = _norm_path(python_include) 309 python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin) 310 python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name 311 python_import_lib_genrule = _symlink_genrule_for_dir( 312 repository_ctx, 313 None, 314 "", 315 "python_import_lib", 316 [python_import_lib_src], 317 [python_import_lib_name], 318 ) 319 numpy_include_rule = _symlink_genrule_for_dir( 320 repository_ctx, 321 numpy_include, 322 "numpy_include/numpy", 323 "numpy_include", 324 ) 325 _tpl(repository_ctx, "BUILD", { 326 "%{PYTHON_INCLUDE_GENRULE}": python_include_rule, 327 "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule, 328 "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, 329 }) 330 331 def _create_remote_python_repository(repository_ctx, remote_config_repo): 332 """Creates pointers to a remotely configured repo set up to build with Python. 333 """ 334 repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {}) 335 336 def _python_autoconf_impl(repository_ctx): 337 """Implementation of the python_autoconf repository rule.""" 338 if _TF_PYTHON_CONFIG_REPO in repository_ctx.os.environ: 339 _create_remote_python_repository( 340 repository_ctx, 341 repository_ctx.os.environ[_TF_PYTHON_CONFIG_REPO], 342 ) 343 else: 344 _create_local_python_repository(repository_ctx) 345 346 python_configure = repository_rule( 347 implementation = _python_autoconf_impl, 348 environ = [ 349 _BAZEL_SH, 350 _PYTHON_BIN_PATH, 351 _PYTHON_LIB_PATH, 352 _TF_PYTHON_CONFIG_REPO, 353 ], 354 ) 355 """Detects and configures the local Python. 356 357 Add the following to your WORKSPACE FILE: 358 359 ```python 360 python_configure(name = "local_config_python") 361 ``` 362 363 Args: 364 name: A unique name for this workspace rule. 365 """