github.com/johnnyeven/libtools@v0.0.0-20191126065708-61829c1adf46/third_party/py/python_configure.bzl (about)

     1  """Repository rule for Python autoconfiguration.
     2  
     3  `python_configure` depends on the following environment variables:
     4  
     5    * `PYTHON_BIN_PATH`: location of python binary.
     6    * `PYTHON_LIB_PATH`: Location of python libraries.
     7  """
     8  
     9  _BAZEL_SH = "BAZEL_SH"
    10  _PYTHON_BIN_PATH = "PYTHON_BIN_PATH"
    11  _PYTHON_LIB_PATH = "PYTHON_LIB_PATH"
    12  _TF_PYTHON_CONFIG_REPO = "TF_PYTHON_CONFIG_REPO"
    13  
    14  def _tpl(repository_ctx, tpl, substitutions = {}, out = None):
    15      if not out:
    16          out = tpl
    17      repository_ctx.template(
    18          out,
    19          Label("//third_party/py:%s.tpl" % tpl),
    20          substitutions,
    21      )
    22  
    23  def _fail(msg):
    24      """Output failure message when auto configuration fails."""
    25      red = "\033[0;31m"
    26      no_color = "\033[0m"
    27      fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg))
    28  
    29  def _is_windows(repository_ctx):
    30      """Returns true if the host operating system is windows."""
    31      os_name = repository_ctx.os.name.lower()
    32      if os_name.find("windows") != -1:
    33          return True
    34      return False
    35  
    36  def _execute(
    37          repository_ctx,
    38          cmdline,
    39          error_msg = None,
    40          error_details = None,
    41          empty_stdout_fine = False):
    42      """Executes an arbitrary shell command.
    43  
    44      Args:
    45        repository_ctx: the repository_ctx object
    46        cmdline: list of strings, the command to execute
    47        error_msg: string, a summary of the error if the command fails
    48        error_details: string, details about the error or steps to fix it
    49        empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise
    50          it's an error
    51      Return:
    52        the result of repository_ctx.execute(cmdline)
    53      """
    54      result = repository_ctx.execute(cmdline)
    55      if result.stderr or not (empty_stdout_fine or result.stdout):
    56          _fail("\n".join([
    57              error_msg.strip() if error_msg else "Repository command failed",
    58              result.stderr.strip(),
    59              error_details if error_details else "",
    60          ]))
    61      return result
    62  
    63  def _read_dir(repository_ctx, src_dir):
    64      """Returns a string with all files in a directory.
    65  
    66      Finds all files inside a directory, traversing subfolders and following
    67      symlinks. The returned string contains the full path of all files
    68      separated by line breaks.
    69      """
    70      if _is_windows(repository_ctx):
    71          src_dir = src_dir.replace("/", "\\")
    72          find_result = _execute(
    73              repository_ctx,
    74              ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"],
    75              empty_stdout_fine = True,
    76          )
    77  
    78          # src_files will be used in genrule.outs where the paths must
    79          # use forward slashes.
    80          result = find_result.stdout.replace("\\", "/")
    81      else:
    82          find_result = _execute(
    83              repository_ctx,
    84              ["find", src_dir, "-follow", "-type", "f"],
    85              empty_stdout_fine = True,
    86          )
    87          result = find_result.stdout
    88      return result
    89  
    90  def _genrule(src_dir, genrule_name, command, outs):
    91      """Returns a string with a genrule.
    92  
    93      Genrule executes the given command and produces the given outputs.
    94      """
    95      return (
    96          "genrule(\n" +
    97          '    name = "' +
    98          genrule_name + '",\n' +
    99          "    outs = [\n" +
   100          outs +
   101          "\n    ],\n" +
   102          '    cmd = """\n' +
   103          command +
   104          '\n   """,\n' +
   105          ")\n"
   106      )
   107  
   108  def _norm_path(path):
   109      """Returns a path with '/' and remove the trailing slash."""
   110      path = path.replace("\\", "/")
   111      if path[-1] == "/":
   112          path = path[:-1]
   113      return path
   114  
   115  def _symlink_genrule_for_dir(
   116          repository_ctx,
   117          src_dir,
   118          dest_dir,
   119          genrule_name,
   120          src_files = [],
   121          dest_files = []):
   122      """Returns a genrule to symlink(or copy if on Windows) a set of files.
   123  
   124      If src_dir is passed, files will be read from the given directory; otherwise
   125      we assume files are in src_files and dest_files
   126      """
   127      if src_dir != None:
   128          src_dir = _norm_path(src_dir)
   129          dest_dir = _norm_path(dest_dir)
   130          files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines()))
   131  
   132          # Create a list with the src_dir stripped to use for outputs.
   133          dest_files = files.replace(src_dir, "").splitlines()
   134          src_files = files.splitlines()
   135      command = []
   136      outs = []
   137      for i in range(len(dest_files)):
   138          if dest_files[i] != "":
   139              # If we have only one file to link we do not want to use the dest_dir, as
   140              # $(@D) will include the full path to the file.
   141              dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i]
   142  
   143              # Copy the headers to create a sandboxable setup.
   144              cmd = "cp -f"
   145              command.append(cmd + ' "%s" "%s"' % (src_files[i], dest))
   146              outs.append('        "' + dest_dir + dest_files[i] + '",')
   147      genrule = _genrule(
   148          src_dir,
   149          genrule_name,
   150          " && ".join(command),
   151          "\n".join(outs),
   152      )
   153      return genrule
   154  
   155  def _get_python_bin(repository_ctx):
   156      """Gets the python bin path."""
   157      python_bin = repository_ctx.os.environ.get(_PYTHON_BIN_PATH)
   158      if python_bin != None:
   159          return python_bin
   160      python_bin_path = repository_ctx.which("python")
   161      if python_bin_path != None:
   162          return str(python_bin_path)
   163      _fail("Cannot find python in PATH, please make sure " +
   164            "python is installed and add its directory in PATH, or --define " +
   165            "%s='/something/else'.\nPATH=%s" % (
   166                _PYTHON_BIN_PATH,
   167                repository_ctx.os.environ.get("PATH", ""),
   168            ))
   169  
   170  def _get_bash_bin(repository_ctx):
   171      """Gets the bash bin path."""
   172      bash_bin = repository_ctx.os.environ.get(_BAZEL_SH)
   173      if bash_bin != None:
   174          return bash_bin
   175      else:
   176          bash_bin_path = repository_ctx.which("bash")
   177          if bash_bin_path != None:
   178              return str(bash_bin_path)
   179          else:
   180              _fail("Cannot find bash in PATH, please make sure " +
   181                    "bash is installed and add its directory in PATH, or --define " +
   182                    "%s='/path/to/bash'.\nPATH=%s" % (
   183                        _BAZEL_SH,
   184                        repository_ctx.os.environ.get("PATH", ""),
   185                    ))
   186  
   187  def _get_python_lib(repository_ctx, python_bin):
   188      """Gets the python lib path."""
   189      python_lib = repository_ctx.os.environ.get(_PYTHON_LIB_PATH)
   190      if python_lib != None:
   191          return python_lib
   192      print_lib = ("<<END\n" +
   193                   "from __future__ import print_function\n" +
   194                   "import site\n" +
   195                   "import os\n" +
   196                   "\n" +
   197                   "try:\n" +
   198                   "  input = raw_input\n" +
   199                   "except NameError:\n" +
   200                   "  pass\n" +
   201                   "\n" +
   202                   "python_paths = []\n" +
   203                   "if os.getenv('PYTHONPATH') is not None:\n" +
   204                   "  python_paths = os.getenv('PYTHONPATH').split(':')\n" +
   205                   "try:\n" +
   206                   "  library_paths = site.getsitepackages()\n" +
   207                   "except AttributeError:\n" +
   208                   " from distutils.sysconfig import get_python_lib\n" +
   209                   " library_paths = [get_python_lib()]\n" +
   210                   "all_paths = set(python_paths + library_paths)\n" +
   211                   "paths = []\n" +
   212                   "for path in all_paths:\n" +
   213                   "  if os.path.isdir(path):\n" +
   214                   "    paths.append(path)\n" +
   215                   "if len(paths) >=1:\n" +
   216                   "  print(paths[0])\n" +
   217                   "END")
   218      cmd = "%s - %s" % (python_bin, print_lib)
   219      result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
   220      return result.stdout.strip("\n")
   221  
   222  def _check_python_lib(repository_ctx, python_lib):
   223      """Checks the python lib path."""
   224      cmd = 'test -d "%s" -a -x "%s"' % (python_lib, python_lib)
   225      result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
   226      if result.return_code == 1:
   227          _fail("Invalid python library path: %s" % python_lib)
   228  
   229  def _check_python_bin(repository_ctx, python_bin):
   230      """Checks the python bin path."""
   231      cmd = '[[ -x "%s" ]] && [[ ! -d "%s" ]]' % (python_bin, python_bin)
   232      result = repository_ctx.execute([_get_bash_bin(repository_ctx), "-c", cmd])
   233      if result.return_code == 1:
   234          _fail("--define %s='%s' is not executable. Is it the python binary?" % (
   235              _PYTHON_BIN_PATH,
   236              python_bin,
   237          ))
   238  
   239  def _get_python_include(repository_ctx, python_bin):
   240      """Gets the python include path."""
   241      result = _execute(
   242          repository_ctx,
   243          [
   244              python_bin,
   245              "-c",
   246              "from __future__ import print_function;" +
   247              "from distutils import sysconfig;" +
   248              "print(sysconfig.get_python_inc())",
   249          ],
   250          error_msg = "Problem getting python include path.",
   251          error_details = ("Is the Python binary path set up right? " +
   252                           "(See ./configure or " + _PYTHON_BIN_PATH + ".) " +
   253                           "Is distutils installed?"),
   254      )
   255      return result.stdout.splitlines()[0]
   256  
   257  def _get_python_import_lib_name(repository_ctx, python_bin):
   258      """Get Python import library name (pythonXY.lib) on Windows."""
   259      result = _execute(
   260          repository_ctx,
   261          [
   262              python_bin,
   263              "-c",
   264              "import sys;" +
   265              'print("python" + str(sys.version_info[0]) + ' +
   266              '      str(sys.version_info[1]) + ".lib")',
   267          ],
   268          error_msg = "Problem getting python import library.",
   269          error_details = ("Is the Python binary path set up right? " +
   270                           "(See ./configure or " + _PYTHON_BIN_PATH + ".) "),
   271      )
   272      return result.stdout.splitlines()[0]
   273  
   274  def _get_numpy_include(repository_ctx, python_bin):
   275      """Gets the numpy include path."""
   276      return _execute(
   277          repository_ctx,
   278          [
   279              python_bin,
   280              "-c",
   281              "from __future__ import print_function;" +
   282              "import numpy;" +
   283              " print(numpy.get_include());",
   284          ],
   285          error_msg = "Problem getting numpy include path.",
   286          error_details = "Is numpy installed?",
   287      ).stdout.splitlines()[0]
   288  
   289  def _create_local_python_repository(repository_ctx):
   290      """Creates the repository containing files set up to build with Python."""
   291      python_bin = _get_python_bin(repository_ctx)
   292      _check_python_bin(repository_ctx, python_bin)
   293      python_lib = _get_python_lib(repository_ctx, python_bin)
   294      _check_python_lib(repository_ctx, python_lib)
   295      python_include = _get_python_include(repository_ctx, python_bin)
   296      numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy"
   297      python_include_rule = _symlink_genrule_for_dir(
   298          repository_ctx,
   299          python_include,
   300          "python_include",
   301          "python_include",
   302      )
   303      python_import_lib_genrule = ""
   304  
   305      # To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
   306      # See https://docs.python.org/3/extending/windows.html
   307      if _is_windows(repository_ctx):
   308          python_include = _norm_path(python_include)
   309          python_import_lib_name = _get_python_import_lib_name(repository_ctx, python_bin)
   310          python_import_lib_src = python_include.rsplit("/", 1)[0] + "/libs/" + python_import_lib_name
   311          python_import_lib_genrule = _symlink_genrule_for_dir(
   312              repository_ctx,
   313              None,
   314              "",
   315              "python_import_lib",
   316              [python_import_lib_src],
   317              [python_import_lib_name],
   318          )
   319      numpy_include_rule = _symlink_genrule_for_dir(
   320          repository_ctx,
   321          numpy_include,
   322          "numpy_include/numpy",
   323          "numpy_include",
   324      )
   325      _tpl(repository_ctx, "BUILD", {
   326          "%{PYTHON_INCLUDE_GENRULE}": python_include_rule,
   327          "%{PYTHON_IMPORT_LIB_GENRULE}": python_import_lib_genrule,
   328          "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule,
   329      })
   330  
   331  def _create_remote_python_repository(repository_ctx, remote_config_repo):
   332      """Creates pointers to a remotely configured repo set up to build with Python.
   333      """
   334      repository_ctx.template("BUILD", Label(remote_config_repo + ":BUILD"), {})
   335  
   336  def _python_autoconf_impl(repository_ctx):
   337      """Implementation of the python_autoconf repository rule."""
   338      if _TF_PYTHON_CONFIG_REPO in repository_ctx.os.environ:
   339          _create_remote_python_repository(
   340              repository_ctx,
   341              repository_ctx.os.environ[_TF_PYTHON_CONFIG_REPO],
   342          )
   343      else:
   344          _create_local_python_repository(repository_ctx)
   345  
   346  python_configure = repository_rule(
   347      implementation = _python_autoconf_impl,
   348      environ = [
   349          _BAZEL_SH,
   350          _PYTHON_BIN_PATH,
   351          _PYTHON_LIB_PATH,
   352          _TF_PYTHON_CONFIG_REPO,
   353      ],
   354  )
   355  """Detects and configures the local Python.
   356  
   357  Add the following to your WORKSPACE FILE:
   358  
   359  ```python
   360  python_configure(name = "local_config_python")
   361  ```
   362  
   363  Args:
   364    name: A unique name for this workspace rule.
   365  """