kythe.io@v0.0.68-0.20240422202219-7225dbc01741/kythe/extractors/bazel/extract.sh (about)

     1  #!/bin/bash
     2  
     3  # Copyright 2019 The Kythe Authors. All rights reserved.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #   http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # Usage:
    18  # extract.sh \
    19  #     --define kythe_corpus=github.com/project/repo \
    20  #     //...
    21  #
    22  # Input `kythe_corpus` should be the repository you're extracting (for example
    23  # github.com/protocolbuffers/protobuf), as well as the build target to extract.
    24  # A good default is //..., which extracts almost everything.
    25  #
    26  # Note that this doesn't some targets - for example build targets with
    27  # `tags = ["manual"]` will not be extracted.  If you have  additional targets to
    28  # extract those can be appended cleanly:
    29  #
    30  #     extract.sh --define... //... //some/manual:target
    31  #
    32  # If you have restrictions on what can or should be extracted, for example an
    33  # entire directory to ignore, you must specify your partitions individually.  A
    34  # good tool for doing this (instead of manually discovering everything) is bazel
    35  # query, described at
    36  # https://docs.bazel.build/versions/master/query-how-to.html.
    37  #
    38  # Outputs $KYTHE_OUTPUT_DIRECTORY/compilations.kzip
    39  #
    40  # Requires having environment variable $KYTHE_OUTPUT_DIRECTORY set, as well
    41  # as kzip tool (kythe/go/platform/tools/kzip) installed to /kythe/kzip and
    42  # kythe/release/base/fix_permissions.sh copied to /kythe/fix_permissions.sh.
    43  # Also assumes you have extractors installed as per
    44  # kythe/extractors/bazel/extractors.bazelrc.
    45  
    46  # Print our commands for easier debugging and exit after our first failed
    47  # command so we avoid silent failures.
    48  set -ex
    49  
    50  : "${KYTHE_OUTPUT_DIRECTORY:?Missing output directory}" "${KYTHE_KZIP_ENCODING:=JSON}"
    51  
    52  if [[ -n "$KYTHE_SYSTEM_DEPS" ]]; then
    53    echo "Installing $KYTHE_SYSTEM_DEPS"
    54    # shellcheck disable=SC2086
    55    # TODO(jaysachs): unclear if we should bail if any packages fail to install
    56    apt-get update && \
    57    apt-get upgrade -y && \
    58    apt-get --fix-broken install -y && \
    59    apt-get install -y $KYTHE_SYSTEM_DEPS && \
    60    apt-get clean
    61  fi
    62  
    63  if [[ -n "$KYTHE_PRE_BUILD_STEP" ]]; then
    64    eval "$KYTHE_PRE_BUILD_STEP"
    65  fi
    66  
    67  KYTHE_RELEASE=/kythe
    68  
    69  if [[ -n "$KYTHE_BAZEL_TARGET" ]]; then
    70    # $KYTHE_BAZEL_TARGET is unquoted because bazel_wrapper needs to see each
    71    # target expression in KYTHE_BAZEL_WRAPPER as individual arguments. For
    72    # example, if KYTHE_BAZEL_TARGET=//foo/... -//foo/test/..., bazel_wrapper
    73    # needs to see two valid target expressions (//foo/... and -//foo/test/...)
    74    # not one invalid target expression with white space
    75    # ("//foo/... -//foo/test/...").
    76    /kythe/bazel_wrapper.sh --bazelrc="$KYTHE_RELEASE/extractors.bazelrc" "$@" --override_repository "kythe_release=$KYTHE_RELEASE" -- "$KYTHE_BAZEL_TARGET"
    77  else
    78    # If the user supplied a bazel query, execute it and run bazel, but we have to
    79    # shard the results to different bazel runs because the bazel command line
    80    # cannot take many arguments. Right now we build 30 targets at a time. We can
    81    # change this value or make it settable once we have more data on the
    82    # implications.
    83    /kythe/bazelisk query "$KYTHE_BAZEL_QUERY" | \
    84      xargs -t -L 30 /kythe/bazel_wrapper.sh --bazelrc=$KYTHE_RELEASE/extractors.bazelrc "$@" --override_repository kythe_release=$KYTHE_RELEASE --
    85  fi
    86  
    87  # Collect any extracted compilations.
    88  mkdir -p "$KYTHE_OUTPUT_DIRECTORY"
    89  find bazel-out/*/extra_actions/ -name '*.kzip' -print0 | \
    90    xargs --null -r /kythe/tools/kzip merge --append --encoding "$KYTHE_KZIP_ENCODING" --output "$KYTHE_OUTPUT_DIRECTORY/compilations.kzip"
    91  
    92  # Record the timestamp of the git commit in a metadata kzip.
    93  /kythe/tools/kzip create_metadata \
    94    --output buildmetadata.kzip \
    95    --corpus "$KYTHE_CORPUS" \
    96    --commit_timestamp "$(git log --pretty='%ad' -n 1 HEAD)"
    97  /kythe/tools/kzip merge --append --encoding "$KYTHE_KZIP_ENCODING" --output "$KYTHE_OUTPUT_DIRECTORY/compilations.kzip" buildmetadata.kzip
    98  
    99  /kythe/fix_permissions.sh "$KYTHE_OUTPUT_DIRECTORY"
   100  test -f "$KYTHE_OUTPUT_DIRECTORY/compilations.kzip"