kythe.io@v0.0.68-0.20240422202219-7225dbc01741/kythe/release/kythe.sh (about)

     1  #!/bin/bash -e
     2  
     3  # Copyright 2014 The Kythe Authors. All rights reserved.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #   http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  set -o pipefail
    18  export SHELL=/bin/bash
    19  
    20  usage() {
    21    cat >&2 <<EOF
    22  usage: kythe [--repo git-url] [--extract extractor] [--index]
    23               [--ignore-unhandled] [--files config-path] [--files-excludes re1,re2]
    24  
    25  example: docker run --rm -t -v "$HOME/repo:/repo" -v "$HOME/gs:/graphstore" \
    26             google/kythe --extract maven --index --files --files-excludes '(^|/)\.,^third_party'
    27  
    28  Extraction:
    29    If given an --extract type, the compilations in the mounted /repo VOLUME (or the given --repo
    30    which will copied to /repo) will be extracted to the /compilations VOLUME w/ subdirectories for
    31    each compilation's language (e.g. /compilations/java, /compilations/go).
    32  
    33    Supported Extractors: maven
    34  
    35  Indexing:
    36    If given the --index flag, each compilation in /compilations will be sent to a corresponding
    37    language indexer and the outputs will be stored in a GraphStore in the /graphstore VOLUME.  If a
    38    compilation is without a corresponding language indexer, an error will be reported unless
    39    --ignore-unhandled is set.
    40  
    41    To emit file nodes for the entire repository, use the --files flag to specify a JSON file VNames
    42    configuration relative to the repository root.  --files-excludes can be used to exclude certain
    43    paths by a comma-separated list regex patterns.  It is highly recommended to exclude build
    44    output directories such as '(^|/)target'.  The --index flag is required for --files to be handled.
    45  
    46    Supported Languages: java,c++
    47  EOF
    48  }
    49  
    50  usage_error() {
    51    echo "ERROR: $*" >&2
    52    usage
    53    exit 1
    54  }
    55  
    56  error() {
    57    echo "ERROR: $*" >&2
    58    exit 1
    59  }
    60  
    61  cleanup() {
    62    fix_permissions /repo
    63    fix_permissions /compilations
    64    fix_permissions /graphstore
    65    fix_permissions /root/.m2
    66  }
    67  trap cleanup EXIT
    68  
    69  REPO=
    70  IGNORE_UNHANDLED=
    71  EXTRACTOR=
    72  INDEXING=
    73  FILES_CONFIG=
    74  FILES_EXCLUDES='(^|/)\.'
    75  
    76  while [[ $# -gt 0 ]]; do
    77    case "$1" in
    78      --repo|-r)
    79        REPO="$2"
    80        shift ;;
    81      --extract|-e)
    82        EXTRACTOR="$2"
    83        shift ;;
    84      --files|-f)
    85        FILES_CONFIG="$2"
    86        shift ;;
    87      --files-excludes)
    88        FILES_EXCLUDES="$2"
    89        shift ;;
    90      --index|-i)
    91        INDEXING=1 ;;
    92      --ignore-unhandled)
    93        IGNORE_UNHANDLED=1 ;;
    94      --help|-h)
    95        usage
    96        exit 0 ;;
    97      *) usage_error "Unknown argument: $1" ;;
    98    esac
    99    shift
   100  done
   101  
   102  mkdir -p /repo /compilations /graphstore
   103  
   104  if [[ -n "$REPO" ]]; then
   105    if [ ! "$(ls -A /repo)" ]; then
   106      error '/repo not empty when given --repo'
   107    fi
   108    git clone "$REPO" /repo
   109  fi
   110  
   111  case "$EXTRACTOR" in
   112    maven)
   113      echo 'Extracting compilations' >&2
   114      "${EXTRACTOR}_extractor" ;;
   115    "")
   116      echo 'Skipping extraction' >&2 ;;
   117    *)
   118      error "Unknown extractor: '$EXTRACTOR'" ;;
   119  esac
   120  
   121  if [[ -z "$INDEXING" ]]; then
   122    echo 'Skipping indexing' >&2
   123    exit
   124  fi
   125  
   126  drive_indexer_kzip() {
   127    local lang
   128    lang="$(basename "$(dirname "$1")")"
   129    local analyzer="/kythe/bin/${lang}_indexer"
   130    if [[ ! -x "$analyzer" ]]; then
   131      if [[ -n "$IGNORE_UNHANDLED" ]]; then
   132        return 0
   133      else
   134        echo "Unhandled index file for '$lang': $*" >&2
   135        return 1
   136      fi
   137    fi
   138    echo "Indexing $*" >&2
   139    "$analyzer" "$@"
   140  }
   141  export -f drive_indexer_kzip
   142  export IGNORE_UNHANDLED
   143  
   144  find /compilations -name '*.kzip' | sort -R | \
   145    { parallel --gnu -L1 drive_indexer_kzip || echo "$? analysis failures" >&2; } | \
   146      dedup_stream | \
   147      write_entries --workers 12 --graphstore /graphstore
   148  
   149  if [[ -z "$FILES_CONFIG" ]]; then
   150    echo "Skipping repository files indexing" >&2
   151    exit
   152  fi
   153  
   154  echo 'Emitting nodes for repository' >&2
   155  cd /repo/ && \
   156    index_repository --vnames "$FILES_CONFIG" --exclude "$FILES_EXCLUDES" | \
   157    write_entries --workers 4 --graphstore /graphstore