github.com/GoogleContainerTools/kpt@v1.0.0-beta.50.0.20240520170205-c25345ffcbee/scripts/create-licenses.sh (about)

     1  #!/usr/bin/env bash
     2  # Copyright 2015 The Kubernetes Authors.
     3  #
     4  # Licensed under the Apache License, Version 2.0 (the "License");
     5  # you may not use this file except in compliance with the License.
     6  # You may obtain a copy of the License at
     7  #
     8  #     http://www.apache.org/licenses/LICENSE-2.0
     9  #
    10  # Unless required by applicable law or agreed to in writing, software
    11  # distributed under the License is distributed on an "AS IS" BASIS,
    12  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  # See the License for the specific language governing permissions and
    14  # limitations under the License.
    15  
    16  # Update the LICENSES document.
    17  # Generates a table of Godep dependencies and their license.
    18  #
    19  # Usage:
    20  #    $0 [--create-missing] [/path/to/licenses]
    21  #
    22  #    --create-missing will write the files that only exist upstream, locally.
    23  #    This option is mostly used for testing as we cannot check-in any of the
    24  #    additionally created files into the vendor auto-generated tree.
    25  #
    26  #    Run every time a license file is added/modified within /vendor to
    27  #    update LICENSES
    28  
    29  set -o errexit
    30  set -o nounset
    31  set -o pipefail
    32  
    33  export LANG=C
    34  export LC_ALL=C
    35  
    36  # Name of file to write licenses.
    37  VENDOR_LICENSE_FILE="LICENSES.txt"
    38  
    39  # Zip package name for Mozilla licensed source code.
    40  ZIP_FILENAME="lib.zip"
    41  
    42  ###############################################################################
    43  # Process package content
    44  #
    45  # @param package  The incoming package name
    46  # @param type     The type of content (LICENSE, COPYRIGHT or COPYING)
    47  #
    48  process_content () {
    49    local package=$1
    50    local type=$2
    51  
    52    local package_root
    53    local ensure_pattern
    54    local dir_root
    55    local find_maxdepth
    56    local find_names
    57    local -a local_files=()
    58  
    59    # Necessary to expand {}
    60    case ${type} in
    61        LICENSE) find_names=(-iname 'licen[sc]e*')
    62                 find_maxdepth=1
    63                 # Sadly inconsistent in the wild, but mostly license files
    64                 # containing copyrights, but no readme/notice files containing
    65                 # licenses (except to "see license file")
    66                 ensure_pattern="license|copyright"
    67                 ;;
    68      # We search READMEs for copyrights and this includes notice files as well
    69      # Look in as many places as we find files matching
    70      COPYRIGHT) find_names=(-iname 'notice*' -o -iname 'readme*')
    71                 find_maxdepth=3
    72                 ensure_pattern="copyright"
    73                 ;;
    74        COPYING) find_names=(-iname 'copying*')
    75                 find_maxdepth=1
    76                 ensure_pattern="license|copyright"
    77                 ;;
    78    esac
    79  
    80    # Start search at package root
    81    case ${package} in
    82      github.com/*|golang.org/*|bitbucket.org/*|gonum.org/*)
    83       package_root=$(echo "${package}" |awk -F/ '{ print $1"/"$2"/"$3 }')
    84       ;;
    85      go4.org/*)
    86       package_root=$(echo "${package}" |awk -F/ '{ print $1 }')
    87       ;;
    88      gopkg.in/*)
    89       # Root of gopkg.in package always ends with '.v(number)' and my contain
    90       # more than two path elements. For example:
    91       # - gopkg.in/yaml.v2
    92       # - gopkg.in/inf.v0
    93       # - gopkg.in/square/go-jose.v2
    94       package_root=$(echo "${package}" |grep -oh '.*\.v[0-9]')
    95       ;;
    96      */*)
    97       package_root=$(echo "${package}" |awk -F/ '{ print $1"/"$2 }')
    98       ;;
    99      *)
   100       package_root="${package}"
   101       ;;
   102    esac
   103  
   104    # Find files - only root and package level
   105    local_files=()
   106    IFS=" " read -r -a local_files <<< "$(
   107      for dir_root in ${package} ${package_root}; do
   108        [[ -d ${DEPS_DIR}/${dir_root} ]] || continue
   109        
   110        # One (set) of these is fine
   111        find "${DEPS_DIR}/${dir_root}" \
   112            -xdev -follow -maxdepth ${find_maxdepth} \
   113            -type f "${find_names[@]}"
   114      done | sort -u)"
   115  
   116    local index
   117    local f
   118    index="${package}-${type}"
   119    if [[ -z "${CONTENT[${index}]-}" ]]; then
   120      for f in "${local_files[@]-}"; do
   121        if [[ -z "$f" ]]; then
   122          # Set the default value and then check it to prevent
   123          # accessing potentially empty array
   124          continue
   125        fi
   126        # Find some copyright info in any file and break
   127        if grep -E -i -wq "${ensure_pattern}" "${f}"; then
   128          CONTENT[${index}]="${f}"
   129          break
   130        fi
   131      done
   132    fi
   133  }
   134  
   135  
   136  #############################################################################
   137  # MAIN
   138  #############################################################################
   139  KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
   140  source "${KUBE_ROOT}/scripts/lib/init.sh"
   141  
   142  export GO111MODULE=on
   143  
   144  # Check bash version
   145  if (( BASH_VERSINFO[0] < 4 )); then
   146    echo
   147    echo "ERROR: Bash v4+ required."
   148    # Extra help for OSX
   149    if [[ "$(uname -s)" == "Darwin" ]]; then
   150      echo
   151      echo "Ensure you are up to date on the following packages:"
   152      echo "$ brew install md5sha1sum bash jq"
   153    fi
   154    echo
   155    exit 9
   156  fi
   157  
   158  TMP_LICENSE_FILE="/tmp/LICENSES.$$"
   159  DEPS_DIR="vendor"
   160  declare -Ag CONTENT
   161  
   162  # Run go mod vendor to create vendor dependencies.
   163  go mod vendor
   164  
   165  # Some cleanups
   166  # Rename all license files to `LICENSE`
   167  find vendor -type f -name LICENSE.md -execdir mv LICENSE.md LICENSE ';'
   168  find vendor -type f -name LICENSE.txt -execdir mv LICENSE.txt LICENSE ';'
   169  # Move LICENSE file to root directory of each dependency. This is necessary for
   170  # dependencies where code is stored in a versioned sub-directory.
   171  V2_LICENSE_DIR="vendor/github.com/cpuguy83/go-md2man"
   172  mv ${V2_LICENSE_DIR}/v2/LICENSE ${V2_LICENSE_DIR}
   173  GO_RESTFUL_LICENSE_DIR="vendor/github.com/emicklei/go-restful"
   174  mv ${GO_RESTFUL_LICENSE_DIR}/v3/LICENSE ${GO_RESTFUL_LICENSE_DIR}
   175  KLOG_LICENSE_DIR="vendor/k8s.io/klog"
   176  mv ${KLOG_LICENSE_DIR}/v2/LICENSE ${KLOG_LICENSE_DIR}
   177  XXHASH_LICENSE_DIR="vendor/github.com/cespare/xxhash"
   178  mv "${XXHASH_LICENSE_DIR}/v2/LICENSE" "${XXHASH_LICENSE_DIR}"
   179  BLACKFRIDAY_LICENSE_DIR="vendor/github.com/russross/blackfriday"
   180  mv "${BLACKFRIDAY_LICENSE_DIR}/v2/LICENSE" "${BLACKFRIDAY_LICENSE_DIR}"
   181  
   182  # Loop through every vendored package
   183  mozilla_repos=()
   184  for PACKAGE in $(go list -mod=mod -m -json all | jq -r .Path | sort -f); do
   185    if [[ -e "staging/src/${PACKAGE}" ]]; then
   186      # echo "$PACKAGE is a staging package, skipping" > /dev/stderr
   187      continue
   188    fi
   189    if [[ ! -e "${DEPS_DIR}/${PACKAGE}" ]]; then
   190      # echo "$PACKAGE doesn't exist in vendor, skipping" > /dev/stderr
   191      continue
   192    fi
   193    # TODO: samwronski - remove this edge case
   194    # The above if statement skips dependencies which did not get checked out with
   195    # `go mod vendor` however that does not catch this edge case.
   196    # Kpt currently depends on 2 versions of posener. Because v2 *is* checked out
   197    # the directory does exist causing the above check to pass. However this repo
   198    # is not included in the vendor directory so a license will not be found.
   199    if [[ "${PACKAGE}" == "github.com/posener/complete" ]]; then
   200      continue
   201    fi
   202  
   203    # Skip self.
   204    # The LICENSE file is at the root but vendor directory contents are selective.
   205    if [[ "${PACKAGE}" =~ ^github.com/GoogleContainerTools/kpt(/.*)?$ ]]; then
   206      # echo "Skipping ${PACKAGE}" > /dev/stderr
   207      continue
   208    fi
   209  
   210    # cloud.google.com/go has a tricky structure in terms of LICENSE files.
   211    # Use the go.mod package path to resolve the license.
   212    if [[ "${PACKAGE}" =~ ^cloud.google.com/go(/.*)?$ ]]; then
   213      # Make sure we have downloaded the package into the package cache
   214      go mod download "${PACKAGE}"
   215      # Look for a license in the package cache
   216      for PACKAGE_DIR in $(go list -mod=mod -m -json ${PACKAGE} | jq -r .Dir); do
   217        if [[ -e "${PACKAGE_DIR}/LICENSE" ]]; then
   218          CONTENT["${PACKAGE}-LICENSE"]="${PACKAGE_DIR}/LICENSE"
   219        fi
   220      done
   221    fi
   222  
   223    process_content "${PACKAGE}" LICENSE
   224    process_content "${PACKAGE}" COPYRIGHT
   225    process_content "${PACKAGE}" COPYING
   226  
   227    # display content
   228    echo
   229    echo "================================================================================"
   230    echo "= ${DEPS_DIR}/${PACKAGE} licensed under: ="
   231    echo
   232  
   233    file=""
   234    if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then
   235        file="${CONTENT[${PACKAGE}-LICENSE]-}"
   236    elif [[ -n "${CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then
   237        file="${CONTENT[${PACKAGE}-COPYRIGHT]-}"
   238    elif [[ -n "${CONTENT[${PACKAGE}-COPYING]-}" ]]; then
   239        file="${CONTENT[${PACKAGE}-COPYING]-}"
   240    fi
   241    if [[ -z "${file}" ]]; then
   242        cat > /dev/stderr << __EOF__
   243  No license could be found for ${PACKAGE} - aborting.
   244  
   245  Options:
   246  1. Check if the upstream repository has a newer version with LICENSE, COPYRIGHT and/or
   247     COPYING files.
   248  2. Contact the author of the package to ensure there is a LICENSE, COPYRIGHT and/or
   249     COPYING file present.
   250  3. Do not use this package in Kubernetes.
   251  __EOF__
   252        exit 9
   253    fi
   254  
   255    # Check to see if its a Mozilla license. If so, we need to package the source code.
   256    license=$(cat "${file}")
   257    if [[ "$license" == *"Mozilla"* ]]
   258    then
   259      mozilla_repos+=("${DEPS_DIR}/${PACKAGE}")
   260    fi
   261  
   262    cat ${file}
   263  
   264    echo
   265    echo "= ${file} $(kube::util::md5 "${file}")"
   266    echo "================================================================================"
   267    echo
   268  done >> ${TMP_LICENSE_FILE}
   269  
   270  cat ${TMP_LICENSE_FILE} > ${VENDOR_LICENSE_FILE}
   271  
   272  # initialize zip file to ensure existence (downstream builds depend on libs.zip)
   273  README="${KUBE_ROOT}/scripts/docs/create-licenses/README.md"
   274  zip -q "${ZIP_FILENAME}" "${README}"
   275  
   276  # Create a package of Mozilla repository source code (only go code).
   277  [ ${#mozilla_repos[@]} != 0 ] && zip -qur "${ZIP_FILENAME}" "${mozilla_repos[@]}" -i '*.go'
   278  
   279  # Cleanup vendor directory
   280  rm -rf vendor