volcano.sh/volcano@v1.9.0/hack/update-vendor-licenses.sh (about) 1 #!/usr/bin/env bash 2 # Copyright 2015 The Kubernetes Authors. 3 # 4 # Licensed under the Apache License, Version 2.0 (the "License"); 5 # you may not use this file except in compliance with the License. 6 # You may obtain a copy of the License at 7 # 8 # http://www.apache.org/licenses/LICENSE-2.0 9 # 10 # Unless required by applicable law or agreed to in writing, software 11 # distributed under the License is distributed on an "AS IS" BASIS, 12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 # See the License for the specific language governing permissions and 14 # limitations under the License. 15 16 # Update the LICENSES directory. 17 # Generates a table of Go dependencies and their licenses. 18 # 19 # Usage: 20 # $0 [--create-missing] [/path/to/licenses] 21 # 22 # --create-missing will write the files that only exist upstream, locally. 23 # This option is mostly used for testing as we cannot check-in any of the 24 # additionally created files into the vendor auto-generated tree. 25 # 26 # Run every time a license file is added/modified within /vendor to 27 # update /LICENSES 28 # 29 # ----------------------------------------------------------------------------- 30 # CHANGELOG 31 # Volcano Authors: 32 # - File derived from kubernetes v1.19.0-beta.2 33 # - Changed KUBE_ROOT value to use absolute path 34 35 36 set -o errexit 37 set -o nounset 38 set -o pipefail 39 40 KUBE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" 41 source "${KUBE_ROOT}/hack/lib/init.sh" 42 43 export LANG=C 44 export LC_ALL=C 45 46 ############################################################################### 47 # Process package content 48 # 49 # @param package The incoming package name 50 # @param type The type of content (LICENSE, COPYRIGHT or COPYING) 51 # 52 process_content () { 53 local package=$1 54 local type=$2 55 56 local package_root 57 local ensure_pattern 58 local dir_root 59 local find_maxdepth 60 local find_names 61 local -a local_files=() 62 63 # Necessary to expand {} 64 case ${type} in 65 LICENSE) find_names=(-iname 'licen[sc]e*') 66 find_maxdepth=1 67 # Sadly inconsistent in the wild, but mostly license files 68 # containing copyrights, but no readme/notice files containing 69 # licenses (except to "see license file") 70 ensure_pattern="license|copyright" 71 ;; 72 # We search READMEs for copyrights and this includes notice files as well 73 # Look in as many places as we find files matching 74 COPYRIGHT) find_names=(-iname 'notice*' -o -iname 'readme*') 75 find_maxdepth=3 76 ensure_pattern="copyright" 77 ;; 78 COPYING) find_names=(-iname 'copying*') 79 find_maxdepth=1 80 ensure_pattern="license|copyright" 81 ;; 82 esac 83 84 # Start search at package root 85 case ${package} in 86 github.com/*|golang.org/*|bitbucket.org/*|gonum.org/*) 87 package_root=$(echo "${package}" |awk -F/ '{ print $1"/"$2"/"$3 }') 88 ;; 89 go4.org/*) 90 package_root=$(echo "${package}" |awk -F/ '{ print $1 }') 91 ;; 92 gopkg.in/*) 93 # Root of gopkg.in package always ends with '.v(number)' and my contain 94 # more than two path elements. For example: 95 # - gopkg.in/yaml.v2 96 # - gopkg.in/inf.v0 97 # - gopkg.in/square/go-jose.v2 98 package_root=$(echo "${package}" |grep -oh '.*\.v[0-9]') 99 ;; 100 */*) 101 package_root=$(echo "${package}" |awk -F/ '{ print $1"/"$2 }') 102 ;; 103 *) 104 package_root="${package}" 105 ;; 106 esac 107 108 # Find files - only root and package level 109 local_files=() 110 IFS=" " read -r -a local_files <<< "$( 111 for dir_root in ${package} ${package_root}; do 112 [[ -d ${DEPS_DIR}/${dir_root} ]] || continue 113 114 # One (set) of these is fine 115 find "${DEPS_DIR}/${dir_root}" \ 116 -xdev -follow -maxdepth ${find_maxdepth} \ 117 -type f "${find_names[@]}" 118 done | sort -u)" 119 120 local index 121 local f 122 index="${package}-${type}" 123 if [[ -z "${CONTENT[${index}]-}" ]]; then 124 for f in "${local_files[@]-}"; do 125 if [[ -z "$f" ]]; then 126 # Set the default value and then check it to prevent 127 # accessing potentially empty array 128 continue 129 fi 130 # Find some copyright info in any file and break 131 if grep -E -i -wq "${ensure_pattern}" "${f}"; then 132 CONTENT[${index}]="${f}" 133 break 134 fi 135 done 136 fi 137 } 138 139 140 ############################################################################# 141 # MAIN 142 ############################################################################# 143 144 export GO111MODULE=on 145 146 # Check bash version 147 if (( BASH_VERSINFO[0] < 4 )); then 148 echo 149 echo "ERROR: Bash v4+ required." 150 # Extra help for OSX 151 if [[ "$(uname -s)" == "Darwin" ]]; then 152 echo 153 echo "Ensure you are up to date on the following packages:" 154 echo "$ brew install md5sha1sum bash jq" 155 fi 156 echo 157 exit 9 158 fi 159 160 # This variable can be injected, as in the verify script. 161 LICENSE_ROOT="${LICENSE_ROOT:-${KUBE_ROOT}}" 162 cd "${LICENSE_ROOT}" 163 164 kube::util::ensure-temp-dir 165 166 # Save the genreated LICENSE file for each package temporarily 167 TMP_LICENSE_FILE="${KUBE_TEMP}/LICENSES.$$" 168 169 # The directory to save all the LICENSE files 170 LICENSES_DIR="${LICENSES_DIR:-${LICENSE_ROOT}/LICENSES}" 171 mkdir -p "${LICENSES_DIR}" 172 173 # The tmp directory to save all the LICENSE files, will move to LICENSES_DIR 174 TMP_LICENSES_DIR="${KUBE_TEMP}/LICENSES.DIR.$$" 175 mkdir -p "${TMP_LICENSES_DIR}" 176 177 DEPS_DIR="vendor" 178 declare -Ag CONTENT 179 180 # Put the K8S LICENSE on top 181 if [ -f "${LICENSE_ROOT}/LICENSE" ]; then 182 ( 183 echo "================================================================================" 184 echo "= Volcano licensed under: =" 185 echo 186 cat "${LICENSE_ROOT}/LICENSE" 187 echo 188 echo "= LICENSE $(kube::util::md5 "${LICENSE_ROOT}/LICENSE")" 189 echo "================================================================================" 190 ) > "${TMP_LICENSE_FILE}" 191 mv "${TMP_LICENSE_FILE}" "${TMP_LICENSES_DIR}/LICENSE" 192 fi 193 194 # Loop through every vendored package 195 for PACKAGE in $(go list -m -mod=mod -json all | jq -r .Path | sort -f); do 196 if [[ -e "staging/src/${PACKAGE}" ]]; then 197 echo "${PACKAGE} is a staging package, skipping" >&2 198 continue 199 fi 200 if [[ ! -e "${DEPS_DIR}/${PACKAGE}" ]]; then 201 echo "${PACKAGE} doesn't exist in ${DEPS_DIR}, skipping" >&2 202 continue 203 fi 204 if [[ "${PACKAGE}" = "sigs.k8s.io/structured-merge-diff" ]]; then 205 # this package doesn't exist, but has v3 subdirectory as a different package 206 # so it can't be filtered by the previous rule 207 # temporarily treat this way until find out a better rule 208 echo "${PACKAGE}, temporarily skipping" >&2 209 continue 210 fi 211 if [[ "${PACKAGE}" = "github.com/blang/semver" ]]; then 212 # this package doesn't exist, but has v4 subdirectory as a different package 213 # so it can't be filtered by the previous rule 214 # temporarily treat this way until find out a better rule 215 echo "${PACKAGE}, temporarily skipping" >&2 216 continue 217 fi 218 if [[ "${PACKAGE}" = "github.com/emicklei/go-restful" ]]; then 219 # this package doesn't exist, but has v3 subdirectory as a different package 220 # so it can't be filtered by the previous rule 221 # temporarily treat this way until find out a better rule 222 echo "${PACKAGE}, temporarily skipping" >&2 223 continue 224 fi 225 if [[ "${PACKAGE}" = "github.com/cespare/xxhash" ]]; then 226 # there are 2 versions v1 and v2 under 2 folders indirectly used 227 # so it can't be filtered by the previous rule 228 # temporarily treat this way until find out a better rule 229 echo "${PACKAGE}, temporarily skipping" >&2 230 continue 231 fi 232 if [[ "${PACKAGE}" = "k8s.io/klog" ]]; then 233 # this package doesn't use, but has v2 subdirectory as a different package 234 # so it can't be filtered by the previous rule 235 # temporarily treat this way until find out a better rule 236 echo "${PACKAGE}, temporarily skipping" >&2 237 continue 238 fi 239 if [[ "${PACKAGE}" = "github.com/onsi/ginkgo" ]]; then 240 # there are 2 versions v1 and v2 under 2 folders indirectly used 241 # so it can't be filtered by the previous rule 242 # temporarily treat this way until find out a better rule 243 echo "${PACKAGE}, temporarily skipping" >&2 244 continue 245 fi 246 echo "${PACKAGE}" 247 248 process_content "${PACKAGE}" LICENSE 249 process_content "${PACKAGE}" COPYRIGHT 250 process_content "${PACKAGE}" COPYING 251 252 # copy content and throw error message 253 { 254 echo "= ${DEPS_DIR}/${PACKAGE} licensed under: =" 255 echo 256 257 file="" 258 if [[ -n "${CONTENT[${PACKAGE}-LICENSE]-}" ]]; then 259 file="${CONTENT[${PACKAGE}-LICENSE]-}" 260 elif [[ -n "${CONTENT[${PACKAGE}-COPYRIGHT]-}" ]]; then 261 file="${CONTENT[${PACKAGE}-COPYRIGHT]-}" 262 elif [[ -n "${CONTENT[${PACKAGE}-COPYING]-}" ]]; then 263 file="${CONTENT[${PACKAGE}-COPYING]-}" 264 fi 265 if [[ -z "${file}" ]]; then 266 cat >&2 << __EOF__ 267 No license could be found for ${PACKAGE} - aborting. 268 269 Options: 270 1. Check if the upstream repository has a newer version with LICENSE, COPYRIGHT and/or 271 COPYING files. 272 2. Contact the author of the package to ensure there is a LICENSE, COPYRIGHT and/or 273 COPYING file present. 274 3. Do not use this package in Volcano. 275 __EOF__ 276 exit 9 277 fi 278 279 cat "${file}" 280 echo 281 echo "= ${file} $(kube::util::md5 "${file}")" 282 } >> "${TMP_LICENSE_FILE}" 283 284 dest_dir="${TMP_LICENSES_DIR}/vendor/${PACKAGE}" 285 mkdir -p "${dest_dir}" 286 mv "${TMP_LICENSE_FILE}" "${dest_dir}/LICENSE" 287 done 288 289 # Leave things like OWNERS alone. 290 rm -f "${LICENSES_DIR}/LICENSE" 291 rm -rf "${LICENSES_DIR}/vendor" 292 mv "${TMP_LICENSES_DIR}"/* "${LICENSES_DIR}"