github.com/lineaje-labs/syft@v0.98.1-0.20231227153149-9e393f60ff1b/Taskfile.yaml (about)

     1  
     2  version: "3"
     3  vars:
     4    OWNER: anchore
     5    PROJECT: syft
     6  
     7    # static file dirs
     8    TOOL_DIR: .tool
     9    TMP_DIR: .tmp
    10  
    11    # used for changelog generation
    12    CHANGELOG: CHANGELOG.md
    13    NEXT_VERSION: VERSION
    14  
    15    # used for snapshot builds
    16    OS:
    17      sh: uname -s | tr '[:upper:]' '[:lower:]'
    18    ARCH:
    19      sh: |
    20        [ "$(uname -m)" = "x86_64" ] && echo "amd64_v1" || echo $(uname -m)
    21    PROJECT_ROOT:
    22      sh: echo $PWD
    23    # note: the snapshot dir must be a relative path starting with ./
    24    SNAPSHOT_DIR: ./snapshot
    25    SNAPSHOT_BIN: "{{ .PROJECT_ROOT }}/{{ .SNAPSHOT_DIR }}/{{ .OS }}-build_{{ .OS }}_{{ .ARCH }}/{{ .PROJECT }}"
    26    SNAPSHOT_CMD: "{{ .TOOL_DIR }}/goreleaser release --config {{ .TMP_DIR }}/goreleaser.yaml --clean --snapshot --skip=publish --skip=sign"
    27    BUILD_CMD:    "{{ .TOOL_DIR }}/goreleaser build   --config {{ .TMP_DIR }}/goreleaser.yaml --clean --snapshot --single-target"
    28    RELEASE_CMD:  "{{ .TOOL_DIR }}/goreleaser release --clean"
    29    VERSION:
    30      sh: git describe --dirty --always --tags
    31  
    32    # used for install and acceptance testing
    33    COMPARE_DIR: ./test/compare
    34    COMPARE_TEST_IMAGE: centos:8.2.2004
    35  
    36  tasks:
    37  
    38    ## High-level tasks #################################
    39  
    40    default:
    41      desc: Run all validation tasks
    42      aliases:
    43        - pr-validations
    44        - validations
    45      cmds:
    46        - task: static-analysis
    47        - task: test
    48        - task: install-test
    49  
    50    static-analysis:
    51      desc: Run all static analysis tasks
    52      cmds:
    53        - task: check-go-mod-tidy
    54        - task: check-licenses
    55        - task: lint
    56        - task: check-json-schema-drift
    57  
    58    test:
    59      desc: Run all levels of test
    60      cmds:
    61        - task: unit
    62        - task: integration
    63        - task: validate-cyclonedx-schema
    64        - task: benchmark
    65        - task: test-utils
    66        - task: cli
    67  
    68    ## Bootstrap tasks #################################
    69  
    70    binny:
    71      internal: true
    72      # desc: Get the binny tool
    73      generates:
    74        - "{{ .TOOL_DIR }}/binny"
    75      status:
    76        - "test -f {{ .TOOL_DIR }}/binny"
    77      cmd: "curl -sSfL https://raw.githubusercontent.com/anchore/binny/main/install.sh | sh -s -- -b .tool"
    78      silent: true
    79  
    80    tools:
    81      desc: Install all tools needed for CI and local development
    82      deps: [binny]
    83      aliases:
    84        - bootstrap
    85      generates:
    86        - ".binny.yaml"
    87        - "{{ .TOOL_DIR }}/*"
    88      status:
    89        - "{{ .TOOL_DIR }}/binny check -v"
    90      cmd: "{{ .TOOL_DIR }}/binny install -v"
    91      silent: true
    92  
    93    update-tools:
    94      desc: Update pinned versions of all tools to their latest available versions
    95      deps: [binny]
    96      generates:
    97        - ".binny.yaml"
    98        - "{{ .TOOL_DIR }}/*"
    99      cmd: "{{ .TOOL_DIR }}/binny update -v"
   100      silent: true
   101  
   102    list-tools:
   103      desc: List all tools needed for CI and local development
   104      deps: [binny]
   105      cmd: "{{ .TOOL_DIR }}/binny list"
   106      silent: true
   107  
   108    list-tool-updates:
   109      desc: List all tools that are not up to date relative to the binny config
   110      deps: [binny]
   111      cmd: "{{ .TOOL_DIR }}/binny list --updates"
   112      silent: true
   113  
   114    tmpdir:
   115      silent: true
   116      generates:
   117        - "{{ .TMP_DIR }}"
   118      cmd: "mkdir -p {{ .TMP_DIR }}"
   119  
   120    ## Static analysis tasks #################################
   121  
   122    format:
   123      desc: Auto-format all source code
   124      deps: [tools]
   125      cmds:
   126        - gofmt -w -s .
   127        - "{{ .TOOL_DIR }}/gosimports -local github.com/anchore -w ."
   128        - go mod tidy
   129  
   130    lint-fix:
   131      desc: Auto-format all source code + run golangci lint fixers
   132      deps: [tools]
   133      cmds:
   134        - task: format
   135        - "{{ .TOOL_DIR }}/golangci-lint run --tests=false --fix"
   136  
   137    lint:
   138      desc: Run gofmt + golangci lint checks
   139      vars:
   140        BAD_FMT_FILES:
   141          sh: gofmt -l -s .
   142        BAD_FILE_NAMES:
   143          sh: "find . | grep -e ':' || true"
   144      deps: [tools]
   145      cmds:
   146        # ensure there are no go fmt differences
   147        - cmd: 'test -z "{{ .BAD_FMT_FILES }}" || (echo "files with gofmt issues: [{{ .BAD_FMT_FILES }}]"; exit 1)'
   148          silent: true
   149        # ensure there are no files with ":" in it (a known back case in the go ecosystem)
   150        - cmd: 'test -z "{{ .BAD_FILE_NAMES }}" || (echo "files with bad names: [{{ .BAD_FILE_NAMES }}]"; exit 1)'
   151          silent: true
   152        # run linting
   153        - "{{ .TOOL_DIR }}/golangci-lint run --tests=false"
   154  
   155  
   156    check-licenses:
   157      # desc: Ensure transitive dependencies are compliant with the current license policy
   158      deps: [tools]
   159      cmd: "{{ .TOOL_DIR }}/bouncer check ./..."
   160  
   161    check-go-mod-tidy:
   162      # desc: Ensure go.mod and go.sum are up to date
   163      cmds:
   164        - cmd: .github/scripts/go-mod-tidy-check.sh && echo "go.mod and go.sum are tidy!"
   165          silent: true
   166  
   167    check-json-schema-drift:
   168      desc: Ensure there is no drift between the JSON schema and the code
   169      cmds:
   170        - .github/scripts/json-schema-drift-check.sh
   171  
   172  
   173    ## Testing tasks #################################
   174  
   175    unit:
   176      desc: Run unit tests
   177      deps:
   178        - tmpdir
   179        - fixtures
   180      vars:
   181        TEST_PKGS:
   182          sh: "go list ./... | grep -v {{ .OWNER }}/{{ .PROJECT }}/test | tr '\n' ' '"
   183  
   184        # unit test coverage threshold (in % coverage)
   185        COVERAGE_THRESHOLD: 62
   186      cmds:
   187        - "go test -coverprofile {{ .TMP_DIR }}/unit-coverage-details.txt {{ .TEST_PKGS }}"
   188        - cmd: ".github/scripts/coverage.py {{ .COVERAGE_THRESHOLD }} {{ .TMP_DIR }}/unit-coverage-details.txt"
   189          silent: true
   190  
   191    integration:
   192      desc: Run integration tests
   193      cmds:
   194        - "go test -v ./test/integration"
   195        # exercise most of the CLI with the data race detector
   196        - "go run -race cmd/syft/main.go alpine:latest"
   197  
   198    validate-cyclonedx-schema:
   199      desc: Run integration tests
   200      cmds:
   201        - "cd schema/cyclonedx && make"
   202  
   203    cli:
   204      desc: Run CLI tests
   205      # note: we don't want to regenerate the snapshot unless we have to. In CI it's probable
   206      # that the cache being restored with the correct binary will be rebuilt since the timestamps
   207      # and local checksums will not line up.
   208      deps: [tools, snapshot]
   209      sources:
   210        - "{{ .SNAPSHOT_BIN }}"
   211        - ./test/cli/**
   212        - ./**/*.go
   213      cmds:
   214        - cmd: "echo 'testing binary: {{ .SNAPSHOT_BIN }}'"
   215          silent: true
   216  
   217        - cmd: "test -f {{ .SNAPSHOT_BIN }} || (find {{ .SNAPSHOT_DIR }} && echo '\nno snapshot found' && false)"
   218          silent: true
   219  
   220        - "go test -count=1 -timeout=15m -v ./test/cli"
   221      env:
   222        SYFT_BINARY_LOCATION: "{{ .SNAPSHOT_BIN }}"
   223  
   224    test-utils:
   225      desc: Run tests for pipeline utils
   226      sources:
   227        - .github/scripts/labeler*.py
   228      cmds:
   229        - cmd: python .github/scripts/labeler_test.py
   230  
   231  
   232    ## Benchmark test targets #################################
   233  
   234    benchmark:
   235      deps: [tmpdir]
   236      sources:
   237        - ./**/*.go
   238      generates:
   239        - "{{ .TMP_DIR }}/benchmark-main.txt"
   240      cmds:
   241        - "go test -p 1 -run=^Benchmark -bench=. -count=7 -benchmem ./... | tee {{ .TMP_DIR }}/benchmark-{{ .VERSION }}.txt"
   242        - |
   243          bash -c "(test -s {{ .TMP_DIR }}/benchmark-main.txt && \
   244          {{ .TOOL_DIR }}/benchstat {{ .TMP_DIR }}/benchmark-main.txt {{ .TMP_DIR }}/benchmark-{{ .VERSION }}.txt || \
   245          {{ .TOOL_DIR }}/benchstat {{ .TMP_DIR }}/benchmark-{{ .VERSION }}.txt) \
   246          | tee {{ .TMP_DIR }}/benchstat.txt"
   247  
   248    show-benchstat:
   249      deps: [benchmark, tmpdir]
   250      sources:
   251        - "{{ .TMP_DIR }}/benchstat.txt"
   252      cmds:
   253        - cmd: "cat {{ .TMP_DIR }}/benchstat.txt"
   254          silent: true
   255  
   256  
   257    ## Test-fixture-related targets #################################
   258  
   259    fingerprints:
   260      desc: Generate test fixture fingerprints
   261      generates:
   262        - test/integration/test-fixtures/cache.fingerprint
   263        - syft/pkg/cataloger/binary/test-fixtures/cache.fingerprint
   264        - syft/pkg/cataloger/java/test-fixtures/java-builds/cache.fingerprint
   265        - syft/pkg/cataloger/golang/test-fixtures/archs/binaries.fingerprint
   266        - syft/pkg/cataloger/redhat/test-fixtures/rpms.fingerprint
   267        - syft/pkg/cataloger/kernel/test-fixtures/cache.fingerprint
   268        - test/install/cache.fingerprint
   269        - test/cli/test-fixtures/cache.fingerprint
   270      cmds:
   271        # for IMAGE integration test fixtures
   272        - "cd test/integration/test-fixtures && make cache.fingerprint"
   273        # for BINARY test fixtures
   274        - "cd syft/pkg/cataloger/binary/test-fixtures && make cache.fingerprint"
   275        # for JAVA BUILD test fixtures
   276        - "cd syft/pkg/cataloger/java/test-fixtures/java-builds && make cache.fingerprint"
   277        # for GO BINARY test fixtures
   278        - "cd syft/pkg/cataloger/golang/test-fixtures/archs && make binaries.fingerprint"
   279        # for RPM test fixtures
   280        - "cd syft/pkg/cataloger/redhat/test-fixtures && make rpms.fingerprint"
   281        # for Kernel test fixtures
   282        - "cd syft/pkg/cataloger/kernel/test-fixtures && make cache.fingerprint"
   283        # for INSTALL integration test fixtures
   284        - "cd test/install && make cache.fingerprint"
   285        # for CLI test fixtures
   286        - "cd test/cli/test-fixtures && make cache.fingerprint"
   287  
   288    fixtures:
   289      desc: Generate test fixtures
   290      cmds:
   291        - "cd syft/pkg/cataloger/java/test-fixtures/java-builds && make"
   292        - "cd syft/pkg/cataloger/redhat/test-fixtures && make"
   293        - "cd syft/pkg/cataloger/binary/test-fixtures && make"
   294  
   295    show-test-image-cache:
   296      silent: true
   297      cmds:
   298        - "echo '\nDocker daemon cache:'"
   299        - "docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}:{{`{{.Tag}}`}}' | grep stereoscope-fixture- | sort"
   300        - "echo '\nTar cache:'"
   301        - 'find . -type f -wholename "**/test-fixtures/snapshot/*" | sort'
   302  
   303  
   304    ## install.sh testing targets #################################
   305  
   306    install-test:
   307      cmds:
   308        - "cd test/install && make"
   309  
   310    install-test-cache-save:
   311      cmds:
   312        - "cd test/install && make save"
   313  
   314    install-test-cache-load:
   315      cmds:
   316        - "cd test/install && make load"
   317  
   318    install-test-ci-mac:
   319      cmds:
   320        - "cd test/install && make ci-test-mac"
   321  
   322    generate-compare-file:
   323      cmd: "go run ./cmd/syft {{ .COMPARE_TEST_IMAGE }} -o json > {{ .COMPARE_DIR }}/test-fixtures/acceptance-{{ .COMPARE_TEST_IMAGE }}.json"
   324  
   325    compare-mac:
   326      deps: [tmpdir]
   327      cmd: |
   328        {{ .COMPARE_DIR }}/mac.sh \
   329          {{ .SNAPSHOT_DIR }} \
   330          {{ .COMPARE_DIR }} \
   331          {{ .COMPARE_TEST_IMAGE }} \
   332          {{ .TMP_DIR }}
   333  
   334    compare-linux:
   335      cmds:
   336        - task: compare-test-deb-package-install
   337        - task: compare-test-rpm-package-install
   338  
   339    compare-test-deb-package-install:
   340      deps: [tmpdir]
   341      cmd: |
   342        {{ .COMPARE_DIR }}/deb.sh \
   343          {{ .SNAPSHOT_DIR }} \
   344          {{ .COMPARE_DIR }} \
   345          {{ .COMPARE_TEST_IMAGE }} \
   346          {{ .TMP_DIR }}
   347  
   348    compare-test-rpm-package-install:
   349      deps: [tmpdir]
   350      cmd: |
   351        {{ .COMPARE_DIR }}/rpm.sh \
   352          {{ .SNAPSHOT_DIR }} \
   353          {{ .COMPARE_DIR }} \
   354          {{ .COMPARE_TEST_IMAGE }} \
   355          {{ .TMP_DIR }}
   356  
   357  
   358    ## Code and data generation targets #################################
   359  
   360    generate:
   361      desc: Add data generation tasks
   362      cmds:
   363        - task: generate-json-schema
   364        - task: generate-license-list
   365        - task: generate-cpe-dictionary-index
   366  
   367    generate-json-schema:
   368      desc: Generate a new JSON schema
   369      cmds:
   370        - "cd syft/internal && go generate . && cd jsonschema && go run . && go fmt ../..."
   371  
   372    generate-license-list:
   373      desc: Generate an updated license processing code off of the latest available SPDX license list
   374      cmds:
   375        - "go generate ./internal/spdxlicense/..."
   376        - "gofmt -s -w ./internal/spdxlicense"
   377  
   378    generate-cpe-dictionary-index:
   379      desc: Generate the CPE index based off of the latest available CPE dictionary
   380      dir: "syft/pkg/cataloger/common/cpe/dictionary"
   381      cmds:
   382        - "go generate"
   383  
   384  
   385    ## Build-related targets #################################
   386  
   387    build:
   388      desc: Build the project
   389      deps: [tools, tmpdir]
   390      generates:
   391        - "{{ .PROJECT }}"
   392      cmds:
   393        - silent: true
   394          cmd: |
   395            echo "dist: {{ .SNAPSHOT_DIR }}" > {{ .TMP_DIR }}/goreleaser.yaml
   396            cat .goreleaser.yaml >> {{ .TMP_DIR }}/goreleaser.yaml
   397  
   398        - "{{ .BUILD_CMD }}"
   399  
   400    snapshot:
   401      desc: Create a snapshot release
   402      aliases:
   403        - build
   404      deps: [tools, tmpdir]
   405      sources:
   406        - cmd/**/*.go
   407        - syft/**/*.go
   408        - internal/**/*.go
   409      method: checksum
   410      generates:
   411        - "{{ .SNAPSHOT_BIN }}"
   412      cmds:
   413        - silent: true
   414          cmd: |
   415            echo "dist: {{ .SNAPSHOT_DIR }}" > {{ .TMP_DIR }}/goreleaser.yaml
   416            cat .goreleaser.yaml >> {{ .TMP_DIR }}/goreleaser.yaml
   417  
   418        - "{{ .SNAPSHOT_CMD }}"
   419  
   420    changelog:
   421      desc: Generate a changelog
   422      deps: [tools]
   423      generates:
   424        - "{{ .CHANGELOG }}"
   425        - "{{ .NEXT_VERSION }}"
   426      cmds:
   427        - "{{ .TOOL_DIR }}/chronicle -vv -n --version-file {{ .NEXT_VERSION }} > {{ .CHANGELOG }}"
   428        - "{{ .TOOL_DIR }}/glow {{ .CHANGELOG }}"
   429  
   430  
   431    ## Release targets #################################
   432  
   433    release:
   434      desc: Create a release
   435      interactive: true
   436      deps: [tools]
   437      cmds:
   438        - cmd: .github/scripts/trigger-release.sh
   439          silent: true
   440  
   441  
   442    ## CI-only targets #################################
   443  
   444    ci-check:
   445      # desc: "[CI only] Are you in CI?"
   446      cmds:
   447        - cmd: .github/scripts/ci-check.sh
   448          silent: true
   449  
   450    ci-release:
   451      # desc: "[CI only] Create a release"
   452      deps: [tools]
   453      cmds:
   454        - task: ci-check
   455        - "{{ .RELEASE_CMD }}"
   456  
   457  
   458    ## Cleanup targets #################################
   459  
   460    clean-snapshot:
   461      desc: Remove any snapshot builds
   462      cmds:
   463        - "rm -rf {{ .SNAPSHOT_DIR }}"
   464        - "rm -rf {{ .TMP_DIR }}/goreleaser.yaml"
   465  
   466    clean-cache:
   467      desc: Remove all docker cache and local image tar cache
   468      cmds:
   469        - 'find . -type f -wholename "**/test-fixtures/cache/stereoscope-fixture-*.tar" -delete'
   470        - "docker images --format '{{`{{.ID}}`}} {{`{{.Repository}}`}}' | grep stereoscope-fixture- | awk '{print $$1}' | uniq | xargs -r docker rmi --force"