github.com/onflow/flow-go@v0.35.7-crescendo-preview.23-atree-inlining/.github/workflows/flaky-test-monitor.yml (about)

     1  # This workflow runs all skipped (flaky) and non-skipped (regular) tests and generates a summary that's uploaded to BigQuery.
     2  
     3  name: Flaky Test Monitor
     4  
     5  on:
     6    schedule:
     7      - cron: '0 */2 * * *' # every 2 hours
     8    push:
     9      paths:
    10        - '.github/workflows/flaky-test-monitor.yml'
    11  
    12  env:
    13    BIGQUERY_DATASET: production_src_flow_test_metrics
    14    BIGQUERY_TABLE: skipped_tests
    15    BIGQUERY_TABLE2: test_results
    16    GO_VERSION: "1.20"
    17    SKIPPED_TESTS_FILE: skipped-tests
    18    RESULTS_FILE: test-results
    19    COMMIT_SHA: ${{ github.sha }}
    20    RUN_ID: ${{ github.run_id }}
    21    JSON_OUTPUT: true
    22    VERBOSE: true
    23    TEST_FLAKY: true
    24  
    25  concurrency:
    26    group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
    27    cancel-in-progress: true
    28  
    29  jobs:
    30    create-dynamic-test-matrix:
    31      name: Create Dynamic Test Matrix
    32      runs-on: ubuntu-latest
    33      outputs:
    34        dynamic-matrix: ${{ steps.set-test-matrix.outputs.dynamicMatrix }}
    35      steps:
    36        - name: Checkout repo
    37          uses: actions/checkout@v3
    38        - name: Setup Go
    39          uses: actions/setup-go@v4
    40          with:
    41            go-version: ${{ env.GO_VERSION }}
    42            cache: true
    43        - name: Set Test Matrix
    44          id: set-test-matrix
    45          run: go run utils/test_matrix/test_matrix.go admin cmd consensus engine fvm ledger module network/test network/p2p utils
    46  
    47    unit-test:
    48      name: Unit Tests (${{ matrix.targets.name }})
    49      needs: create-dynamic-test-matrix
    50      strategy:
    51        fail-fast: false
    52        matrix:
    53          targets: ${{ fromJSON(needs.create-dynamic-test-matrix.outputs.dynamic-matrix)}}
    54      # need to set image explicitly due to GitHub logging issue as described in https://github.com/onflow/flow-go/pull/3087#issuecomment-1234383202
    55      runs-on: ubuntu-20.04
    56      steps:
    57        - name: Checkout repo
    58          uses: actions/checkout@v3
    59        - name: Setup Go
    60          uses: actions/setup-go@v4
    61          with:
    62            go-version: ${{ env.GO_VERSION }}
    63            cache: true
    64        - name: Setup tests (${{ matrix.targets.name }})
    65          run: make -e GO_TEST_PACKAGES="${{ matrix.targets.packages }}" install-tools
    66        - name: Run tests (${{ matrix.targets.name }})
    67          run: make -es GO_TEST_PACKAGES="${{ matrix.targets.packages }}" unittest-main > test-output
    68          timeout-minutes: 100
    69          # test run should continue even if there are failed tests
    70          continue-on-error: true
    71        - name: Process test results
    72          env:
    73            TEST_CATEGORY: unit
    74          uses: ./.github/workflows/actions/test-monitor-process-results
    75          with:
    76            gcp_sa_key: ${{ secrets.GCP_SA_KEY }}
    77  
    78    unit-test-modules:
    79      name: Unit Tests (Modules)
    80      strategy:
    81        fail-fast: false
    82        matrix:
    83          include:
    84            - name: crypto
    85              setup: noop
    86              race: 1
    87              test_category: unit-crypto
    88            - name: insecure
    89              setup: install-tools
    90              race: 0
    91              test_category: unit-insecure
    92            - name: integration
    93              setup: install-tools
    94              race: 0
    95              test_category: unit-integration
    96      runs-on: ubuntu-latest
    97      steps:
    98        - name: Checkout repo
    99          uses: actions/checkout@v3
   100        - name: Setup Go
   101          uses: actions/setup-go@v4
   102          with:
   103            go-version: ${{ env.GO_VERSION }}
   104            cache: true
   105        - name: Setup tests (${{ matrix.name }})
   106          run: make ${{ matrix.setup }}
   107        - name: Run tests (${{ matrix.name }})
   108          env:
   109            RACE_DETECTOR: ${{ matrix.race }}
   110          run: make -es -C ${{ matrix.name }} test > test-output
   111          timeout-minutes: 100
   112          continue-on-error: true
   113        - name: Process test results (${{ matrix.name }})
   114          env:
   115            TEST_CATEGORY: ${{ matrix.test_category }}
   116          uses: ./.github/workflows/actions/test-monitor-process-results
   117          with:
   118            gcp_sa_key: ${{ secrets.GCP_SA_KEY }}
   119  
   120    integration-test:
   121      name: Integration Tests
   122      strategy:
   123        fail-fast: false
   124        matrix:
   125          include:
   126            - target: access-tests
   127              test_category: integration-access
   128            - target: bft-protocol-tests
   129              test_category: integration-bft-protocol
   130            - target: bft-framework-tests
   131              test_category: integration-bft-framework
   132            - target: bft-gossipsub-tests
   133              test_category: integration-bft-gossipsub
   134            - target: collection-tests
   135              test_category: integration-collection
   136            - target: consensus-tests
   137              test_category: integration-consensus
   138            - target: epochs-cohort1-tests
   139              test_category: integration-epochs
   140            - target: epochs-cohort2-tests
   141              test_category: integration-epochs
   142            - target: execution-tests
   143              test_category: integration-execution
   144            - target: ghost-tests
   145              test_category: integration-ghost
   146            - target: mvp-tests
   147              test_category: integration-mvp
   148            - target: network-tests
   149              test_category: integration-network
   150            - target: verification-tests
   151              test_category: integration-verification
   152            - target: upgrades-tests
   153              test_category: integration-upgrades
   154  
   155      runs-on: ubuntu-latest
   156      steps:
   157        - name: Checkout repo
   158          uses: actions/checkout@v3
   159          with:
   160            # all tags are needed for integration tests
   161            fetch-depth: 0
   162        - name: Setup Go
   163          uses: actions/setup-go@v4
   164          with:
   165            go-version: ${{ env.GO_VERSION }}
   166            cache: true
   167        - name: Docker build
   168          run: make docker-native-build-flow docker-native-build-flow-corrupt
   169        - name: Run tests
   170          run: make -es -C integration ${{ matrix.target }} > test-output
   171          timeout-minutes: 100
   172          continue-on-error: true
   173        - name: Process test results
   174          env:
   175            TEST_CATEGORY: ${{ matrix.test_category }}
   176          uses: ./.github/workflows/actions/test-monitor-process-results
   177          with:
   178            gcp_sa_key: ${{ secrets.GCP_SA_KEY }}