github.com/wasilibs/wazerox@v0.0.0-20240124024944-4923be63ab5f/.github/workflows/integration.yaml (about) 1 name: Standard Library Integration Tests 2 on: 3 pull_request: 4 branches: [main] 5 paths-ignore: # ignore docs as they are built with Netlify. 6 - '**/*.md' 7 - 'site/**' 8 - 'netlify.toml' 9 push: 10 branches: [main] 11 paths-ignore: # ignore docs as they are built with Netlify. 12 - '**/*.md' 13 - 'site/**' 14 - 'netlify.toml' 15 16 defaults: 17 run: # use bash for all operating systems unless overridden 18 shell: bash 19 20 env: # Update this prior to requiring a higher minor version in go.mod 21 GO_VERSION: "1.21" # 1.xx == latest patch of 1.xx 22 TINYGO_VERSION: "0.30.0" 23 ZIG_VERSION: "0.11.0" 24 25 concurrency: 26 # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-concurrency-to-cancel-any-in-progress-job-or-run 27 group: ${{ github.ref }}-${{ github.workflow }}-${{ github.actor }} 28 cancel-in-progress: true 29 30 jobs: 31 # This builds a zig test binary only if the same version tag hasn't been build before. 32 # This saves time as we rarely update the zig version. 33 build_zig_test_binary: 34 name: Build Zig test binary 35 runs-on: ubuntu-22.04 36 env: 37 ZIG_INSTALL: ~/zig-install 38 ZIG_SOURCE: ~/zig-source 39 40 steps: 41 - name: Checkout wazero 42 uses: actions/checkout@v3 43 44 - uses: actions/cache@v3 45 id: binary-cache 46 with: 47 # Use share the cache containing archives across OSes. 48 enableCrossOsArchive: true 49 key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} 50 path: 51 ./zigbin/ 52 53 - name: Install Zig build 54 if: steps.binary-cache.outputs.cache-hit != 'true' 55 run: | 56 mkdir -p ${{ env.ZIG_INSTALL }} 57 curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-linux-x86_64-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_INSTALL }} 58 59 - name: Download Zig source code 60 if: steps.binary-cache.outputs.cache-hit != 'true' 61 run: | 62 mkdir -p ${{ env.ZIG_SOURCE }} 63 curl -sSL https://ziglang.org/download/${{ env.ZIG_VERSION }}/zig-${{ env.ZIG_VERSION }}.tar.xz | tar -xJ --strip-components=1 -C ${{ env.ZIG_SOURCE }} 64 65 - name: Build Stdlib test binary 66 if: steps.binary-cache.outputs.cache-hit != 'true' 67 # --test-no-exec allows building of the test Wasm binary without executing command. 68 # We use find because the test.wasm will be something like ./zig-cache/o/dd6df1361b2134adc5eee9d027495436/test.wasm 69 run: | 70 mkdir ${{ github.workspace }}/zigbin 71 cd ${{ env.ZIG_SOURCE }} 72 ${{ env.ZIG_INSTALL }}/zig test --test-no-exec -target wasm32-wasi --zig-lib-dir ./lib ./lib/std/std.zig 73 _ZIG_TEST_BINARY_PATH=$(find . -name test.wasm) 74 cp ${_ZIG_TEST_BINARY_PATH} ${{ github.workspace }}/zigbin/test.wasm 75 76 zig: 77 needs: build_zig_test_binary 78 name: Zig (${{ matrix.os }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler) 79 runs-on: ${{ matrix.os }} 80 strategy: 81 fail-fast: false # don't fail fast as sometimes failures are arch/OS specific 82 matrix: 83 os: [ubuntu-22.04, macos-12, windows-2022] 84 compiler: [baseline] 85 arch: [amd64] 86 include: 87 - os: ubuntu-22.04 88 compiler: optimizing 89 arch: "arm64" 90 91 steps: 92 - name: Checkout wazero 93 uses: actions/checkout@v3 94 95 - uses: actions/cache@v3 96 id: binary-cache 97 with: 98 # Use share the cache containing archives across OSes. 99 enableCrossOsArchive: true 100 # We need this cache to run tests. 101 fail-on-cache-miss: true 102 key: zig-stdlib-test-binary-${{ env.ZIG_VERSION }} 103 path: 104 ./zigbin/ 105 106 - uses: actions/setup-go@v4 107 with: 108 go-version: ${{ env.GO_VERSION }} 109 110 - name: Build wazero 111 run: go build -o ./wazerocli ./cmd/wazero 112 env: 113 GOARCH: ${{ matrix.arch }} 114 115 - name: Set up QEMU 116 if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} 117 uses: docker/setup-qemu-action@v2 118 with: # Avoid docker.io rate-limits; built with internal-images.yml 119 image: ghcr.io/tetratelabs/wazero/internal-binfmt 120 platforms: ${{ matrix.arch }} 121 122 - name: Build scratch container 123 if: ${{ matrix.compiler == 'optimizing' }} 124 run: | 125 echo 'FROM scratch' >> Dockerfile 126 echo 'CMD ["/test"]' >> Dockerfile 127 docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} . 128 129 130 # The following two steps run the previously compiled Zig tests with wazero. 131 # If you need to troubleshoot any of the test, you can add "-hostlogging=filesystem" after 132 # adding filter argument either to step "Run built test binaries" or 133 # "Run built test binaries (container)". 134 # e.g. --test-filter "Dir.Iterator but dir is deleted during iteration" 135 136 - name: Run built test binaries (container) 137 if: ${{ matrix.compiler == 'optimizing' }} 138 run: | 139 docker run --platform linux/${{ matrix.arch }} -v $(pwd)/zigbin:/test -v $(pwd)/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ 140 /wazero run -optimizing-compiler -mount=:/ ./test/test.wasm 141 142 - name: Run built test binaries 143 if: ${{ matrix.compiler != 'optimizing' }} 144 run: ./wazerocli run -mount=:/ ./zigbin/test.wasm 145 146 build_tinygo_test_binary: 147 name: Build TinyGo test binary 148 runs-on: ubuntu-22.04 149 steps: 150 - name: Checkout wazero 151 uses: actions/checkout@v3 152 153 - uses: actions/cache@v3 154 id: binary-cache 155 with: 156 # Use share the cache containing archives across OSes. 157 enableCrossOsArchive: true 158 key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} 159 path: 160 ./tinygobin/ 161 162 - name: Install TinyGo 163 if: steps.binary-cache.outputs.cache-hit != 'true' 164 run: | # installing via curl so commands are similar on OS/x 165 tinygo_version=${{ env.TINYGO_VERSION }} 166 curl -sSL https://github.com/tinygo-org/tinygo/releases/download/v${tinygo_version}/tinygo${tinygo_version}.linux-amd64.tar.gz | sudo tar -C /usr/local -xzf - 167 echo "TINYGOROOT=/usr/local/tinygo" >> $GITHUB_ENV 168 echo "/usr/local/tinygo/bin" >> $GITHUB_PATH 169 170 - uses: actions/setup-go@v4 171 if: steps.binary-cache.outputs.cache-hit != 'true' 172 with: 173 go-version: ${{ env.GO_VERSION }} 174 175 - name: Build Test Binaries 176 if: steps.binary-cache.outputs.cache-hit != 'true' 177 # The following list of packages is derived from: 178 # https://github.com/tinygo-org/tinygo/blob/v0.28.1/Makefile#L281-L322 179 # Note: 180 # - index/suffixarray is extremely slow, so skip it. 181 # - compress/zlib is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/compress/zlib/writer_test.go#L16-L30 182 # - debug/macho is skipped as it depends on the local files https://github.com/golang/go/blob/go1.20/src/debug/macho/file_test.go#L25 183 run: | 184 mkdir ./tinygobin 185 for value in container/heap \ 186 container/list \ 187 container/ring \ 188 crypto/des \ 189 crypto/md5 \ 190 crypto/rc4 \ 191 crypto/sha1 \ 192 crypto/sha256 \ 193 crypto/sha512 \ 194 embed/internal/embedtest \ 195 encoding \ 196 encoding/ascii85 \ 197 encoding/base32 \ 198 encoding/csv \ 199 encoding/hex \ 200 go/scanner \ 201 hash \ 202 hash/adler32 \ 203 hash/crc64 \ 204 hash/fnv \ 205 html \ 206 internal/itoa \ 207 internal/profile \ 208 math \ 209 math/cmplx \ 210 net \ 211 net/http/internal/ascii \ 212 net/mail \ 213 os \ 214 path \ 215 reflect \ 216 sync \ 217 testing \ 218 testing/iotest \ 219 text/scanner \ 220 unicode \ 221 unicode/utf16 \ 222 unicode/utf8 223 do 224 tinygo test -target wasi -c -o ./tinygobin/${value/\//_}.test $value 225 done 226 227 tinygo: 228 needs: build_tinygo_test_binary 229 name: TinyGo (${{ matrix.os }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler) 230 runs-on: ${{ matrix.os }} 231 strategy: 232 fail-fast: false # don't fail fast as sometimes failures are arch/OS specific 233 matrix: 234 os: [ubuntu-22.04, macos-12, windows-2022] 235 compiler: [baseline] 236 arch: [amd64] 237 include: 238 - os: ubuntu-22.04 239 compiler: optimizing 240 arch: "arm64" 241 242 steps: 243 - name: Checkout wazero 244 uses: actions/checkout@v3 245 246 - uses: actions/cache@v3 247 id: binary-cache 248 with: 249 # Use share the cache containing archives across OSes. 250 enableCrossOsArchive: true 251 # We need this cache to run tests. 252 fail-on-cache-miss: true 253 key: tinygo-test-binaries-${{ env.TINYGO_VERSION }} 254 path: 255 ./tinygobin/ 256 257 - uses: actions/setup-go@v4 258 with: 259 go-version: ${{ env.GO_VERSION }} 260 261 - name: Build wazero 262 run: go build -o ~/wazerocli ./cmd/wazero 263 env: 264 GOARCH: ${{ matrix.arch }} 265 266 - name: Set up QEMU 267 if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} 268 uses: docker/setup-qemu-action@v2 269 with: # Avoid docker.io rate-limits; built with internal-images.yml 270 image: ghcr.io/tetratelabs/wazero/internal-binfmt 271 platforms: ${{ matrix.arch }} 272 273 - name: Build scratch container 274 if: ${{ matrix.compiler == 'optimizing' }} 275 run: | 276 echo 'FROM scratch' >> Dockerfile 277 echo 'CMD ["/test"]' >> Dockerfile 278 docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} . 279 280 # The following steps run the previously compiled TinyGo tests with wazero. If you need 281 # to troubleshoot one, you can add "-hostlogging=filesystem" and also a 282 # trailing argument narrowing which test to execute. 283 # e.g. "-test.run '^TestStatBadDir$'" 284 285 - name: Run test binaries (container) 286 if: ${{ matrix.compiler == 'optimizing' }} 287 # This runs all tests compiled above in sequence. Note: This mounts /tmp to allow t.TempDir() in tests. 288 run: | 289 cd ./tinygobin 290 for bin in *.test; do 291 echo $bin 292 docker run --platform linux/${{ matrix.arch }} -v $(pwd):/test -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ 293 /wazero run -optimizing-compiler -mount=:/ /test/$bin -- -test.v 294 done 295 296 - name: Run test binaries 297 if: ${{ matrix.compiler != 'optimizing' }} 298 run: | 299 cd ./tinygobin 300 for bin in *.test; do 301 echo $bin 302 ~/wazerocli run -mount=:/ -mount=:/tmp $bin -- -test.v 303 done 304 305 wasi-testsuite: 306 name: wasi-testsuite 307 runs-on: ${{ matrix.os }} 308 strategy: 309 fail-fast: false # don't fail fast as sometimes failures are arch/OS specific 310 matrix: 311 os: [ubuntu-22.04, macos-12, windows-2022] 312 313 steps: 314 - uses: actions/cache@v3 315 id: cache 316 with: 317 path: 318 ~/go/pkg/mod 319 key: integration-test-wasi-testsuite-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ hashFiles('**/go.sum') }} 320 321 - uses: actions/setup-go@v4 322 with: 323 go-version: ${{ env.GO_VERSION }} 324 325 - name: Checkout wazero 326 uses: actions/checkout@v3 327 328 - name: Install wazero 329 run: go install ./cmd/wazero 330 331 - name: Checkout wasi-testsuite 332 uses: actions/checkout@v3 333 with: 334 repository: WebAssembly/wasi-testsuite 335 # prod/testsuite-base branch, as of May 12, 2023. 336 # TODO: once the wasi-testsuite is stable, we should use the latest tag instead of a branch. 337 ref: c9c751586fd86b321d595bbef13f2c7403cfdbc5 338 path: wasi-testsuite 339 340 - name: Initialize Python environment 341 uses: actions/setup-python@v4 342 with: 343 python-version: '3.11' # latest version of python 3 344 cache: pip 345 346 - name: Install dependencies 347 working-directory: wasi-testsuite/test-runner 348 run: | 349 python3 -m pip install -r requirements.txt 350 351 - name: Run all wasi-testsuite 352 working-directory: wasi-testsuite 353 run: | 354 python3 test-runner/wasi_test_runner.py \ 355 -t ./tests/assemblyscript/testsuite/ \ 356 ./tests/c/testsuite/ \ 357 ./tests/rust/testsuite/ \ 358 -f ../.github/wasi_testsuite_skip.json \ 359 -r ../.github/wasi_testsuite_adapter.py 360 361 gojs_stdlib: 362 name: Go (js) (${{ matrix.os }}) 363 runs-on: ${{ matrix.os }} 364 strategy: 365 fail-fast: false # don't fail fast as sometimes failures are arch/OS specific 366 matrix: 367 os: [ubuntu-22.04, macos-12] # GOOS=js isn't supposed to work on windows. See #1222 368 369 steps: 370 - uses: actions/setup-go@v4 371 with: 372 go-version: ${{ env.GO_VERSION }} 373 374 - name: Checkout wazero 375 uses: actions/checkout@v3 376 377 - name: Install wazero 378 run: go install ./cmd/wazero 379 380 - name: Build gojs test binaries 381 env: 382 GOOS: js 383 GOARCH: wasm 384 run: | # Only test os package as this is being replaced by GOOS=wasip1 385 mkdir ~/bin && cd ~/bin 386 go test -c -o os.wasm os 387 388 - name: Run tests 389 run: | # skip tests that use functionality not also used in GOOS=wasip1 390 cd $(go env GOROOT)/src/os; wazero run -mount=/:/ ~/bin/os.wasm -test.v -test.skip '^Test(Chmod|Truncate|LongPath|Chown|FileChown).*$' 391 392 go_tests: 393 # Due to the embedding of the GOROOT of the building env(https://github.com/golang/go/blob/3c59639b902fada0a2e5a6a35bafd10fc9183b89/src/os/os_test.go#L112), 394 # we have to build and cache on each OS unlike others in this file. 395 name: Go (${{ matrix.os }}, Go-${{ matrix.go-version }}, ${{ matrix.arch }}, ${{ matrix.compiler }}-compiler) 396 runs-on: ${{ matrix.os }} 397 strategy: 398 fail-fast: false # don't fail fast as sometimes failures are arch/OS specific 399 matrix: 400 os: [ubuntu-22.04, macos-12, windows-2022] 401 compiler: [baseline] 402 arch: [amd64] 403 go-version: 404 - "1.21" # Current Go version && The only version that supports wasip1. 405 include: 406 - os: ubuntu-22.04 407 compiler: optimizing 408 arch: "arm64" 409 go-version: "1.21" 410 steps: 411 - id: setup-go 412 uses: actions/setup-go@v4 413 with: 414 go-version: ${{ matrix.go-version }} 415 416 - name: Cache Go test binaries 417 id: cache-go-test-binaries 418 uses: actions/cache@v3 419 with: 420 path: ~/tests 421 # Use precise Go version from setup-go as patch version differences can effect tests. 422 key: go-wasip1-binaries-${{ matrix.os }}-${{ steps.setup-go.outputs.go-version }} 423 424 - if: ${{ steps.cache-go-test-binaries.outputs.cache-hit != 'true' }} 425 name: Build Test Binaries 426 run: | 427 mkdir ~/tests 428 cd $(go env GOROOT) 429 # Choose important packages to limit execution time. 430 for value in src/archive/tar \ 431 src/bufio \ 432 src/bytes \ 433 src/context \ 434 src/encoding/ascii85 \ 435 src/encoding/asn1 \ 436 src/encoding/base32 \ 437 src/encoding/base64 \ 438 src/encoding/binary \ 439 src/encoding/csv \ 440 src/encoding/gob \ 441 src/encoding/hex \ 442 src/encoding/json \ 443 src/encoding/pem \ 444 src/encoding/xml \ 445 src/errors \ 446 src/expvar \ 447 src/flag \ 448 src/fmt \ 449 src/hash \ 450 src/hash/adler32 \ 451 src/hash/crc32 \ 452 src/hash/crc64 \ 453 src/hash/fnv \ 454 src/hash/maphash \ 455 src/io \ 456 src/io/fs \ 457 src/io/ioutil \ 458 src/log \ 459 src/log/syslog \ 460 src/maps \ 461 src/math \ 462 src/math/big \ 463 src/math/bits \ 464 src/math/cmplx \ 465 src/math/rand \ 466 src/mime \ 467 src/mime/multipart \ 468 src/mime/quotedprintable \ 469 src/os \ 470 src/os/exec \ 471 src/os/signal \ 472 src/os/user \ 473 src/path \ 474 src/reflect \ 475 src/regexp \ 476 src/regexp/syntax \ 477 src/runtime \ 478 src/runtime/internal/atomic \ 479 src/runtime/internal/math \ 480 src/runtime/internal/sys \ 481 src/slices \ 482 src/sort \ 483 src/strconv \ 484 src/strings \ 485 src/sync \ 486 src/sync/atomic \ 487 src/syscall \ 488 src/testing \ 489 src/testing/fstest \ 490 src/testing/iotest \ 491 src/testing/quick \ 492 src/time 493 do 494 echo "GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value" 495 GOOS=wasip1 GOARCH=wasm go test -v -c -o ~/tests/${value//\//_}.test ./$value 496 done 497 498 - name: Checkout wazero 499 uses: actions/checkout@v3 500 501 - name: Build wazero 502 run: go build -o ~/wazerocli ./cmd/wazero 503 env: 504 GOARCH: ${{ matrix.arch }} 505 506 - name: Set up QEMU 507 if: ${{ matrix.compiler == 'optimizing' && matrix.arch == 'arm64' }} 508 uses: docker/setup-qemu-action@v2 509 with: # Avoid docker.io rate-limits; built with internal-images.yml 510 image: ghcr.io/tetratelabs/wazero/internal-binfmt 511 platforms: ${{ matrix.arch }} 512 513 - name: Build scratch container 514 if: ${{ matrix.compiler == 'optimizing' }} 515 run: | 516 echo 'FROM scratch' >> Dockerfile 517 echo 'CMD ["/test"]' >> Dockerfile 518 docker buildx build -t wazero:test --platform linux/${{ matrix.arch }} . 519 520 - if: ${{ matrix.compiler == 'optimizing' }} 521 name: Run test binaries (container) 522 run: | 523 echo "Running $(find ~/tests -name *.test | wc -l) test binaries" 524 525 # Skip tests that are hideously slow (mostly compilation time) for running inside QEMU. 526 skip_targets="src_encoding_json|src_runtime|src_os_exec|src_math_big|src_encoding_gob|src_time|src_archive_tar|src_math_rand|src_expvar|src_testing|src_os" 527 # Go tests often look for files relative to the source. Change to the corresponding directory. 528 for bin in $(find ~/tests -name "*.test" | grep -vE "$skip_targets"); do 529 dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} 530 pushd $(go env GOROOT)/$dir 531 # Mount / as /ROOT in docker and then remount /ROOT as / in wazero; $bin is now in /ROOT/$bin. 532 docker run --platform linux/arm64 -v /:/ROOT -v ~/wazerocli:/wazero -e WAZEROCLI=/wazero --tmpfs /tmp --rm -t wazero:test \ 533 /wazero run -optimizing-compiler -mount=/ROOT:/ -mount=/tmp:/tmp -env PWD=$PWD /ROOT/$bin -- -test.short -test.v 534 popd 535 done 536 537 - if: ${{ matrix.compiler != 'optimizing' && runner.os != 'Windows' }} 538 name: Run test binaries 539 run: | 540 echo "Running $(find ~/tests -name *.test | wc -l) test binaries" 541 542 # Go tests often look for files relative to the source. Change to the corresponding directory. 543 for bin in ~/tests/*.test; do 544 dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} 545 pushd $(go env GOROOT)/$dir 546 ~/wazerocli run -mount=/:/ -env PWD=$PWD $bin -- -test.short -test.v 547 popd 548 done 549 550 - if: ${{ runner.os == 'Windows' }} 551 name: Run test binaries (Windows) 552 # Ack failures on Windows. https://github.com/tetratelabs/wazero/issues/1410 553 continue-on-error: true 554 run: | 555 GOOS=$(go env GOOS) 556 echo "Running $(find ~/tests -name *.test | wc -l) test binaries" 557 558 MOUNT=c:\\:/ 559 SCRIPT="$HOME/tests.cmd" 560 # Trim `/c` from the in-Wasm GOROOT. 561 REAL_GOROOT=$(go env GOROOT) 562 GOROOT=$(cygpath -u $REAL_GOROOT | cut -c 3-) 563 # Append early exit on cmd. 564 POSTFIX="if %errorlevel% neq 0 exit /b %errorlevel%" 565 RUNNER="cmd //c %USERPROFILE%\tests.cmd" 566 EXTRAPARAMS="-mount=%TEMP%:/tmp" 567 568 # Go tests often look for files relative to the source. Change to the corresponding directory. 569 for bin in ~/tests/*.test; do 570 dir=$(basename $bin); dir=${dir%.test}; dir=${dir//_/\/} 571 pushd $REAL_GOROOT/$dir 572 573 # Trim `/c` from the in-Wasm pwd. 574 IN_WASM_PWD=$(pwd | cut -c 3-) 575 # Convert to a Windows path. 576 bin=`cygpath -w $bin` 577 578 # Create a script with all the tests (do not run yet). 579 echo ${MOUNT} ${IN_WASM_PWD} $GOROOT/$dir 580 COMMAND="~/wazerocli run -mount=${MOUNT} ${EXTRAPARAMS} -hostlogging=filesystem -env PWD=${IN_WASM_PWD} -env GOROOT=${GOROOT} -env GOOS=wasip1 $bin -- -test.short -test.v" 581 echo $COMMAND >> $SCRIPT 582 # Uncomment the following line for early exit on error on Windows. 583 # Otherwise the tests will report are successful evne on failure. 584 # echo $POSTFIX >> $SCRIPT 585 popd 586 done 587 588 # Run all the tests in their own shell. 589 $RUNNER