github.com/klaytn/klaytn@v1.10.2/.circleci/config.yml (about)

     1  version: 2.1
     2  
     3  aliases:
     4    - &filter-only-version-tag
     5      tags:
     6        only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
     7      branches:
     8        ignore: /.*/
     9    - &filter-version-not-release
    10      tags:
    11        only: /^v[0-9]+\.[0-9]+\.[0-9]+.*/
    12      branches:
    13        ignore:
    14          - /release\/.*/
    15          - master
    16  
    17  orbs:
    18    codecov: codecov/codecov@1.0.5
    19    docker: circleci/docker@2.1.1
    20  
    21  executors:
    22    test-executor:
    23      working_directory: /go/src/github.com/klaytn/klaytn
    24      docker:
    25        - image: klaytn/build_base:1.2-go.1.18-solc0.8.13
    26    test-others-executor:
    27      working_directory: /go/src/github.com/klaytn/klaytn
    28      docker:
    29        - image: klaytn/build_base:1.2-go.1.18-solc0.8.13
    30        - image: localstack/localstack:0.13.0
    31        - image: circleci/redis:6.0.8-alpine
    32        - name: zookeeper
    33          image: wurstmeister/zookeeper
    34        - name: kafka
    35          image: wurstmeister/kafka
    36          environment:
    37            KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    38            KAFKA_ADVERTISED_LISTENERS: INSIDE://:9092,OUTSIDE://kafka:9094
    39            KAFKA_LISTENERS: INSIDE://:9092,OUTSIDE://kafka:9094
    40            KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT
    41            KAFKA_INTER_BROKER_LISTENER_NAME: INSIDE
    42    rpm-executor:
    43      working_directory: /go/src/github.com/klaytn/klaytn
    44      docker:
    45        - image: klaytn/circleci-rpmbuild:1.18
    46    default:
    47      working_directory: ~/go/src/github.com/klaytn/klaytn
    48      docker:
    49        - image: cimg/go:1.18
    50  
    51  commands:
    52    install-netcat:
    53      steps:
    54        - run:
    55            name: "Install netcat"
    56            command: |
    57              apt install -y netcat
    58    install-golang:
    59      steps:
    60        - run:
    61            name: "Install golang on machine"
    62            command: |
    63              curl -O https://dl.google.com/go/go1.18.linux-amd64.tar.gz
    64              mkdir $HOME/go1.18
    65              tar -C $HOME/go1.18 -xzf go1.18.linux-amd64.tar.gz
    66    install-golangci-lint:
    67      steps:
    68        - run:
    69            name: "Install golangci-lint"
    70            command: |
    71              curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.45.2
    72    pre-build:
    73      description: "before build, set version"
    74      steps:
    75        - run:
    76            name: "set variables"
    77            command: |
    78              export GOPATH=~/go
    79              export PATH=$HOME/go1.18/go/bin:$PATH
    80              pat="^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+.*"
    81  
    82              if [[ $CIRCLE_TAG =~ $pat ]]; then
    83                echo "this is rc version $CIRCLE_TAG"
    84                rc_num=$(echo $CIRCLE_TAG | cut -d '-' -f 2)
    85                sed -i 's/%d.%d.%d/%d.%d.%d~'$rc_num'/' params/version.go
    86                sed -n '/%d.%d.%d/p' params/version.go
    87              else
    88                echo "this is not RC version"
    89              fi
    90              echo "export KLAYTN_VERSION=$(go run build/rpm/main.go version)" >> $BASH_ENV
    91    build-packaging:
    92      description: "Build for each OS/Network"
    93      parameters:
    94        os-network:
    95          type: string
    96          default: "tar-linux-amd64-all"
    97      steps:
    98        - run:
    99            name: "build new(Cross compile)"
   100            command: |
   101              export GOPATH=~/go
   102              export PATH=$HOME/go1.18/go/bin:$PATH
   103              make << parameters.os-network >>
   104    upload-repo:
   105      description: "upload packaging tar.gz"
   106      parameters:
   107        item:
   108          type: string
   109          default: "kcn kpn ken kgen kscn kbn kspn ksen homi"
   110      steps:
   111        - run:
   112            name: "upload S3 repo"
   113            command: |
   114              export GOPATH=~/go
   115              export PATH=$HOME/go1.18/go/bin:$PATH
   116              KLAYTN_VERSION=$(go run build/rpm/main.go version)
   117              for item in << parameters.item >>; do aws s3 cp packages/${item}-*.tar.gz s3://$FRONTEND_BUCKET/packages/klaytn/$KLAYTN_VERSION/; done
   118    rpm-tagging:
   119      description: "rpm tagging for cypress"
   120      steps:
   121        - run:
   122            name: "rpm tagging"
   123            command: make rpm-all
   124        - run:
   125            name: "upload S3 repo"
   126            command: |
   127              PLATFORM_SUFFIX=$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m)
   128              KLAYTN_VERSION=$(go run build/rpm/main.go version)
   129  
   130              for item in kcn kpn ken kscn kspn ksen kbn kgen homi;
   131              do
   132                TARGET_RPM=$(find $item-linux-x86_64/rpmbuild/RPMS/x86_64/ | awk -v pat="$item(d)?-v" '$0~pat')
   133                aws s3 cp $TARGET_RPM s3://$FRONTEND_BUCKET/packages/rhel/7/prod/
   134                aws s3 cp $TARGET_RPM s3://$FRONTEND_BUCKET/packages/klaytn/$KLAYTN_VERSION/
   135              done
   136    rpm-tagging-baobab:
   137      description: "rpm tagging for baobab"
   138      steps:
   139        - run:
   140            name: "rpm tagging baobab"
   141            command: |
   142                make rpm-baobab-kcn
   143                make rpm-baobab-kpn
   144                make rpm-baobab-ken
   145        - run:
   146            name: "upload S3 repo"
   147            command: |
   148              for item in kcn kpn ken;
   149              do
   150                PLATFORM_SUFFIX=$(uname -s | tr '[:upper:]' '[:lower:]')-$(uname -m)
   151                TARGET_RPM=$(find $item-linux-x86_64/rpmbuild/RPMS/x86_64/ | awk -v pat="$item(d)?-baobab-v" '$0~pat')
   152                aws s3 cp $TARGET_RPM s3://$FRONTEND_BUCKET/packages/rhel/7/prod/
   153                aws s3 cp $TARGET_RPM s3://$FRONTEND_BUCKET/packages/klaytn/$KLAYTN_VERSION/
   154              done
   155    createrepo-update:
   156      steps:
   157        - run:
   158            name: "createrepo update"
   159            command: |
   160              aws s3 sync s3://$FRONTEND_BUCKET/packages/rhel/7/prod/ rhel/7/prod/
   161              createrepo --update rhel/7/prod
   162              aws s3 sync --delete rhel/7/prod/repodata/ s3://$FRONTEND_BUCKET/packages/rhel/7/prod/repodata/
   163    tag-verify:
   164      steps:
   165        - run:
   166            name: "Verify tag and file version match"
   167            command: |
   168              echo "tag version is " $CIRCLE_TAG
   169              KLAYTN_VERSION=$(go run build/rpm/main.go version)
   170              echo "version on version.go" $KLAYTN_VERSION
   171              if [ $KLAYTN_VERSION == ${CIRCLE_TAG%-*} ]; then
   172                echo "verification pass"
   173              else
   174                echo "It's not same version."
   175                exit 1
   176              fi
   177    tagger-verify:
   178      steps:
   179        - run:
   180            name: "Verify tag and file version match"
   181            command: |
   182                TAGGER=$(git for-each-ref --format='%(tagger)' refs/tags/$CIRCLE_TAG | sed 's/ .*//')
   183                if [ $TAGGER == 'circleci-klaytn' ]; then
   184                  echo "Pass! Tagger is circleci-klaytn"
   185                else
   186                  echo "only circleci-klaytn can tagging major version"
   187                  exit 1
   188                fi
   189    make-pr:
   190      steps:
   191        - run:
   192            name: "Push to release branch"
   193            command: |
   194                echo "push to release branch: /release/${CIRCLE_TAG%-*}"
   195                git checkout -b release/${CIRCLE_TAG%-*}
   196                git push origin release/${CIRCLE_TAG%-*}
   197        - run:
   198            name: "Install hub"
   199            command: |
   200                curl -sSLf https://github.com/github/hub/releases/download/v2.12.3/hub-linux-amd64-2.12.3.tgz | \
   201                  tar zxf - --strip-components=1 -C /tmp && \
   202                  sudo mv /tmp/bin/hub /usr/local/bin/hub
   203                type hub
   204        - run:
   205            name: "Create pull request"
   206            command: |
   207                version=$(hub pr list -s open -L 10 -f "%H%n")
   208                echo $version
   209                if [[ $version == *"release/${CIRCLE_TAG%-*}"* ]]; then
   210                  echo "PR already exist"
   211                else
   212                  echo "hub pull-request -m "[Master] release/$CIRCLE_TAG QA Signoff" -b $CIRCLE_PROJECT_USERNAME:master -h $CIRCLE_PROJECT_USERNAME:${CIRCLE_TAG%-*}"
   213                  echo -e "[Master] release/${CIRCLE_TAG%-*} QA Sign-off\n\nThis PR is automatically created by CI to release a new official version of $CIRCLE_PROJECT_REPONAME.\n\nWhen this PR is approved by QA team, a new version will be released." | hub pull-request -b $CIRCLE_PROJECT_USERNAME:master -h $CIRCLE_PROJECT_USERNAME:release/${CIRCLE_TAG%-*} -r $GITHUB_reviewer -l circleci -F-
   214                fi
   215        - run:
   216            name: "build announce"
   217            command: .circleci/scripts/build_announce.sh
   218    tagging-delete-branch:
   219      steps:
   220        - run:
   221            name: "Generate tag"
   222            command: |
   223                export version=$(go run build/rpm/main.go version) >> $BASH_ENV
   224                echo "git tag $version"
   225                git config --global user.email "team.devops@groundx.xyz"
   226                git config --global user.name "circleci-klaytn"
   227                git tag -a $version -m "$CIRCLE_STAGE"
   228                git push origin $version
   229        - run:
   230            name: "Delete release branch"
   231            command: |
   232                export version=$(go run build/rpm/main.go version) >> $BASH_ENV
   233                #delete release branch. it trigger by merge title
   234                if [[ "release/v" = $(git log --oneline -1 | grep -o "release/v") ]]; then
   235                  echo "Delete branch release/$version"
   236                  git push origin --delete release/$version
   237                else
   238                  echo "Need to delete branch manually"
   239                fi
   240    wait-other-containers-ready:
   241      steps:
   242        - run:
   243            name: "Waiting for Redis to be ready"
   244            command: |
   245              for i in `seq 1 10`;
   246              do
   247                nc -z 127.0.0.1 6379 && echo Success && exit 0
   248                echo -n .
   249                sleep 1
   250              done
   251              echo Failed waiting for Redis && exit 1
   252        - run:
   253            name: "Waiting for zookeeper to be ready"
   254            command: |
   255              for i in `seq 1 10`;
   256              do
   257                nc -z zookeeper 2181 && echo Success && exit 0
   258                echo -n .
   259                sleep 1
   260              done
   261              echo Failed waiting for zookeeper && exit 1
   262        - run:
   263            name: "Waiting for Kafka to be ready"
   264            command: |
   265              for i in `seq 1 10`;
   266              do
   267                nc -z kafka 9092 && echo Success && exit 0
   268                echo -n .
   269                sleep 1
   270              done
   271              echo Failed waiting for Kafka && exit 1
   272        - run:
   273            name: "Wait until localstack container running"
   274            command: |
   275              timeout 60 sh -c \
   276              'until nc -z localhost 4566; do
   277                echo "Waiting for Localstack ..."
   278                sleep 1
   279              done'
   280    notify-success:
   281      steps:
   282        - run:
   283            name: "notify slack when job success"
   284            command : |
   285              curl --data '{"text": "✅ Job *'$CIRCLE_JOB'* succeeded on *'$CIRCLE_BRANCH''$CIRCLE_TAG'*. Please see '$CIRCLE_BUILD_URL' for details."}' "$SLACK_WEBHOOK_URL"
   286            when: on_success
   287    notify-failure:
   288      steps:
   289        - run:
   290            name: "notify slack when job fail"
   291            command : |
   292              curl --data '{"text": "❌ Job *'$CIRCLE_JOB'* failed on *'$CIRCLE_BRANCH'*. Please see '$CIRCLE_BUILD_URL' for details."}' "$SLACK_WEBHOOK_URL"
   293            when: on_fail
   294    run-rpc:
   295      steps:
   296        - run:
   297            name: "Run rpc-tester on 1CN private network"
   298            no_output_timeout: 30m
   299            command: |
   300              make kcn 
   301              git clone https://$TEST_TOKEN@github.com/klaytn/klaytn-rpc-tester.git
   302              cd klaytn-rpc-tester
   303              cp ../build/bin/kcn script/cn/bin/
   304              cd script
   305              ./set_CNonly.sh
   306              cd ..
   307              pip3 install -r requirements.txt
   308              cp config_template.json config.json
   309              python3 main.py --protocol rpc
   310  jobs:
   311    build:
   312      machine:
   313        image: ubuntu-2004:202201-02
   314      working_directory: ~/go/src/github.com/klaytn/klaytn
   315      steps:
   316        - checkout
   317        - install-golang
   318        - pre-build
   319        - run:
   320            name: "Build"
   321            command: |
   322              export PATH=$HOME/go1.18/go/bin:$PATH
   323              make fmt
   324              make all
   325  
   326    test-linter:
   327      executor: test-executor
   328      steps:
   329        - install-golangci-lint
   330        - checkout
   331        - run:
   332            name: "Run golangci-lint"
   333            no_output_timeout: 30m
   334            command: golangci-lint run --new-from-rev=dev --presets format,performance -v --timeout 10m
   335  
   336    test-datasync:
   337      executor: test-others-executor
   338      steps:
   339        - install-netcat
   340        - checkout
   341        - wait-other-containers-ready
   342        - run:
   343            name: "Run test datasync"
   344            no_output_timeout: 30m
   345            command: make test-datasync
   346  
   347    test-networks:
   348      executor: test-executor
   349      steps:
   350        - checkout
   351        - run:
   352            name: "Run test networks"
   353            no_output_timeout: 30m
   354            command: make test-networks
   355  
   356    test-node:
   357      executor: test-executor
   358      steps:
   359        - checkout
   360        - run:
   361            name: "Run test node"
   362            no_output_timeout: 30m
   363            command: make test-node
   364  
   365    test-tests:
   366      executor: test-executor
   367      steps:
   368        - checkout
   369        - run:
   370            name: "Run test tests"
   371            no_output_timeout: 30m
   372            command: |
   373              git clone --depth 1 https://$TEST_TOKEN@github.com/klaytn/klaytn-tests.git tests/testdata
   374              make test-tests
   375  
   376    test-others:
   377      executor: test-others-executor
   378      steps:
   379        - install-netcat
   380        - checkout
   381        - wait-other-containers-ready
   382        - run:
   383            name: "Run test others"
   384            no_output_timeout: 30m
   385            command: |
   386              make test-others
   387      resource_class: xlarge
   388  
   389    test-rpc:
   390      executor: test-executor
   391      steps:
   392        - checkout
   393        - install-golang
   394        - pre-build
   395        - run-rpc
   396  
   397    pass-tests:
   398      executor: default
   399      steps:
   400        - run:
   401            name: "tests pass!"
   402            command: echo "tests pass!"
   403  
   404    tagger-verify:
   405      executor: default
   406      steps:
   407        - checkout
   408        - tagger-verify
   409  
   410    coverage:
   411      executor: test-others-executor
   412      resource_class: xlarge
   413      steps:
   414        - install-netcat
   415        - checkout
   416        - wait-other-containers-ready
   417        - run:
   418            shell: /bin/bash
   419            name: "Run coverage tests"
   420            command: |
   421              set -e
   422              export GOPATH=/go
   423              git clone --depth 1 https://$TEST_TOKEN@github.com/klaytn/klaytn-tests.git tests/testdata
   424              make cover
   425              mkdir -p /tmp/coverage_reports
   426              cp coverage_report.txt /tmp/coverage_reports/
   427              cp coverage_report.html /tmp/coverage_reports/
   428        - notify-failure
   429        - notify-success
   430        - store_artifacts:
   431            path: /tmp/coverage_reports
   432        - codecov/upload:
   433            file: /tmp/coverage_reports/coverage_*
   434  
   435    linters:
   436      executor: test-executor
   437      steps:
   438        - checkout
   439        - run:
   440            name: "Run linters"
   441            command: |
   442              make lint-try
   443              mkdir -p /tmp/linter_reports
   444              cp linter_report.txt /tmp/linter_reports/
   445        - notify-failure
   446        - notify-success
   447        - store_artifacts:
   448            path: /tmp/linter_reports
   449    
   450    rpc-tester-report:
   451      executor: test-executor
   452      steps:
   453        - checkout
   454        - install-golang
   455        - pre-build
   456        - run-rpc
   457        - notify-failure
   458        - notify-success
   459  
   460    packaging-linux:
   461      machine: 
   462        image: ubuntu-2004:202201-02
   463      resource_class: large
   464      working_directory: ~/go/src/github.com/klaytn/klaytn
   465      steps:
   466        - checkout
   467        - install-golang
   468        - pre-build
   469        - build-packaging
   470        - upload-repo
   471  
   472    packaging-linux-baobab:
   473      machine: 
   474        image: ubuntu-2004:202201-02
   475      resource_class: large
   476      working_directory: ~/go/src/github.com/klaytn/klaytn
   477      steps:
   478        - checkout
   479        - install-golang
   480        - pre-build
   481        - build-packaging:
   482            os-network: "tar-baobab-linux-amd64-all"
   483        - upload-repo:
   484            item: "kcn kpn ken"
   485  
   486    packaging-darwin:
   487      machine: 
   488        image: ubuntu-2004:202201-02
   489      resource_class: large
   490      working_directory: ~/go/src/github.com/klaytn/klaytn
   491      steps:
   492        - checkout
   493        - install-golang
   494        - pre-build
   495        - build-packaging:
   496            os-network: "tar-darwin-amd64-all"
   497        - upload-repo
   498  
   499    packaging-darwin-baobab:
   500      machine: 
   501        image: ubuntu-2004:202201-02
   502      resource_class: large
   503      working_directory: ~/go/src/github.com/klaytn/klaytn
   504      steps:
   505        - checkout
   506        - install-golang
   507        - pre-build
   508        - build-packaging:
   509            os-network: "tar-baobab-darwin-amd64-all"
   510        - upload-repo:
   511            item: "kcn kpn ken"
   512  
   513    rpm-tagged:
   514      executor: rpm-executor
   515      steps:
   516        - checkout
   517        - pre-build
   518        - rpm-tagging
   519  
   520    rpm-tagged-baobab:
   521      executor: rpm-executor
   522      steps:
   523        - checkout
   524        - pre-build
   525        - rpm-tagging-baobab
   526  
   527    deploy-rpm-public:
   528      executor: rpm-executor
   529      steps:
   530        - add_ssh_keys
   531        - createrepo-update
   532        - notify-failure
   533        - notify-success
   534  
   535    tag-verify:
   536      executor: test-executor
   537      steps:
   538        - checkout
   539        - tag-verify
   540  
   541    release-PR:
   542      executor: default
   543      steps:
   544        - add_ssh_keys
   545        - checkout
   546        - make-pr
   547        - notify-failure
   548        - notify-success
   549  
   550    major-tagging:
   551      executor: default
   552      steps:
   553        - add_ssh_keys
   554        - checkout
   555        - tagging-delete-branch
   556  
   557  workflows:
   558    version: 2
   559    build_n_packaging:
   560      jobs:
   561        - build:
   562            filters: *filter-version-not-release
   563        - test-datasync:
   564            filters: *filter-version-not-release
   565        - test-networks:
   566            filters: *filter-version-not-release
   567        - test-node:
   568            filters: *filter-version-not-release
   569        - test-tests:
   570            filters: *filter-version-not-release
   571        - test-others:
   572           filters: *filter-version-not-release
   573        - test-linter:
   574            filters: *filter-version-not-release
   575        - tagger-verify:
   576           filters:
   577              tags:
   578                only: /^v[0-9]+\.[0-9]+\.[0-9]+/
   579              branches:
   580                ignore: /.*/
   581        - test-rpc:
   582           filters: *filter-only-version-tag
   583  
   584        - pass-tests:
   585            requires:
   586              - build
   587              - test-datasync
   588              - test-networks
   589              - test-node
   590              - test-tests
   591              - test-others
   592              - tag-verify
   593              - tagger-verify
   594            filters: *filter-version-not-release
   595  
   596        - docker/publish: # for dev branch
   597            filters:
   598              branches:
   599                only: dev
   600            requires:
   601              - pass-tests
   602            extra_build_args: '--platform=linux/amd64'
   603            image: klaytn/klaytn
   604            tag: dev
   605            executor: docker/docker
   606            use-remote-docker: true
   607            remote-docker-version: 20.10.14
   608            use-buildkit: true
   609  
   610        - docker/publish: # for release versions
   611            filters:
   612              tags:
   613                only: /^v[0-9]+\.[0-9]+\.[0-9]+/
   614              branches:
   615                ignore: /.*/
   616            requires:
   617              - pass-tests
   618            extra_build_args: '--platform=linux/amd64'
   619            image: klaytn/klaytn
   620            tag: latest,$CIRCLE_TAG
   621            executor: docker/docker
   622            use-remote-docker: true
   623            remote-docker-version: 20.10.14
   624            use-buildkit: true
   625  
   626        - tag-verify:
   627            filters: *filter-only-version-tag
   628  
   629        - deploy-rpm-public:
   630            requires:
   631              - rpm-tagged
   632              - rpm-tagged-baobab
   633              - packaging-linux
   634              - packaging-linux-baobab
   635              - packaging-darwin
   636              - packaging-darwin-baobab
   637            filters:
   638              tags:
   639                only: /^v[0-9]+\.[0-9]+\.[0-9]/
   640              branches:
   641                ignore: /.*/
   642  
   643        - release-PR:
   644            requires:
   645              - rpm-tagged
   646              - rpm-tagged-baobab
   647              - packaging-linux
   648              - packaging-linux-baobab
   649              - packaging-darwin
   650              - packaging-darwin-baobab
   651            filters:
   652              tags:
   653                only: /^v[0-9]+\.[0-9]+\.[0-9]+-rc\.[0-9]+.*/
   654              branches:
   655                ignore: /.*/
   656  
   657        - rpm-tagged:
   658            filters: *filter-only-version-tag
   659            requires:
   660              - pass-tests
   661        - rpm-tagged-baobab:
   662            filters: *filter-only-version-tag
   663            requires:
   664              - pass-tests
   665        - packaging-linux:
   666            filters: *filter-only-version-tag
   667            requires:
   668              - pass-tests
   669        - packaging-linux-baobab:
   670            filters: *filter-only-version-tag
   671            requires:
   672              - pass-tests
   673        - packaging-darwin:
   674            filters: *filter-only-version-tag
   675            requires:
   676              - pass-tests
   677        - packaging-darwin-baobab:
   678            filters: *filter-only-version-tag
   679            requires:
   680              - pass-tests
   681  
   682        - major-tagging:
   683            filters:
   684              branches:
   685                only: master
   686  
   687    nightly-coverage:
   688      triggers:
   689        - schedule:
   690            cron: "0 18 * * *"
   691            filters:
   692              branches:
   693                only: dev
   694      jobs:
   695        - coverage
   696  
   697    nightly-linters:
   698      triggers:
   699        - schedule:
   700            cron: "0 19 * * *"
   701            filters:
   702              branches:
   703                only: dev
   704      jobs:
   705        - linters
   706    
   707    nightly-rpc:
   708      triggers:
   709        - schedule:
   710            cron: "0 20 * * *"
   711            filters:
   712              branches:
   713                only: dev
   714      jobs:
   715        - rpc-tester-report