github.com/mattermosttest/mattermost-server/v5@v5.0.0-20200917143240-9dfa12e121f9/.circleci/config.yml (about)

     1  version: 2.1
     2  
     3  orbs:
     4    aws-s3: circleci/aws-s3@1.0.11
     5    owasp: entur/owasp@0.0.10
     6  
     7  executors:
     8    ubuntu:
     9      working_directory: ~/mattermost/
    10      resource_class: xlarge
    11      machine:
    12        image: "ubuntu-1604:201903-01"
    13      environment:
    14        COMPOSE_PROJECT_NAME: "circleci"
    15  
    16  jobs:
    17    setup:
    18      working_directory: ~/mattermost/mattermost-server
    19      docker:
    20        - image: mattermost/mattermost-build-webapp:oct-2-2018
    21      steps:
    22        - checkout
    23        - run: |
    24            cd ../
    25            git clone https://github.com/mattermost/mattermost-webapp.git
    26            cd mattermost-webapp
    27            git checkout $CIRCLE_BRANCH || git checkout master
    28            export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD)
    29            echo "$WEBAPP_GIT_COMMIT"
    30  
    31            trap 'npm ci && make build' ERR
    32            curl -f -o ./dist.tar.gz https://pr-builds.mattermost.com/mattermost-webapp/commit/${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz
    33            mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1
    34            trap - ERR
    35        - persist_to_workspace:
    36            root: ~/mattermost
    37            paths:
    38              - mattermost-webapp
    39              - mattermost-server
    40  
    41    check-deps:
    42      parameters:
    43        cve_data_directory:
    44          type: string
    45          default: "~/.owasp/dependency-check-data"
    46      working_directory: ~/mattermost/mattermost-server
    47      executor: owasp/default
    48      environment:
    49        version_url: "https://jeremylong.github.io/DependencyCheck/current.txt"
    50        executable_url: "https://dl.bintray.com/jeremy-long/owasp/dependency-check-VERSION-release.zip"
    51      steps:
    52        - attach_workspace:
    53            at: ~/mattermost/
    54        - run:
    55            name: Checkout config
    56            command: cd .. && git clone https://github.com/mattermost/security-automation-config
    57        - run:
    58            name: Install Go
    59            command: sudo apt-get update && sudo apt-get install golang
    60        - owasp/with_commandline:
    61            steps:
    62              # Taken from https://github.com/entur/owasp-orb/blob/master/src/%40orb.yml#L349-L361
    63              - owasp/generate_cache_keys:
    64                  cache_key: commmandline-default-cache-key-v6
    65              - owasp/restore_owasp_cache
    66              - run:
    67                  name: Update OWASP Dependency-Check Database
    68                  command: ~/.owasp/dependency-check/bin/dependency-check.sh --data << parameters.cve_data_directory >> --updateonly
    69              - owasp/store_owasp_cache:
    70                  cve_data_directory: <<parameters.cve_data_directory>>
    71              - run:
    72                  name: Run OWASP Dependency-Check Analyzer
    73                  command: |
    74                    ~/.owasp/dependency-check/bin/dependency-check.sh \
    75                      --data << parameters.cve_data_directory >> --format ALL --noupdate --enableExperimental \
    76                      --propertyfile ../security-automation-config/dependency-check/dependencycheck.properties \
    77                      --suppression ../security-automation-config/dependency-check/suppression.xml \
    78                      --suppression ../security-automation-config/dependency-check/suppression.$CIRCLE_PROJECT_REPONAME.xml \
    79                      --scan './**/*' || true
    80              - owasp/collect_reports:
    81                  persist_to_workspace: false
    82        - run:
    83            name: Post results to Mattermost
    84            command: go run ../security-automation-config/dependency-check/post_results.go
    85  
    86    # TODO: enable this step when the i18n-extract works with mattermost-server only
    87    # and not depend on both mm-server/enterprise
    88    # check-i18n:
    89    #   docker:
    90    #   - image: circleci/golang:1.12
    91    #   working_directory: ~/mattermost/
    92    #   steps:
    93    #     - attach_workspace:
    94    #         at: ~/mattermost/
    95    #     - run:
    96    #         command: |
    97    #           cd mattermost-server
    98    #           cp i18n/en.json /tmp/en.json
    99    #           make i18n-extract
   100    #           diff /tmp/en.json i18n/en.json
   101  
   102    check-app-layers:
   103      docker:
   104        - image: mattermost/mattermost-build-server:20200731_golang-1.14.6
   105      working_directory: ~/mattermost
   106      steps:
   107        - attach_workspace:
   108            at: ~/mattermost/
   109        - run:
   110            command: |
   111              cd mattermost-server
   112              make app-layers
   113              if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi
   114  
   115    check-store-layers:
   116      docker:
   117        - image: mattermost/mattermost-build-server:20200731_golang-1.14.6
   118      working_directory: ~/mattermost
   119      steps:
   120        - attach_workspace:
   121            at: ~/mattermost/
   122        - run:
   123            command: |
   124              cd mattermost-server
   125              make store-layers
   126              if [[ -n $(git status --porcelain) ]]; then echo "Please update the store layers using make store-layers"; exit 1; fi
   127  
   128    build-api-spec:
   129      docker:
   130        - image: circleci/node:lts
   131      working_directory: ~/mattermost
   132      steps:
   133        - attach_workspace:
   134            at: ~/mattermost/
   135        - run:
   136            command: |
   137              git clone https://github.com/mattermost/mattermost-api-reference.git
   138              cd mattermost-api-reference
   139              echo "Trying to checkout the same branch on mattermost-api-reference as mattermost-server"
   140              git checkout ${CIRCLE_BRANCH} || true
   141              make build
   142        - persist_to_workspace:
   143            root: ~/mattermost
   144            paths:
   145              - mattermost-api-reference
   146  
   147    build:
   148      docker:
   149        - image: mattermost/mattermost-build-server:20200731_golang-1.14.6
   150      resource_class: xlarge
   151      working_directory: ~/mattermost
   152      steps:
   153        - attach_workspace:
   154            at: ~/mattermost/
   155        - run:
   156            command: |
   157              echo "Installing golangci-lint"
   158              curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b /usr/local/bin v1.25.1
   159              echo "Installing mattermost-govet"
   160              export GOBIN=${PWD}/mattermost-server/bin
   161              GO111MODULE=off go get -u github.com/mattermost/mattermost-govet
   162              export PATH=$PATH:$GOBIN
   163              cd mattermost-server
   164              make config-reset
   165              make check-style BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true MM_VET_OPENSPEC_PATH='${PWD}/../mattermost-api-reference/v4/html/static/mattermost-openapi-v4.yaml'
   166              make build BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
   167              make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}'
   168              curl -sL https://sentry.io/get-cli/ | bash
   169              export VERSION=$(sentry-cli releases propose-version)
   170              sentry-cli --log-level=debug releases new  --finalize -p mattermost-server $VERSION
   171              sentry-cli --log-level=debug releases set-commits --auto $VERSION
   172        - store_artifacts:
   173            path: ~/mattermost/mattermost-server/dist/mattermost-team-linux-amd64.tar.gz
   174        - store_artifacts:
   175            path: ~/mattermost/mattermost-server/dist/mattermost-team-osx-amd64.tar.gz
   176        - store_artifacts:
   177            path: ~/mattermost/mattermost-server/dist/mattermost-team-windows-amd64.zip
   178        - persist_to_workspace:
   179            root: ~/mattermost
   180            paths:
   181              - mattermost-server
   182              - mattermost-webapp
   183  
   184    test:
   185      executor:
   186        name: ubuntu
   187      parameters:
   188        dbdriver:
   189          type: string
   190        dbsource:
   191          type: string
   192      steps:
   193        - attach_workspace:
   194            at: ~/mattermost
   195        - run:
   196            name: Run Docker compose
   197            command: |
   198              cd mattermost-server/build
   199              docker-compose --no-ansi run --rm start_dependencies
   200              cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
   201              docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
   202              docker-compose --no-ansi ps
   203            background: true
   204        - run:
   205            name: Wait for docker compose
   206            command: |
   207              sleep 5
   208              docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
   209              docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
   210        - run:
   211            name: Run Tests
   212            command: |
   213              ulimit -n 8096
   214              mkdir -p mattermost-server/client/plugins
   215              docker run -it --net circleci_mm-test \
   216                --env TEST_DATABASE_MYSQL_DSN="mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \
   217                --env TEST_DATABASE_POSTGRESQL_DSN="postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \
   218                --env TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest \
   219                --env CI_INBUCKET_HOST=inbucket \
   220                --env CI_MINIO_HOST=minio \
   221                --env CI_INBUCKET_PORT=10080 \
   222                --env CI_MINIO_PORT=9000 \
   223                --env CI_INBUCKET_SMTP_PORT=10025 \
   224                --env CI_LDAP_HOST=openldap \
   225                --env IS_CI="true" \
   226                --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \
   227                --env MM_EMAILSETTINGS_SMTPPORT=10025 \
   228                --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \
   229                --env MM_SQLSETTINGS_DATASOURCE="<<parameters.dbsource>>" \
   230                --env MM_SQLSETTINGS_DRIVERNAME=<<parameters.dbdriver>> \
   231                --env GOBIN=/mattermost/mattermost-server/bin \
   232                -v ~/mattermost:/mattermost \
   233                -w /mattermost/mattermost-server \
   234                mattermost/mattermost-build-server:20200731_golang-1.14.6 \
   235                bash -c 'ulimit -n 8096; make test-server BUILD_NUMBER="$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM" TESTFLAGS= TESTFLAGSEE='
   236            no_output_timeout: 1h
   237        - run:
   238            name: Capture docker logs
   239            when: always
   240            command: |
   241              cd mattermost-server/build
   242              # Capture docker logs
   243              mkdir -p logs
   244              docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs
   245              docker ps -a --no-trunc > logs/docker_ps
   246              docker stats -a --no-stream > logs/docker_stats
   247              tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats
   248        - store_artifacts:
   249            path: ~/mattermost/mattermost-server/build/logs
   250        - run:
   251            when: always
   252            command: |
   253              cd mattermost-server
   254              mkdir -p test-results
   255              cp report.xml test-results
   256        - store_test_results:
   257            path: ~/mattermost/mattermost-server/test-results/
   258        - store_artifacts:
   259            path: ~/mattermost/mattermost-server/test-results/
   260  
   261    test-schema:
   262      executor:
   263        name: ubuntu
   264      steps:
   265        - attach_workspace:
   266            at: ~/mattermost
   267        - run:
   268            name: Run Docker compose
   269            command: |
   270              cd mattermost-server/build
   271              docker-compose --no-ansi run --rm start_dependencies
   272              cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest';
   273              docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test';
   274              docker-compose --no-ansi ps
   275            background: true
   276        - run:
   277            name: Wait for docker compose
   278            command: |
   279              sleep 5
   280              docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;"
   281              docker run --net circleci_mm-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;"
   282        - run:
   283            name: Postgres schema migration validation
   284            command: |
   285              ulimit -n 8096
   286              mkdir -p mattermost-server/client/plugins
   287              cd mattermost-server/build
   288              echo "Creating databases"
   289              docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U mmuser mattermost_test'
   290              echo "Importing postgres dump from version 5.0"
   291              docker-compose --no-ansi exec -T postgres psql -U mmuser -d migrated < ../scripts/mattermost-postgresql-5.0.sql
   292              docker run -it --net circleci_mm-test \
   293                --env CI_INBUCKET_HOST=inbucket \
   294                --env CI_MINIO_HOST=minio \
   295                --env CI_INBUCKET_PORT=10080 \
   296                --env CI_MINIO_PORT=9000 \
   297                --env CI_INBUCKET_SMTP_PORT=10025 \
   298                --env CI_LDAP_HOST=openldap \
   299                --env IS_CI="true" \
   300                --env MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \
   301                --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \
   302                --env MM_SQLSETTINGS_DRIVERNAME=postgres \
   303                --env MM_EMAILSETTINGS_SMTPPORT=10025 \
   304                --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \
   305                -v ~/mattermost:/mattermost \
   306                -w /mattermost/mattermost-server \
   307                mattermost/mattermost-build-server:20200731_golang-1.14.6 \
   308                bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="postgres://mmuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10" ARGS="version" run-cli'
   309              echo "Generating dump"
   310              docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U mmuser > migrated.sql
   311              docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U mmuser > latest.sql
   312              echo "Removing databases created for db comparison"
   313              docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U mmuser mattermost_test'
   314  
   315              echo "Generating diff"
   316              diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
   317            no_output_timeout: 1h
   318        - run:
   319            name: MySQL schema migration validation
   320            command: |
   321              ulimit -n 8096
   322              mkdir -p mattermost-server/client/plugins
   323              cd mattermost-server/build
   324              echo "Creating databases"
   325              docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO mmuser; GRANT ALL PRIVILEGES ON latest.* TO mmuser"
   326              echo "Importing mysql dump from version 5.0"
   327              docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-5.0.sql
   328              docker run -it --net circleci_mm-test \
   329                --env CI_INBUCKET_HOST=inbucket \
   330                --env CI_MINIO_HOST=minio \
   331                --env CI_INBUCKET_PORT=10080 \
   332                --env CI_MINIO_PORT=9000 \
   333                --env CI_INBUCKET_SMTP_PORT=10025 \
   334                --env CI_LDAP_HOST=openldap \
   335                --env IS_CI="true" \
   336                --env MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \
   337                --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \
   338                --env MM_SQLSETTINGS_DRIVERNAME=mysql \
   339                --env MM_EMAILSETTINGS_SMTPPORT=10025 \
   340                --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \
   341                -v ~/mattermost:/mattermost \
   342                -w /mattermost/mattermost-server \
   343                mattermost/mattermost-build-server:20200731_golang-1.14.6 \
   344                bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="mmuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" ARGS="version" run-cli'
   345  
   346              echo "Ignoring known MySQL mismatch: ChannelMembers.SchemeGuest"
   347              docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
   348              docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ChannelMembers DROP COLUMN SchemeGuest;"
   349  
   350              echo "Generating dump"
   351              docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql
   352              docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql
   353  
   354              echo "Removing databases created for db comparison"
   355              docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest'
   356  
   357              echo "Generating diff"
   358              diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1)
   359            no_output_timeout: 1h
   360  
   361    upload-s3-sha:
   362      docker:
   363        - image: 'circleci/python:2.7'
   364      working_directory: ~/mattermost/enterprise
   365      steps:
   366        - attach_workspace:
   367            at: ~/mattermost/
   368        - run:
   369            command: |
   370              cd ~/mattermost/mattermost-server/dist/
   371              rm -rf mattermost
   372        - aws-s3/copy:
   373            from: ~/mattermost/mattermost-server/dist/
   374            to: "s3://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/"
   375            arguments: --acl public-read --cache-control "no-cache" --recursive
   376  
   377    upload-s3:
   378      docker:
   379      - image: 'circleci/python:2.7'
   380      working_directory: ~/mattermost/enterprise
   381      steps:
   382        - attach_workspace:
   383            at: ~/mattermost/
   384        - run:
   385            command: |
   386              cd ~/mattermost/mattermost-server/dist/
   387              rm -rf mattermost
   388        - aws-s3/copy:
   389            from: ~/mattermost/mattermost-server/dist/
   390            to: s3://pr-builds.mattermost.com/mattermost-server/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/
   391            arguments: --acl public-read --cache-control "no-cache" --recursive
   392  
   393    build-docker:
   394      working_directory: ~/
   395      docker:
   396        - image: circleci/buildpack-deps:stretch
   397      resource_class: xlarge
   398      steps:
   399        - attach_workspace:
   400            at: .
   401        - setup_remote_docker
   402        - run:
   403            command: |
   404              export TAG="${CIRCLE_SHA1:0:7}"
   405              cd mattermost-server
   406              export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-team-linux-amd64.tar.gz
   407              docker build --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mattermost-team-edition:${TAG} build
   408              echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin
   409              docker push mattermost/mattermost-team-edition:${TAG}
   410  
   411  workflows:
   412    version: 2
   413    untagged-build:
   414      jobs:
   415        - setup:
   416            filters:
   417              branches:
   418                  ignore:
   419                    - master
   420                    - /^release-.*/
   421        - check-deps:
   422            context: sast-webhook
   423            requires:
   424              - setup
   425        # - check-i18n:
   426        #     requires:
   427        #       - setup
   428        - check-app-layers:
   429            requires:
   430              - setup
   431        - check-store-layers:
   432            requires:
   433              - setup
   434        - build-api-spec:
   435            requires:
   436              - setup
   437        - build:
   438            context: matterbuild-sentry
   439            requires:
   440              - check-app-layers
   441              - check-store-layers
   442              - build-api-spec
   443        - upload-s3-sha:
   444            context: mattermost-ci-pr-builds-s3
   445            requires:
   446              - build
   447        - upload-s3:
   448            context: mattermost-ci-pr-builds-s3
   449            requires:
   450              - build
   451        - build-docker:
   452            context: matterbuild-docker
   453            requires:
   454              - upload-s3-sha
   455        - test:
   456            name: test-mysql
   457            dbdriver: mysql
   458            dbsource: "mmuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8"
   459            requires:
   460              - check-app-layers
   461              - check-store-layers
   462        - test:
   463            name: test-postgres
   464            dbdriver: postgres
   465            dbsource: "postgres://mmuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10"
   466            requires:
   467              - check-app-layers
   468              - check-store-layers
   469        - test-schema:
   470            requires:
   471              - check-app-layers
   472              - check-store-layers