github.com/vnforks/kid/v5@v5.22.1-0.20200408055009-b89d99c65676/.circleci/config.yml (about) 1 version: 2.1 2 3 orbs: 4 aws-s3: circleci/aws-s3@1.0.11 5 owasp: entur/owasp@0.0.10 6 7 executors: 8 ubuntu: 9 working_directory: ~/mattermost/ 10 machine: 11 image: "ubuntu-1604:201903-01" 12 environment: 13 COMPOSE_PROJECT_NAME: "circleci" 14 resource_class: "large" 15 16 jobs: 17 setup: 18 working_directory: ~/mattermost/mattermost-server 19 docker: 20 - image: mattermost/mattermost-build-webapp:oct-2-2018 21 resource_class: xlarge 22 steps: 23 - checkout 24 - run: | 25 cd ../ 26 git clone https://github.com/mattermost/mattermost-webapp.git 27 cd mattermost-webapp 28 git checkout $CIRCLE_BRANCH || git checkout master 29 export WEBAPP_GIT_COMMIT=$(git rev-parse HEAD) 30 echo "$WEBAPP_GIT_COMMIT" 31 export CURL_FAILED=0 32 curl -f -o ./dist.tar.gz https://pr-builds.mattermost.com/mattermost-webapp/commit/${WEBAPP_GIT_COMMIT}/mattermost-webapp.tar.gz && mkdir ./dist && tar -xvf ./dist.tar.gz -C ./dist --strip-components=1 || export CURL_FAILED=1 33 34 if [ $CURL_FAILED -eq 1 ] 35 then 36 npm ci && cd node_modules/mattermost-redux && npm install && npm run build && cd ../.. && make build 37 fi 38 - persist_to_workspace: 39 root: ~/mattermost 40 paths: 41 - mattermost-webapp 42 - mattermost-server 43 44 check-deps: 45 parameters: 46 cve_data_directory: 47 type: string 48 default: "~/.owasp/dependency-check-data" 49 working_directory: ~/mattermost/mattermost-server 50 executor: owasp/default 51 environment: 52 version_url: "https://jeremylong.github.io/DependencyCheck/current.txt" 53 executable_url: "https://dl.bintray.com/jeremy-long/owasp/dependency-check-VERSION-release.zip" 54 steps: 55 - attach_workspace: 56 at: ~/mattermost/ 57 - run: 58 name: Install Go 59 command: sudo apt-get update && sudo apt-get install golang 60 - owasp/with_commandline: 61 steps: 62 # Taken from https://github.com/entur/owasp-orb/blob/master/src/%40orb.yml#L349-L361 63 - owasp/generate_cache_keys: 64 cache_key: commmandline-default-cache-key-v6 65 - owasp/restore_owasp_cache 66 - run: 67 name: Update OWASP Dependency-Check Database 68 command: ~/.owasp/dependency-check/bin/dependency-check.sh --data << parameters.cve_data_directory >> --updateonly 69 - owasp/store_owasp_cache: 70 cve_data_directory: <<parameters.cve_data_directory>> 71 - run: 72 name: Run OWASP Dependency-Check Analyzer 73 command: | 74 ~/.owasp/dependency-check/bin/dependency-check.sh --data << parameters.cve_data_directory >> \ 75 --format ALL --noupdate --enableExperimental --suppression ./dependency-suppression.xml \ 76 --scan './**/*' || true 77 - owasp/collect_reports: 78 persist_to_workspace: false 79 - run: 80 name: Post results to Mattermost 81 command: ./scripts/depcheck-alert.sh 82 83 # TODO: enable this step when the i18n-extract works with mattermost-server only 84 # and not depend on both mm-server/enterprise 85 # check-i18n: 86 # docker: 87 # - image: circleci/golang:1.12 88 # working_directory: ~/mattermost/ 89 # steps: 90 # - attach_workspace: 91 # at: ~/mattermost/ 92 # - run: 93 # command: | 94 # cd mattermost-server 95 # cp i18n/en.json /tmp/en.json 96 # make i18n-extract 97 # diff /tmp/en.json i18n/en.json 98 99 check-app-layers: 100 docker: 101 - image: mattermost/mattermost-build-server:20200322_golang-1.14.1 102 working_directory: ~/mattermost 103 steps: 104 - attach_workspace: 105 at: ~/mattermost/ 106 - run: 107 command: | 108 cd mattermost-server 109 make app-layers 110 if [[ -n $(git status --porcelain) ]]; then echo "Please update the app layers using make app-layers"; exit 1; fi 111 112 build: 113 docker: 114 - image: mattermost/mattermost-build-server:20200322_golang-1.14.1 115 resource_class: xlarge 116 working_directory: ~/mattermost 117 steps: 118 - attach_workspace: 119 at: ~/mattermost/ 120 - run: 121 command: | 122 echo "Installing golangci-lint" 123 curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b /usr/local/bin v1.23.6 # Keep this in sync with Jenkinsfile.pr 124 echo "Installing mattermost-govet" 125 export GOBIN=${PWD}/mattermost-server/bin 126 GO111MODULE=off go get -u github.com/mattermost/mattermost-govet 127 export PATH=$PATH:$GOBIN 128 cd mattermost-server 129 make config-reset 130 make check-style BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' MM_NO_ENTERPRISE_LINT=true 131 make build BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' 132 make package BUILD_NUMBER='${CIRCLE_BRANCH}-${CIRCLE_BUILD_NUM}' 133 - store_artifacts: 134 path: ~/mattermost/mattermost-server/dist/mattermost-branch-linux-amd64.tar.gz 135 - store_artifacts: 136 path: ~/mattermost/mattermost-server/dist/mattermost-branch-osx-amd64.tar.gz 137 - store_artifacts: 138 path: ~/mattermost/mattermost-server/dist/mattermost-branch-windows-amd64.zip 139 - persist_to_workspace: 140 root: ~/mattermost 141 paths: 142 - mattermost-server 143 - mattermost-webapp 144 145 test: 146 executor: 147 name: ubuntu 148 parameters: 149 dbdriver: 150 type: string 151 dbsource: 152 type: string 153 steps: 154 - attach_workspace: 155 at: ~/mattermost 156 - run: 157 name: Run Docker compose 158 command: | 159 cd mattermost-server/build 160 docker-compose --no-ansi run --rm start_dependencies 161 cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; 162 docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; 163 docker-compose --no-ansi ps 164 background: true 165 - run: 166 name: Wait for docker compose 167 command: | 168 sleep 5 169 docker run --net circleci_k-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" 170 docker run --net circleci_k-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" 171 - run: 172 name: Run Tests 173 command: | 174 ulimit -n 8096 175 mkdir -p mattermost-server/client/plugins 176 docker run -it --net circleci_k-test \ 177 --env TEST_DATABASE_MYSQL_DSN="kuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8\u0026readTimeout=30s\u0026writeTimeout=30s" \ 178 --env TEST_DATABASE_POSTGRESQL_DSN="postgres://kuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" \ 179 --env TEST_DATABASE_MYSQL_ROOT_PASSWD=mostest \ 180 --env CI_INBUCKET_HOST=inbucket \ 181 --env CI_MINIO_HOST=minio \ 182 --env CI_INBUCKET_PORT=10080 \ 183 --env CI_MINIO_PORT=9000 \ 184 --env CI_INBUCKET_SMTP_PORT=10025 \ 185 --env CI_LDAP_HOST=openldap \ 186 --env IS_CI="true" \ 187 --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \ 188 --env MM_EMAILSETTINGS_SMTPPORT=10025 \ 189 --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \ 190 --env MM_SQLSETTINGS_DATASOURCE="<<parameters.dbsource>>" \ 191 --env MM_SQLSETTINGS_DRIVERNAME=<<parameters.dbdriver>> \ 192 --env GOBIN=/mattermost/mattermost-server/bin \ 193 -v ~/mattermost:/mattermost \ 194 -w /mattermost/mattermost-server \ 195 mattermost/mattermost-build-server:20200322_golang-1.14.1 \ 196 bash -c 'ulimit -n 8096; make test-server BUILD_NUMBER="$CIRCLE_BRANCH-$CIRCLE_PREVIOUS_BUILD_NUM" TESTFLAGS= TESTFLAGSEE=' 197 no_output_timeout: 1h 198 - run: 199 name: Capture docker logs 200 when: always 201 command: | 202 cd mattermost-server/build 203 # Capture docker logs 204 mkdir -p logs 205 docker-compose logs --tail="all" -t --no-color > logs/docker-compose_logs 206 docker ps -a --no-trunc > logs/docker_ps 207 docker stats -a --no-stream > logs/docker_stats 208 tar -czvf logs/docker_logs.tar.gz logs/docker-compose_logs logs/docker_ps logs/docker_stats 209 - store_artifacts: 210 path: ~/mattermost/mattermost-server/build/logs 211 - run: 212 when: always 213 command: | 214 cd mattermost-server 215 mkdir -p test-results 216 cp report.xml test-results 217 - store_test_results: 218 path: ~/mattermost/mattermost-server/test-results/ 219 - store_artifacts: 220 path: ~/mattermost/mattermost-server/test-results/ 221 222 test-schema: 223 executor: 224 name: ubuntu 225 steps: 226 - attach_workspace: 227 at: ~/mattermost 228 - run: 229 name: Run Docker compose 230 command: | 231 cd mattermost-server/build 232 docker-compose --no-ansi run --rm start_dependencies 233 cat ../tests/test-data.ldif | docker-compose --no-ansi exec -T openldap bash -c 'ldapadd -x -D "cn=admin,dc=mm,dc=test,dc=com" -w mostest'; 234 docker-compose --no-ansi exec -T minio sh -c 'mkdir -p /data/mattermost-test'; 235 docker-compose --no-ansi ps 236 background: true 237 - run: 238 name: Wait for docker compose 239 command: | 240 sleep 5 241 docker run --net circleci_k-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://mysql:3306; do echo waiting for mysql; sleep 5; done;" 242 docker run --net circleci_k-test appropriate/curl:latest sh -c "until curl --max-time 5 --output - http://elasticsearch:9200; do echo waiting for elasticsearch; sleep 5; done;" 243 - run: 244 name: Postgres schema migration validation 245 command: | 246 ulimit -n 8096 247 mkdir -p mattermost-server/client/plugins 248 cd mattermost-server/build 249 echo "Creating databases" 250 docker-compose --no-ansi exec -T postgres sh -c 'exec echo "CREATE DATABASE migrated; CREATE DATABASE latest;" | exec psql -U kuser mattermost_test' 251 echo "Importing postgres dump from version 5.0" 252 docker-compose --no-ansi exec -T postgres psql -U kuser -d migrated < ../scripts/mattermost-postgresql-5.0.sql 253 docker run -it --net circleci_k-test \ 254 --env CI_INBUCKET_HOST=inbucket \ 255 --env CI_MINIO_HOST=minio \ 256 --env CI_INBUCKET_PORT=10080 \ 257 --env CI_MINIO_PORT=9000 \ 258 --env CI_INBUCKET_SMTP_PORT=10025 \ 259 --env CI_LDAP_HOST=openldap \ 260 --env IS_CI="true" \ 261 --env MM_SQLSETTINGS_DATASOURCE="postgres://kuser:mostest@postgres:5432/migrated?sslmode=disable&connect_timeout=10" \ 262 --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \ 263 --env MM_SQLSETTINGS_DRIVERNAME=postgres \ 264 --env MM_EMAILSETTINGS_SMTPPORT=10025 \ 265 --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \ 266 -v ~/mattermost:/mattermost \ 267 -w /mattermost/mattermost-server \ 268 mattermost/mattermost-build-server:20200322_golang-1.14.1 \ 269 bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="postgres://kuser:mostest@postgres:5432/latest?sslmode=disable&connect_timeout=10" ARGS="version" run-cli' 270 echo "Generating dump" 271 docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d migrated -U kuser > migrated.sql 272 docker-compose --no-ansi exec -T postgres pg_dump --schema-only -d latest -U kuser > latest.sql 273 echo "Removing databases created for db comparison" 274 docker-compose --no-ansi exec -T postgres sh -c 'exec echo "DROP DATABASE migrated; DROP DATABASE latest;" | exec psql -U kuser mattermost_test' 275 276 echo "Generating diff" 277 diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) 278 no_output_timeout: 1h 279 - run: 280 name: MySQL schema migration validation 281 command: | 282 ulimit -n 8096 283 mkdir -p mattermost-server/client/plugins 284 cd mattermost-server/build 285 echo "Creating databases" 286 docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e "CREATE DATABASE migrated; CREATE DATABASE latest; GRANT ALL PRIVILEGES ON migrated.* TO kuser; GRANT ALL PRIVILEGES ON latest.* TO kuser" 287 echo "Importing mysql dump from version 5.0" 288 docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest < ../scripts/mattermost-mysql-5.0.sql 289 docker run -it --net circleci_k-test \ 290 --env CI_INBUCKET_HOST=inbucket \ 291 --env CI_MINIO_HOST=minio \ 292 --env CI_INBUCKET_PORT=10080 \ 293 --env CI_MINIO_PORT=9000 \ 294 --env CI_INBUCKET_SMTP_PORT=10025 \ 295 --env CI_LDAP_HOST=openldap \ 296 --env IS_CI="true" \ 297 --env MM_SQLSETTINGS_DATASOURCE="kuser:mostest@tcp(mysql:3306)/migrated?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" \ 298 --env MM_EMAILSETTINGS_SMTPSERVER=inbucket \ 299 --env MM_SQLSETTINGS_DRIVERNAME=mysql \ 300 --env MM_EMAILSETTINGS_SMTPPORT=10025 \ 301 --env MM_ELASTICSEARCHSETTINGS_CONNECTIONURL=http://elasticsearch:9200 \ 302 -v ~/mattermost:/mattermost \ 303 -w /mattermost/mattermost-server \ 304 mattermost/mattermost-build-server:20200322_golang-1.14.1 \ 305 bash -c 'ulimit -n 8096; make ARGS="version" run-cli && make MM_SQLSETTINGS_DATASOURCE="kuser:mostest@tcp(mysql:3306)/latest?charset=utf8mb4,utf8&readTimeout=30s&writeTimeout=30s" ARGS="version" run-cli' 306 307 echo "Ignoring known MySQL mismatch: ClassMembers.SchemeGuest" 308 docker-compose --no-ansi exec -T mysql mysql -D migrated -uroot -pmostest -e "ALTER TABLE ClassMembers DROP COLUMN SchemeGuest;" 309 docker-compose --no-ansi exec -T mysql mysql -D latest -uroot -pmostest -e "ALTER TABLE ClassMembers DROP COLUMN SchemeGuest;" 310 311 echo "Generating dump" 312 docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest migrated > migrated.sql 313 docker-compose --no-ansi exec -T mysql mysqldump --skip-opt --no-data --compact -u root -pmostest latest > latest.sql 314 315 echo "Removing databases created for db comparison" 316 docker-compose --no-ansi exec -T mysql mysql -uroot -pmostest -e 'DROP DATABASE migrated; DROP DATABASE latest' 317 318 echo "Generating diff" 319 diff migrated.sql latest.sql > diff.txt && echo "Both schemas are same" || (echo "Schema mismatch" && cat diff.txt && exit 1) 320 no_output_timeout: 1h 321 322 upload-s3-sha: 323 docker: 324 - image: 'circleci/python:2.7' 325 working_directory: ~/mattermost/enterprise 326 steps: 327 - attach_workspace: 328 at: ~/mattermost/ 329 - run: 330 command: | 331 cd ~/mattermost/mattermost-server/dist/ 332 rm -rf mattermost 333 - aws-s3/copy: 334 from: ~/mattermost/mattermost-server/dist/ 335 to: "s3://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/" 336 arguments: --acl public-read --cache-control "no-cache" --recursive 337 338 upload-s3: 339 docker: 340 - image: 'circleci/python:2.7' 341 working_directory: ~/mattermost/enterprise 342 steps: 343 - attach_workspace: 344 at: ~/mattermost/ 345 - run: 346 command: | 347 cd ~/mattermost/mattermost-server/dist/ 348 rm -rf mattermost 349 - aws-s3/copy: 350 from: ~/mattermost/mattermost-server/dist/ 351 to: s3://pr-builds.mattermost.com/mattermost-server/$(echo "${CIRCLE_BRANCH}" | sed 's/pull\//PR-/g')/ 352 arguments: --acl public-read --cache-control "no-cache" --recursive 353 354 build-docker: 355 working_directory: ~/ 356 docker: 357 - image: circleci/buildpack-deps:stretch 358 resource_class: xlarge 359 steps: 360 - attach_workspace: 361 at: . 362 - setup_remote_docker 363 - run: 364 command: | 365 export TAG="${CIRCLE_SHA1:0:7}" 366 cd mattermost-server 367 export MM_PACKAGE=https://pr-builds.mattermost.com/mattermost-server/commit/${CIRCLE_SHA1}/mattermost-branch-linux-amd64.tar.gz 368 docker build --build-arg MM_PACKAGE=$MM_PACKAGE -t mattermost/mattermost-branch-edition:${TAG} build 369 echo $DOCKER_PASSWORD | docker login --username $DOCKER_USERNAME --password-stdin 370 docker push mattermost/mattermost-branch-edition:${TAG} 371 372 trigger-ee-tests: 373 machine: true 374 steps: 375 - run: 376 name: Trigger enterprise 377 command: | 378 curl \ 379 --request POST \ 380 --url https://circleci.com/api/v2/project/gh/mattermost/enterprise/pipeline \ 381 --user ${CIRCLECI_TOKEN} \ 382 --data parameters[external_branch]=$CIRCLE_BRANCH \ 383 --data parameters[external_sha]=$CIRCLE_SHA1 384 385 workflows: 386 version: 2 387 untagged-build: 388 jobs: 389 - setup: 390 filters: 391 branches: 392 ignore: 393 - master 394 - /^release-.*/ 395 - check-deps: 396 context: sast-webhook 397 requires: 398 - setup 399 # - check-i18n: 400 # requires: 401 # - setup 402 - check-app-layers: 403 requires: 404 - setup 405 - build: 406 requires: 407 - check-app-layers 408 - upload-s3-sha: 409 context: mattermost-ci-pr-builds-s3 410 requires: 411 - build 412 - upload-s3: 413 context: mattermost-ci-pr-builds-s3 414 requires: 415 - build 416 - build-docker: 417 context: matterbuild-docker 418 requires: 419 - upload-s3-sha 420 - test: 421 name: test-mysql 422 dbdriver: mysql 423 dbsource: "kuser:mostest@tcp(mysql:3306)/mattermost_test?charset=utf8mb4,utf8" 424 requires: 425 - check-app-layers 426 - test: 427 name: test-postgres 428 dbdriver: postgres 429 dbsource: "postgres://kuser:mostest@postgres:5432/mattermost_test?sslmode=disable&connect_timeout=10" 430 requires: 431 - check-app-layers 432 - test-schema: 433 requires: 434 - check-app-layers 435 - trigger-ee-tests: 436 context: matterbuild-circleci-token 437 requires: 438 - test-mysql 439 - test-postgres 440 - test-schema 441 - upload-s3 442 - upload-s3-sha 443 - build-docker 444 filters: 445 branches: 446 only: /^pull\/.*$/