storj.io/uplink@v1.13.0/Jenkinsfile (about)

     1  pipeline {
     2      agent none
     3  
     4      options {
     5            timeout(time: 35, unit: 'MINUTES')
     6      }
     7      environment {
     8          NPM_CONFIG_CACHE = '/tmp/npm/cache'
     9          COCKROACH_MEMPROF_INTERVAL=0
    10      }
    11      stages {
    12          stage('Build') {
    13              agent {
    14                  docker {
    15                      label 'main'
    16                      image 'storjlabs/ci:latest'
    17                      alwaysPull true
    18                      args '-u root:root --cap-add SYS_PTRACE -v "/tmp/gomod":/go/pkg/mod -v /tmp/golangci-lint:/root/.cache/golangci-lint -v /tmp/gocache:/root/.cache/go-build'
    19                  }
    20              }
    21              stages {
    22                  stage('Preparation') {
    23                      parallel {
    24                          stage('Checkout') {
    25                              steps {
    26                                  checkout scm
    27  
    28                                  sh 'mkdir -p .build'
    29                                  // make a backup of the mod file in case, for later linting
    30                                  sh 'cp go.mod .build/go.mod.orig'
    31                                  sh 'cp testsuite/go.mod .build/testsuite.go.mod.orig'
    32                              }
    33                          }
    34                          stage('Start databases') {
    35                              steps {
    36                                  sh 'service postgresql start'
    37  
    38                                  dir('.build') {
    39                                      sh 'cockroach start-single-node --insecure --store=type=mem,size=4GiB --listen-addr=localhost:26257 --http-addr=localhost:8086 --cache 1024MiB --max-sql-memory 1024MiB --background'
    40                                  }
    41                              }
    42                          }
    43                      }
    44                  }
    45  
    46                  stage('Verification') {
    47                      parallel {
    48                          stage('Lint') {
    49                              steps {
    50                                  sh 'check-copyright'
    51                                  sh 'check-large-files'
    52                                  sh 'check-imports ./...'
    53                                  sh 'check-peer-constraints'
    54                                  sh 'storj-protobuf --protoc=$HOME/protoc/bin/protoc lint'
    55                                  sh 'storj-protobuf --protoc=$HOME/protoc/bin/protoc check-lock'
    56                                  sh 'check-atomic-align ./...'
    57                                  sh 'check-monkit ./...'
    58                                  sh 'check-errs ./...'
    59                                  sh './scripts/check-dependencies.sh'
    60                                  sh 'staticcheck ./...'
    61                                  sh 'GOOS=linux GOARCH=arm staticcheck ./...'
    62                                  sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run'
    63                                  sh 'go-licenses check ./...'
    64                                  sh './scripts/check-libuplink-size.sh'
    65                                  sh 'check-mod-tidy -mod .build/go.mod.orig'
    66  
    67                                  dir("testsuite") {
    68                                      sh 'check-imports ./...'
    69                                      sh 'check-atomic-align ./...'
    70                                      sh 'check-monkit ./...'
    71                                      sh 'check-errs ./...'
    72                                      sh 'staticcheck ./...'
    73                                      sh 'golangci-lint --config /go/ci/.golangci.yml -j=2 run'
    74                                      sh 'check-mod-tidy -mod ../.build/testsuite.go.mod.orig'
    75                                  }
    76                              }
    77                          }
    78  
    79                          stage('Tests') {
    80                              environment {
    81                                  COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=./...' : ''}"
    82                              }
    83                              steps {
    84                                  sh 'go vet ./...'
    85                                  sh 'go test -parallel 4 -p 6 -vet=off $COVERFLAGS -timeout 20m -json -race ./... > .build/tests.json'
    86                              }
    87  
    88                              post {
    89                                  always {
    90                                      sh script: 'cat .build/tests.json | xunit -out .build/tests.xml', returnStatus: true
    91                                      sh script: 'cat .build/tests.json | tparse -all -slow 100', returnStatus: true
    92                                      archiveArtifacts artifacts: '.build/tests.json'
    93                                      junit '.build/tests.xml'
    94  
    95                                      script {
    96                                          if(fileExists(".build/coverprofile")){
    97                                              sh script: 'filter-cover-profile < .build/coverprofile > .build/clean.coverprofile', returnStatus: true
    98                                              sh script: 'gocov convert .build/clean.coverprofile > .build/cover.json', returnStatus: true
    99                                              sh script: 'gocov-xml  < .build/cover.json > .build/cobertura.xml', returnStatus: true
   100                                              cobertura coberturaReportFile: '.build/cobertura.xml'
   101                                          }
   102                                      }
   103                                  }
   104                              }
   105                          }
   106  
   107                          stage('Testsuite') {
   108                              environment {
   109                                  STORJ_TEST_COCKROACH = 'cockroach://root@localhost:26257/testcockroach?sslmode=disable'
   110                                  STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable'
   111                                  STORJ_TEST_COCKROACH_NODROP = 'true'
   112                                  STORJ_TEST_LOG_LEVEL = 'info'
   113                                  COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=../.build/testsuite_coverprofile -coverpkg=storj.io/uplink/...' : ''}"
   114                              }
   115                              steps {
   116                                  sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database testcockroach;\''
   117                                  sh 'psql -U postgres -c \'create database teststorj;\''
   118                                  dir('testsuite'){
   119                                      sh 'go vet ./...'
   120                                      sh 'go test -parallel 4 -p 6 -vet=off $COVERFLAGS -timeout 20m -json -race ./... > ../.build/testsuite.json'
   121                                  }
   122                              }
   123  
   124                              post {
   125                                  always {
   126                                      dir('testsuite'){
   127                                          sh script: 'cat ../.build/testsuite.json | xunit -out ../.build/testsuite.xml', returnStatus: true
   128                                      }
   129                                      sh script: 'cat .build/testsuite.json | tparse -all -slow 100', returnStatus: true
   130                                      archiveArtifacts artifacts: '.build/testsuite.json'
   131                                      junit '.build/testsuite.xml'
   132  
   133                                      script {
   134                                          if(fileExists(".build/testsuite_coverprofile")){
   135                                              sh script: 'filter-cover-profile < .build/testsuite_coverprofile > .build/clean.testsuite_coverprofile', returnStatus: true
   136                                              sh script: 'gocov convert .build/clean.testsuite_coverprofile > .build/testsuite_cover.json', returnStatus: true
   137                                              sh script: 'gocov-xml  < .build/testsuite_cover.json > .build/testsuite_cobertura.xml', returnStatus: true
   138                                              cobertura coberturaReportFile: '.build/testsuite_cobertura.xml'
   139                                          }
   140                                      }
   141                                  }
   142                              }
   143                          }
   144  
   145                          stage('Integration [storj/storj]') {
   146                              environment {
   147                                  STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj2?sslmode=disable'
   148                                  STORJ_TEST_COCKROACH = 'omit'
   149                                  // TODO add 'omit' for metabase STORJ_TEST_DATABASES
   150                                  STORJ_TEST_DATABASES = 'pg|pgx|postgres://postgres@localhost/testmetabase?sslmode=disable'
   151                              }
   152                              steps {
   153                                  sh 'psql -U postgres -c \'create database teststorj2;\''
   154                                  sh 'psql -U postgres -c \'create database testmetabase;\''
   155                                  dir('testsuite'){
   156                                      sh 'cp go.mod go-temp.mod'
   157                                      sh 'go vet -modfile go-temp.mod -mod=mod storj.io/storj/...'
   158                                      sh 'go test -modfile go-temp.mod -mod=mod -tags noembed -parallel 4 -p 6 -vet=off -timeout 20m -json storj.io/storj/... 2>&1 | tee ../.build/testsuite-storj.json | xunit -out ../.build/testsuite-storj.xml'
   159                                  }
   160                              }
   161  
   162                              post {
   163                                  always {
   164                                      sh script: 'cat .build/testsuite-storj.json | tparse -all -slow 100', returnStatus: true
   165                                      archiveArtifacts artifacts: '.build/testsuite-storj.json'
   166                                      junit '.build/testsuite-storj.xml'
   167                                  }
   168                              }
   169                          }
   170                          stage('Go Compatibility') {
   171                              steps {
   172                                  sh 'check-cross-compile -compiler "go,go.min" storj.io/uplink/...'
   173                              }
   174                          }
   175                      }
   176                  }
   177              }
   178              post {
   179                  always {
   180                      sh "chmod -R 777 ." // ensure Jenkins agent can delete the working directory
   181                      deleteDir()
   182                  }
   183              }
   184          }
   185  
   186          stage('Integration [rclone]') {
   187              agent {
   188                  node {
   189                      label 'ondemand'
   190                  }
   191              }
   192              steps {
   193                      echo 'Testing rclone integration'
   194                      sh './testsuite/scripts/rclone.sh'
   195              }
   196              post {
   197                  always {
   198                      zip zipFile: 'rclone-integration-tests.zip', archive: true, dir: '.build/rclone-integration-tests'
   199                      sh "chmod -R 777 ." // ensure Jenkins agent can delete the working directory
   200                      deleteDir()
   201                  }
   202              }
   203          }
   204      }
   205  }