github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/kettle/update.py (about)

     1  #!/usr/bin/env python
     2  
     3  # Copyright 2017 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  
    18  import os
    19  
    20  
    21  def call(cmd):
    22      print '+', cmd
    23      status = os.system(cmd)
    24      if status:
    25          raise OSError('invocation failed')
    26  
    27  
    28  def main():
    29      call('time python make_db.py --buckets buckets.yaml --junit --threads 32')
    30  
    31      bq_cmd = 'bq load --source_format=NEWLINE_DELIMITED_JSON --max_bad_records=1000'
    32      mj_cmd = 'pypy make_json.py'
    33  
    34      mj_ext = ''
    35      bq_ext = ''
    36      try:
    37          call(mj_cmd + ' --days 1 --assert-oldest 1.9')
    38      except OSError:
    39          # cycle daily/weekly tables
    40          bq_ext = ' --replace'
    41          mj_ext = ' --reset-emitted'
    42  
    43      call(mj_cmd + mj_ext + ' --days 1 | pv | gzip > build_day.json.gz')
    44      call(bq_cmd + bq_ext + ' k8s-gubernator:build.day build_day.json.gz schema.json')
    45  
    46      call(mj_cmd + mj_ext + ' --days 7 | pv | gzip > build_week.json.gz')
    47      call(bq_cmd + bq_ext + ' k8s-gubernator:build.week build_week.json.gz schema.json')
    48  
    49      call(mj_cmd + ' | pv | gzip > build_all.json.gz')
    50      call(bq_cmd + ' k8s-gubernator:build.all build_all.json.gz schema.json')
    51  
    52      call('python stream.py --poll kubernetes-jenkins/gcs-changes/kettle '
    53           ' --dataset k8s-gubernator:build --tables all:0 day:1 week:7 --stop_at=1')
    54  
    55  
    56  if __name__ == '__main__':
    57      os.chdir(os.path.dirname(__file__))
    58      os.environ['TZ'] = 'America/Los_Angeles'
    59      main()