github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/kettle/stream_test.py (about)

     1  #!/usr/bin/env python2
     2  
     3  # Copyright 2017 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  # pylint: disable=missing-docstring
    18  
    19  import unittest
    20  
    21  import stream
    22  import make_db_test
    23  import model
    24  
    25  
    26  class FakeSub(object):
    27      def __init__(self, pulls):
    28          self.pulls = pulls
    29          self.trace = []
    30  
    31      def pull(self, return_immediately=False, **_kwargs):
    32          self.trace.append(['pull', return_immediately])
    33          return self.pulls.pop(0)
    34  
    35      def acknowledge(self, acks):
    36          self.trace.append(['ack', acks])
    37  
    38      def modify_ack_deadline(self, acks, time):
    39          self.trace.append(['modify-ack', acks, time])
    40  
    41  
    42  class FakeTable(object):
    43      def __init__(self, name, schema, trace=None):
    44          self.name = name
    45          self.schema = schema
    46          self.trace = [] if trace is None else trace
    47  
    48      def insert_data(self, *args, **kwargs):
    49          self.trace.append(['insert-data', args, kwargs])
    50          return []
    51  
    52  
    53  class Attrs(object):
    54      def __init__(self, attributes):
    55          self.attributes = attributes
    56  
    57  
    58  class FakeSchemaField(object):
    59      def __init__(self, **kwargs):
    60          self.__dict__ = kwargs
    61  
    62  
    63  class StreamTest(unittest.TestCase):
    64      def test_main(self):
    65          # It's easier to run a full integration test with stubbed-out
    66          # external interfaces and validate the trace than it is to test
    67          # each individual piece.
    68          # The components are mostly tested in make_*_test.py.
    69  
    70          db = model.Database(':memory:')
    71          fakesub = FakeSub([
    72              [
    73                  ('a', Attrs({'eventType': 'OBJECT_DELETE'})),
    74              ],
    75              [
    76                  ('b', Attrs({
    77                      'eventType': 'OBJECT_FINALIZE',
    78                      'objectId': 'logs/fake/123/finished.json',
    79                      'bucketId': 'kubernetes-jenkins'})),
    80              ],
    81              [],
    82              [
    83                  ('c', Attrs({
    84                      'eventType': 'OBJECT_FINALIZE',
    85                      'objectId': 'logs/fake/123/finished.json',
    86                      'bucketId': 'kubernetes-jenkins'})),
    87              ],
    88              [],
    89              [
    90                  ('d', Attrs({
    91                      'eventType': 'OBJECT_FINALIZE',
    92                      'objectId': 'logs/fake/124/started.json'})),
    93              ],
    94              [],
    95          ])
    96          faketable = FakeTable('day', stream.load_schema(FakeSchemaField), fakesub.trace)
    97          tables = {'day': (faketable, 'incr')}
    98          stream.main(
    99              db, fakesub, tables, make_db_test.MockedClient, [1, 0, 0, 0].pop)
   100  
   101          # uncomment if the trace changes
   102          # import pprint; pprint.pprint(fakesub.trace)
   103          # self.maxDiff = 3000
   104  
   105          now = make_db_test.MockedClient.NOW
   106  
   107          self.assertEqual(
   108              fakesub.trace,
   109              [['pull', False], ['pull', True], ['pull', True],
   110               ['ack', ['a']],
   111               ['modify-ack', ['b'], 180],
   112               ['ack', ['b']],
   113               ['insert-data',
   114                ([[5,
   115                   now - 5,
   116                   now,
   117                   True,
   118                   u'SUCCESS',
   119                   None,
   120                   u'gs://kubernetes-jenkins/logs/fake/123',
   121                   u'fake',
   122                   123,
   123                   [],
   124                   [{'name': 'Foo', 'time': 3.0},
   125                    {'failed': True,
   126                     'failure_text': 'stacktrace',
   127                     'name': 'Bad',
   128                     'time': 4.0}],
   129                   2,
   130                   1,
   131                   None]],
   132                 [1]),
   133                {'skip_invalid_rows': True}],
   134               ['pull', False], ['pull', True],
   135               ['modify-ack', ['c'], 180],
   136               ['ack', ['c']],
   137               ['pull', False], ['pull', True],
   138               ['ack', ['d']]])
   139  
   140  
   141  if __name__ == '__main__':
   142      unittest.main()