github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/utils/thread_pool_executor_test.py (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  """Unit tests for UnboundedThreadPoolExecutor."""
    19  
    20  # pytype: skip-file
    21  
    22  import itertools
    23  import threading
    24  import time
    25  import traceback
    26  import unittest
    27  
    28  from apache_beam.utils import thread_pool_executor
    29  from apache_beam.utils.thread_pool_executor import UnboundedThreadPoolExecutor
    30  
    31  
    32  class UnboundedThreadPoolExecutorTest(unittest.TestCase):
    33    def setUp(self):
    34      self._lock = threading.Lock()
    35      self._worker_idents = []
    36  
    37    def append_and_sleep(self, sleep_time):
    38      with self._lock:
    39        self._worker_idents.append(threading.current_thread().ident)
    40      time.sleep(sleep_time)
    41  
    42    def raise_error(self, message):
    43      raise ValueError(message)
    44  
    45    def test_shutdown_with_no_workers(self):
    46      with UnboundedThreadPoolExecutor():
    47        pass
    48  
    49    def test_shutdown_with_fast_workers(self):
    50      futures = []
    51      with UnboundedThreadPoolExecutor() as executor:
    52        for _ in range(0, 5):
    53          futures.append(executor.submit(self.append_and_sleep, 0.01))
    54  
    55      for future in futures:
    56        future.result(timeout=10)
    57  
    58      with self._lock:
    59        self.assertEqual(5, len(self._worker_idents))
    60  
    61    def test_shutdown_with_slow_workers(self):
    62      futures = []
    63      with UnboundedThreadPoolExecutor() as executor:
    64        for _ in range(0, 5):
    65          futures.append(executor.submit(self.append_and_sleep, 1))
    66  
    67      for future in futures:
    68        future.result(timeout=10)
    69  
    70      with self._lock:
    71        self.assertEqual(5, len(self._worker_idents))
    72  
    73    def test_worker_reuse(self):
    74      futures = []
    75      with UnboundedThreadPoolExecutor() as executor:
    76        for _ in range(0, 5):
    77          futures.append(executor.submit(self.append_and_sleep, 0.01))
    78        time.sleep(3)
    79        for _ in range(0, 5):
    80          futures.append(executor.submit(self.append_and_sleep, 0.01))
    81  
    82      for future in futures:
    83        future.result(timeout=10)
    84  
    85      with self._lock:
    86        self.assertEqual(10, len(self._worker_idents))
    87        self.assertTrue(len(set(self._worker_idents)) < 10)
    88  
    89    def test_exception_propagation(self):
    90      with UnboundedThreadPoolExecutor() as executor:
    91        future = executor.submit(self.raise_error, 'footest')
    92  
    93      try:
    94        future.result()
    95      except Exception:
    96        message = traceback.format_exc()
    97      else:
    98        raise AssertionError('expected exception not raised')
    99  
   100      self.assertIn('footest', message)
   101      self.assertIn('raise_error', message)
   102  
   103    def test_map(self):
   104      with UnboundedThreadPoolExecutor() as executor:
   105        executor.map(self.append_and_sleep, itertools.repeat(0.01, 5))
   106  
   107      with self._lock:
   108        self.assertEqual(5, len(self._worker_idents))
   109  
   110    def test_shared_shutdown_does_nothing(self):
   111      thread_pool_executor.shared_unbounded_instance().shutdown()
   112  
   113      futures = []
   114      with thread_pool_executor.shared_unbounded_instance() as executor:
   115        for _ in range(0, 5):
   116          futures.append(executor.submit(self.append_and_sleep, 0.01))
   117  
   118      for future in futures:
   119        future.result(timeout=10)
   120  
   121      with self._lock:
   122        self.assertEqual(5, len(self._worker_idents))
   123  
   124  
   125  if __name__ == '__main__':
   126    unittest.main()