github.com/jlmucb/cloudproxy@v0.0.0-20170830161738-b5aa0b619bc4/src/third_party/googlemock/gtest/test/gtest_shuffle_test.py (about)

     1  #!/usr/bin/env python
     2  #
     3  # Copyright 2009 Google Inc. All Rights Reserved.
     4  #
     5  # Redistribution and use in source and binary forms, with or without
     6  # modification, are permitted provided that the following conditions are
     7  # met:
     8  #
     9  #     * Redistributions of source code must retain the above copyright
    10  # notice, this list of conditions and the following disclaimer.
    11  #     * Redistributions in binary form must reproduce the above
    12  # copyright notice, this list of conditions and the following disclaimer
    13  # in the documentation and/or other materials provided with the
    14  # distribution.
    15  #     * Neither the name of Google Inc. nor the names of its
    16  # contributors may be used to endorse or promote products derived from
    17  # this software without specific prior written permission.
    18  #
    19  # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
    20  # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
    21  # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
    22  # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
    23  # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
    24  # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
    25  # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
    26  # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
    27  # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
    28  # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
    29  # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
    30  
    31  """Verifies that test shuffling works."""
    32  
    33  __author__ = 'wan@google.com (Zhanyong Wan)'
    34  
    35  import os
    36  import gtest_test_utils
    37  
    38  # Command to run the gtest_shuffle_test_ program.
    39  COMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_')
    40  
    41  # The environment variables for test sharding.
    42  TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
    43  SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
    44  
    45  TEST_FILTER = 'A*.A:A*.B:C*'
    46  
    47  ALL_TESTS = []
    48  ACTIVE_TESTS = []
    49  FILTERED_TESTS = []
    50  SHARDED_TESTS = []
    51  
    52  SHUFFLED_ALL_TESTS = []
    53  SHUFFLED_ACTIVE_TESTS = []
    54  SHUFFLED_FILTERED_TESTS = []
    55  SHUFFLED_SHARDED_TESTS = []
    56  
    57  
    58  def AlsoRunDisabledTestsFlag():
    59    return '--gtest_also_run_disabled_tests'
    60  
    61  
    62  def FilterFlag(test_filter):
    63    return '--gtest_filter=%s' % (test_filter,)
    64  
    65  
    66  def RepeatFlag(n):
    67    return '--gtest_repeat=%s' % (n,)
    68  
    69  
    70  def ShuffleFlag():
    71    return '--gtest_shuffle'
    72  
    73  
    74  def RandomSeedFlag(n):
    75    return '--gtest_random_seed=%s' % (n,)
    76  
    77  
    78  def RunAndReturnOutput(extra_env, args):
    79    """Runs the test program and returns its output."""
    80  
    81    environ_copy = os.environ.copy()
    82    environ_copy.update(extra_env)
    83  
    84    return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output
    85  
    86  
    87  def GetTestsForAllIterations(extra_env, args):
    88    """Runs the test program and returns a list of test lists.
    89  
    90    Args:
    91      extra_env: a map from environment variables to their values
    92      args: command line flags to pass to gtest_shuffle_test_
    93  
    94    Returns:
    95      A list where the i-th element is the list of tests run in the i-th
    96      test iteration.
    97    """
    98  
    99    test_iterations = []
   100    for line in RunAndReturnOutput(extra_env, args).split('\n'):
   101      if line.startswith('----'):
   102        tests = []
   103        test_iterations.append(tests)
   104      elif line.strip():
   105        tests.append(line.strip())  # 'TestCaseName.TestName'
   106  
   107    return test_iterations
   108  
   109  
   110  def GetTestCases(tests):
   111    """Returns a list of test cases in the given full test names.
   112  
   113    Args:
   114      tests: a list of full test names
   115  
   116    Returns:
   117      A list of test cases from 'tests', in their original order.
   118      Consecutive duplicates are removed.
   119    """
   120  
   121    test_cases = []
   122    for test in tests:
   123      test_case = test.split('.')[0]
   124      if not test_case in test_cases:
   125        test_cases.append(test_case)
   126  
   127    return test_cases
   128  
   129  
   130  def CalculateTestLists():
   131    """Calculates the list of tests run under different flags."""
   132  
   133    if not ALL_TESTS:
   134      ALL_TESTS.extend(
   135          GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0])
   136  
   137    if not ACTIVE_TESTS:
   138      ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0])
   139  
   140    if not FILTERED_TESTS:
   141      FILTERED_TESTS.extend(
   142          GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0])
   143  
   144    if not SHARDED_TESTS:
   145      SHARDED_TESTS.extend(
   146          GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
   147                                    SHARD_INDEX_ENV_VAR: '1'},
   148                                   [])[0])
   149  
   150    if not SHUFFLED_ALL_TESTS:
   151      SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations(
   152          {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0])
   153  
   154    if not SHUFFLED_ACTIVE_TESTS:
   155      SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations(
   156          {}, [ShuffleFlag(), RandomSeedFlag(1)])[0])
   157  
   158    if not SHUFFLED_FILTERED_TESTS:
   159      SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations(
   160          {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0])
   161  
   162    if not SHUFFLED_SHARDED_TESTS:
   163      SHUFFLED_SHARDED_TESTS.extend(
   164          GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
   165                                    SHARD_INDEX_ENV_VAR: '1'},
   166                                   [ShuffleFlag(), RandomSeedFlag(1)])[0])
   167  
   168  
   169  class GTestShuffleUnitTest(gtest_test_utils.TestCase):
   170    """Tests test shuffling."""
   171  
   172    def setUp(self):
   173      CalculateTestLists()
   174  
   175    def testShufflePreservesNumberOfTests(self):
   176      self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
   177      self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
   178      self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
   179      self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
   180  
   181    def testShuffleChangesTestOrder(self):
   182      self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
   183      self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
   184      self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
   185                   SHUFFLED_FILTERED_TESTS)
   186      self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
   187                   SHUFFLED_SHARDED_TESTS)
   188  
   189    def testShuffleChangesTestCaseOrder(self):
   190      self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS),
   191                   GetTestCases(SHUFFLED_ALL_TESTS))
   192      self.assert_(
   193          GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS),
   194          GetTestCases(SHUFFLED_ACTIVE_TESTS))
   195      self.assert_(
   196          GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS),
   197          GetTestCases(SHUFFLED_FILTERED_TESTS))
   198      self.assert_(
   199          GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS),
   200          GetTestCases(SHUFFLED_SHARDED_TESTS))
   201  
   202    def testShuffleDoesNotRepeatTest(self):
   203      for test in SHUFFLED_ALL_TESTS:
   204        self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
   205                         '%s appears more than once' % (test,))
   206      for test in SHUFFLED_ACTIVE_TESTS:
   207        self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
   208                         '%s appears more than once' % (test,))
   209      for test in SHUFFLED_FILTERED_TESTS:
   210        self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
   211                         '%s appears more than once' % (test,))
   212      for test in SHUFFLED_SHARDED_TESTS:
   213        self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
   214                         '%s appears more than once' % (test,))
   215  
   216    def testShuffleDoesNotCreateNewTest(self):
   217      for test in SHUFFLED_ALL_TESTS:
   218        self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,))
   219      for test in SHUFFLED_ACTIVE_TESTS:
   220        self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,))
   221      for test in SHUFFLED_FILTERED_TESTS:
   222        self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,))
   223      for test in SHUFFLED_SHARDED_TESTS:
   224        self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,))
   225  
   226    def testShuffleIncludesAllTests(self):
   227      for test in ALL_TESTS:
   228        self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,))
   229      for test in ACTIVE_TESTS:
   230        self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,))
   231      for test in FILTERED_TESTS:
   232        self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,))
   233      for test in SHARDED_TESTS:
   234        self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,))
   235  
   236    def testShuffleLeavesDeathTestsAtFront(self):
   237      non_death_test_found = False
   238      for test in SHUFFLED_ACTIVE_TESTS:
   239        if 'DeathTest.' in test:
   240          self.assert_(not non_death_test_found,
   241                       '%s appears after a non-death test' % (test,))
   242        else:
   243          non_death_test_found = True
   244  
   245    def _VerifyTestCasesDoNotInterleave(self, tests):
   246      test_cases = []
   247      for test in tests:
   248        [test_case, _] = test.split('.')
   249        if test_cases and test_cases[-1] != test_case:
   250          test_cases.append(test_case)
   251          self.assertEqual(1, test_cases.count(test_case),
   252                           'Test case %s is not grouped together in %s' %
   253                           (test_case, tests))
   254  
   255    def testShuffleDoesNotInterleaveTestCases(self):
   256      self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS)
   257      self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS)
   258      self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS)
   259      self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS)
   260  
   261    def testShuffleRestoresOrderAfterEachIteration(self):
   262      # Get the test lists in all 3 iterations, using random seed 1, 2,
   263      # and 3 respectively.  Google Test picks a different seed in each
   264      # iteration, and this test depends on the current implementation
   265      # picking successive numbers.  This dependency is not ideal, but
   266      # makes the test much easier to write.
   267      [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
   268          GetTestsForAllIterations(
   269              {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
   270  
   271      # Make sure running the tests with random seed 1 gets the same
   272      # order as in iteration 1 above.
   273      [tests_with_seed1] = GetTestsForAllIterations(
   274          {}, [ShuffleFlag(), RandomSeedFlag(1)])
   275      self.assertEqual(tests_in_iteration1, tests_with_seed1)
   276  
   277      # Make sure running the tests with random seed 2 gets the same
   278      # order as in iteration 2 above.  Success means that Google Test
   279      # correctly restores the test order before re-shuffling at the
   280      # beginning of iteration 2.
   281      [tests_with_seed2] = GetTestsForAllIterations(
   282          {}, [ShuffleFlag(), RandomSeedFlag(2)])
   283      self.assertEqual(tests_in_iteration2, tests_with_seed2)
   284  
   285      # Make sure running the tests with random seed 3 gets the same
   286      # order as in iteration 3 above.  Success means that Google Test
   287      # correctly restores the test order before re-shuffling at the
   288      # beginning of iteration 3.
   289      [tests_with_seed3] = GetTestsForAllIterations(
   290          {}, [ShuffleFlag(), RandomSeedFlag(3)])
   291      self.assertEqual(tests_in_iteration3, tests_with_seed3)
   292  
   293    def testShuffleGeneratesNewOrderInEachIteration(self):
   294      [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
   295          GetTestsForAllIterations(
   296              {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)]))
   297  
   298      self.assert_(tests_in_iteration1 != tests_in_iteration2,
   299                   tests_in_iteration1)
   300      self.assert_(tests_in_iteration1 != tests_in_iteration3,
   301                   tests_in_iteration1)
   302      self.assert_(tests_in_iteration2 != tests_in_iteration3,
   303                   tests_in_iteration2)
   304  
   305    def testShuffleShardedTestsPreservesPartition(self):
   306      # If we run M tests on N shards, the same M tests should be run in
   307      # total, regardless of the random seeds used by the shards.
   308      [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
   309                                           SHARD_INDEX_ENV_VAR: '0'},
   310                                          [ShuffleFlag(), RandomSeedFlag(1)])
   311      [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
   312                                           SHARD_INDEX_ENV_VAR: '1'},
   313                                          [ShuffleFlag(), RandomSeedFlag(20)])
   314      [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3',
   315                                           SHARD_INDEX_ENV_VAR: '2'},
   316                                          [ShuffleFlag(), RandomSeedFlag(25)])
   317      sorted_sharded_tests = tests1 + tests2 + tests3
   318      sorted_sharded_tests.sort()
   319      sorted_active_tests = []
   320      sorted_active_tests.extend(ACTIVE_TESTS)
   321      sorted_active_tests.sort()
   322      self.assertEqual(sorted_active_tests, sorted_sharded_tests)
   323  
   324  if __name__ == '__main__':
   325    gtest_test_utils.Main()