31"""Verifies that test shuffling works."""
34import gtest_test_utils
40TOTAL_SHARDS_ENV_VAR =
'GTEST_TOTAL_SHARDS'
41SHARD_INDEX_ENV_VAR =
'GTEST_SHARD_INDEX'
43TEST_FILTER =
'A*.A:A*.B:C*'
50SHUFFLED_ALL_TESTS = []
51SHUFFLED_ACTIVE_TESTS = []
52SHUFFLED_FILTERED_TESTS = []
53SHUFFLED_SHARDED_TESTS = []
57 return '--gtest_also_run_disabled_tests'
61 return '--gtest_filter=%s' % (test_filter,)
65 return '--gtest_repeat=%s' % (n,)
69 return '--gtest_shuffle'
73 return '--gtest_random_seed=%s' % (n,)
77 """Runs the test program and returns its output."""
79 environ_copy = os.environ.copy()
80 environ_copy.update(extra_env)
86 """Runs the test program and returns a list of test lists.
89 extra_env: a map from environment variables to their values
90 args: command line flags to
pass to googletest-shuffle-test_
93 A list where the i-th element
is the list of tests run
in the i-th
99 if line.startswith(
'----'):
101 test_iterations.append(tests)
103 tests.append(line.strip())
105 return test_iterations
109 """Returns a list of test cases in the given full test names.
112 tests: a list of full test names
115 A list of test cases from 'tests',
in their original order.
116 Consecutive duplicates are removed.
121 test_case = test.split(
'.')[0]
122 if not test_case
in test_cases:
123 test_cases.append(test_case)
129 """Calculates the list of tests run under different flags."""
138 if not FILTERED_TESTS:
139 FILTERED_TESTS.extend(
142 if not SHARDED_TESTS:
143 SHARDED_TESTS.extend(
145 SHARD_INDEX_ENV_VAR:
'1'},
148 if not SHUFFLED_ALL_TESTS:
152 if not SHUFFLED_ACTIVE_TESTS:
156 if not SHUFFLED_FILTERED_TESTS:
160 if not SHUFFLED_SHARDED_TESTS:
161 SHUFFLED_SHARDED_TESTS.extend(
163 SHARD_INDEX_ENV_VAR:
'1'},
168 """Tests test shuffling."""
174 self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS))
175 self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS))
176 self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS))
177 self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS))
180 self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS)
181 self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS)
182 self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS,
183 SHUFFLED_FILTERED_TESTS)
184 self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS,
185 SHUFFLED_SHARDED_TESTS)
201 for test
in SHUFFLED_ALL_TESTS:
202 self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test),
203 '%s appears more than once' % (test,))
204 for test
in SHUFFLED_ACTIVE_TESTS:
205 self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test),
206 '%s appears more than once' % (test,))
207 for test
in SHUFFLED_FILTERED_TESTS:
208 self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test),
209 '%s appears more than once' % (test,))
210 for test
in SHUFFLED_SHARDED_TESTS:
211 self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test),
212 '%s appears more than once' % (test,))
215 for test
in SHUFFLED_ALL_TESTS:
216 self.assert_(test
in ALL_TESTS,
'%s is an invalid test' % (test,))
217 for test
in SHUFFLED_ACTIVE_TESTS:
218 self.assert_(test
in ACTIVE_TESTS,
'%s is an invalid test' % (test,))
219 for test
in SHUFFLED_FILTERED_TESTS:
220 self.assert_(test
in FILTERED_TESTS,
'%s is an invalid test' % (test,))
221 for test
in SHUFFLED_SHARDED_TESTS:
222 self.assert_(test
in SHARDED_TESTS,
'%s is an invalid test' % (test,))
225 for test
in ALL_TESTS:
226 self.assert_(test
in SHUFFLED_ALL_TESTS,
'%s is missing' % (test,))
227 for test
in ACTIVE_TESTS:
228 self.assert_(test
in SHUFFLED_ACTIVE_TESTS,
'%s is missing' % (test,))
229 for test
in FILTERED_TESTS:
230 self.assert_(test
in SHUFFLED_FILTERED_TESTS,
'%s is missing' % (test,))
231 for test
in SHARDED_TESTS:
232 self.assert_(test
in SHUFFLED_SHARDED_TESTS,
'%s is missing' % (test,))
235 non_death_test_found =
False
236 for test
in SHUFFLED_ACTIVE_TESTS:
237 if 'DeathTest.' in test:
238 self.assert_(
not non_death_test_found,
239 '%s appears after a non-death test' % (test,))
241 non_death_test_found =
True
243 def _VerifyTestCasesDoNotInterleave(self, tests):
246 [test_case, _] = test.split(
'.')
247 if test_cases
and test_cases[-1] != test_case:
248 test_cases.append(test_case)
249 self.assertEqual(1, test_cases.count(test_case),
250 'Test case %s is not grouped together in %s' %
265 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
273 self.assertEqual(tests_in_iteration1, tests_with_seed1)
281 self.assertEqual(tests_in_iteration2, tests_with_seed2)
289 self.assertEqual(tests_in_iteration3, tests_with_seed3)
292 [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = (
296 self.assert_(tests_in_iteration1 != tests_in_iteration2,
298 self.assert_(tests_in_iteration1 != tests_in_iteration3,
300 self.assert_(tests_in_iteration2 != tests_in_iteration3,
307 SHARD_INDEX_ENV_VAR:
'0'},
310 SHARD_INDEX_ENV_VAR:
'1'},
313 SHARD_INDEX_ENV_VAR:
'2'},
315 sorted_sharded_tests = tests1 + tests2 + tests3
316 sorted_sharded_tests.sort()
317 sorted_active_tests = []
318 sorted_active_tests.extend(ACTIVE_TESTS)
319 sorted_active_tests.sort()
320 self.assertEqual(sorted_active_tests, sorted_sharded_tests)
322if __name__ ==
'__main__':
const std::vector< std::string > split(const std::string &s, char c)
def FilterFlag(test_filter)
def GetTestsForAllIterations(extra_env, args)
def AlsoRunDisabledTestsFlag()
def RunAndReturnOutput(extra_env, args)
def GetTestExecutablePath(executable_name, build_dir=None)
def testShuffleRestoresOrderAfterEachIteration(self)
def testShuffleLeavesDeathTestsAtFront(self)
def testShufflePreservesNumberOfTests(self)
def testShuffleChangesTestOrder(self)
def testShuffleDoesNotCreateNewTest(self)
def _VerifyTestCasesDoNotInterleave(self, tests)
def testShuffleDoesNotInterleaveTestCases(self)
def testShuffleDoesNotRepeatTest(self)
def testShuffleShardedTestsPreservesPartition(self)
def testShuffleIncludesAllTests(self)
def testShuffleChangesTestCaseOrder(self)
def testShuffleGeneratesNewOrderInEachIteration(self)