Merge pull request #1068 from oren0e/oren0e/allow-resuming-without-sk…

…ipped

Add flag for resuming session without skipped tests

doc/slash_run.rst

@@ -109,7 +109,7 @@ When you run a session that fails, Slash automatically saves the tests intended
 
   $ slash resume -vv <session id>
 
-This command receives all flags which can be passed to ``slash run``, but receives an id of a previously run session for resuming. All unsuccessful tests are then rerun in a new session. You can control whether to attempt failed tests first or planned (not started) tests through the ``--failed-first`` and ``--unstarted-first`` command-line flags respectively.
+This command receives all flags which can be passed to ``slash run``, but receives an id of a previously run session for resuming. All unsuccessful tests are then rerun in a new session. You can control whether to attempt failed tests first or planned (not started) tests through the ``--failed-first`` and ``--unstarted-first`` command-line flags respectively. You can also control which tests to resume through the ``--failed-only``, ``--unstarted-only``, and ``--no-skipped`` command-line flags. You can use combinations of the latter two but not with the first (failed-ony).
 
 
 Rerunning Previous Sessions

slash/conf.py

@@ -123,6 +123,7 @@
         "unstarted_first": False // Doc("Run unstarted tests of previous session before all others") // Cmdline(on='--unstarted-first', metavar="UNSTARTED_FIRST"),
         "failed_only": False // Doc("Run only failed tests of previous session") // Cmdline(on='--failed-only', metavar="FAILED_ONLY"),
         "unstarted_only": False // Doc("Run only unstarted tests of previous session") // Cmdline(on='--unstarted-only', metavar="UNSTARTED_ONLY"),
+        "no_skipped": False // Doc("Run tests of previous session without skipped tests") // Cmdline(on='--no-skipped', metavar="NO_SKIPPED"),
         "state_retention_days": 10 // Doc("Number of days to keep session entries for resuming session")
     },
     "tmux": {

slash/resuming.py

@@ -44,6 +44,7 @@ class ResumeTestStatus(object):
     PLANNED = 'planned'
     SUCCESS = 'success'
     FAILED = 'failed'
+    SKIPPED = 'skipped'
 
 
 class ResumedTestData(object):
@@ -110,10 +111,13 @@ def save_resume_state(session_result, collected_tests):
         test_to_resume.variation = str(metadata.variation.id) if metadata.variation else None
         if result.is_success_finished():
             test_to_resume.status = ResumeTestStatus.SUCCESS
-        elif not result.is_started() or result.is_skip():
+        elif not result.is_started():
             test_to_resume.status = ResumeTestStatus.PLANNED
         else:
             test_to_resume.status = ResumeTestStatus.FAILED
+
+        if result.is_skip():
+            test_to_resume.status = ResumeTestStatus.SKIPPED
         tests_to_resume.append(test_to_resume)
     tests_with_no_results = {test.__slash__ for test in collected_tests if session_result.safe_get_result(test) is None}
 
@@ -144,6 +148,8 @@ def _get_resume_status_from_backslash_status(backslash_status):
         return ResumeTestStatus.FAILED
     elif backslash_status in ['SUCCESS']:
         return ResumeTestStatus.SUCCESS
+    elif backslash_status in ['SKIPPED']:
+        return ResumeTestStatus.SKIPPED
     else:
         return ResumeTestStatus.PLANNED
 
@@ -179,15 +185,24 @@ def get_tests_locally(session_id):
 
 def _filter_tests(tests, get_successful_tests):
     unstarted_only, failed_only = config.root.resume.unstarted_only, config.root.resume.failed_only
+    no_skipped = config.root.resume.no_skipped
     if unstarted_only and failed_only:
         raise CannotResume("Got both unstarted_only and failed_only, cannot resume")
+    if no_skipped and failed_only:
+        raise CannotResume("Got both no_skipped and failed_only, cannot resume")
     filtered_status = [ResumeTestStatus.SUCCESS] if get_successful_tests else []
     if unstarted_only:
         filtered_status.append(ResumeTestStatus.PLANNED)
+        filtered_status.append(ResumeTestStatus.SKIPPED)
     elif failed_only:
         filtered_status.append(ResumeTestStatus.FAILED)
     else:
-        filtered_status.extend([ResumeTestStatus.FAILED, ResumeTestStatus.PLANNED])
+        filtered_status.extend([ResumeTestStatus.FAILED, ResumeTestStatus.PLANNED, ResumeTestStatus.SKIPPED])
+    if no_skipped:
+        try:
+            filtered_status.remove(ResumeTestStatus.SKIPPED)
+        except ValueError:
+            _logger.warning("Didn't find any skipped tests to exclude")
     return list(filter(lambda test: test.status in filtered_status, tests))
 
 def _sort_tests(tests):

tests/test_slash_resume.py

@@ -166,3 +166,64 @@ def test_resuming_interrupted_session(suite):
     resumed = get_tests_from_previous_session(results.session.id)
     assert len(resumed) == len(suite)
     assert set(test.address_in_file for test in resumed) == set(address_in_file(test) for test in suite)
+
+@pytest.mark.parametrize('no_skipped,unstarted_only', [
+    (True, True),
+    (False, True),
+    (True, False),
+    (False, False),
+    ])
+def test_no_skipped(suite, no_skipped, unstarted_only, config_override):
+    fail_index = len(suite) // 2
+    skip_index = fail_index - 1
+    suite[skip_index].when_run.skip(decorator=True)
+    suite[fail_index].when_run.fail()
+    for index, test in enumerate(suite):
+        if index > fail_index:
+            test.expect_not_run()
+    result = suite.run(additional_args=['-x'])
+
+    if no_skipped and unstarted_only:
+        config_override('resume.unstarted_only', True)
+        config_override('resume.no_skipped', True)
+        expected_resume_tests_num = 4
+        expected_status = 'planned'
+        resumed = get_tests_from_previous_session(result.session.id)
+        assert len(resumed) == expected_resume_tests_num
+        for test in resumed:
+            assert test.status == expected_status
+    elif (not no_skipped) and unstarted_only:
+        config_override('resume.unstarted_only', True)
+        expected_resume_tests_num = len(suite) - fail_index
+        expected_planned = 4
+        expected_skipped = 1
+        resumed = get_tests_from_previous_session(result.session.id)
+        assert len(resumed) == expected_resume_tests_num
+        num_planned = sum(1 for test in resumed if test.status == "planned")
+        num_skipped = sum(1 for test in resumed if test.status == "skipped")
+        assert num_planned == expected_planned
+        assert num_skipped == expected_skipped
+    elif no_skipped and (not unstarted_only):
+        config_override('resume.no_skipped', True)
+        expected_resume_tests_num = len(suite) - fail_index
+        expected_planned = 4
+        expected_failed = 1
+        resumed = get_tests_from_previous_session(result.session.id)
+        assert len(resumed) == expected_resume_tests_num
+        num_planned = sum(1 for test in resumed if test.status == "planned")
+        num_failed = sum(1 for test in resumed if test.status == "failed")
+        assert num_planned == expected_planned
+        assert num_failed == expected_failed
+    else:
+        expected_resume_tests_num = len(suite) - skip_index
+        expected_planned = 4
+        expected_failed = 1
+        expected_skipped = 1
+        resumed = get_tests_from_previous_session(result.session.id)
+        assert len(resumed) == expected_resume_tests_num
+        num_planned = sum(1 for test in resumed if test.status == "planned")
+        num_failed = sum(1 for test in resumed if test.status == "failed")
+        num_skipped = sum(1 for test in resumed if test.status == "skipped")
+        assert num_planned == expected_planned
+        assert num_failed == expected_failed
+        assert num_skipped == expected_skipped