adding simple checkpoint to openhtf.util (#816)

* adding simple checkpoint to openhtf.util

* adding full output to json in example

* adding better docstring for checkpoints

* removing unused phase in examples/checkpoints.py

examples/checkpoints.py

@@ -0,0 +1,68 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Example OpenHTF test demonstrating use of checkpoints and measurements."""
+
+import time
+
+import openhtf as htf
+from openhtf.output.callbacks import console_summary
+from openhtf.output.callbacks import json_factory
+from openhtf.util import checkpoints
+
+from examples import measurements as measurements_example
+
+
+@htf.measures(htf.Measurement('fixed_time').in_range(0, 10).doc(
+    'This is going to fail validation.').with_units(htf.units.SECOND))
+def failing_phase(test):
+  # The 'outcome' of this measurement in the test_record result will be a FAIL
+  # because its value fails the validator specified (0 <= 5 <= 10).
+  test.measurements.fixed_time = 20
+
+
+def long_running_phase(test):
+  # A long running phase could be something like a hardware burn-in.  This
+  # phase should not run if previous phases have failed, so we make sure
+  # checkpoint phase is run right before this phase.
+  for i in xrange(10):
+    test.logger.info('Still running....')
+    time.sleep(10)
+  test.logger.info('Done with long_running_phase')
+
+if __name__ == '__main__':
+  # We instantiate our OpenHTF test with the phases we want to run as args.
+  test = htf.Test(
+      measurements_example.hello_phase,
+      measurements_example.again_phase,
+      failing_phase,
+      measurements_example.lots_of_measurements,
+      checkpoints.Checkpoint(),
+      long_running_phase,
+  )
+
+  # In order to view the result of the test, we have to output it somewhere,
+  # outputting to console is an easy way to do this.
+  test.add_output_callbacks(
+      console_summary.ConsoleSummary()
+  )
+
+  # The complete summary is viable in json, including the measurements
+  # included in measurements_example.lots_of_measurements.
+  test.add_output_callbacks(
+      json_factory.OutputToJSON('./checkpoints.json', indent=2))
+
+  # Unlike hello_world.py, where we prompt for a DUT ID, here we'll just
+  # use an arbitrary one.
+  test.execute(test_start=lambda: 'MyDutId')

openhtf/util/checkpoints.py

@@ -0,0 +1,50 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""A simple utility to check whether all previous phases have passed.
+
+In general test execution stops on a raised Exception but will continue if a
+phase has a failed measurement.  This is desirable behavior in many cases
+because you want to acquire additional information before the test stops.
+
+However, in some cases it's less desirable because downstream long-running
+phases will be executed only to FAIL at test completion due to the failing
+measurements.  This can have negative implications for tact time.
+
+Checkpoints allow test authors to have their cake and eat it too.  A checkpoint
+checks the result of all previous phases and will stop test execution if a
+previous phase has failed.
+"""
+
+from openhtf.core import phase_descriptor
+from openhtf.core import test_record
+
+def Checkpoint(checkpoint_name=None):
+  name = checkpoint_name if checkpoint_name else 'Checkpoint'
+
+  @phase_descriptor.PhaseOptions(name=name)
+  def _Checkpoint(test_run):
+    failed_phases = []
+    for phase_record in test_run.test_record.phases:
+      if (phase_record.result.is_fail_and_continue or
+          phase_record.outcome == test_record.PhaseOutcome.FAIL):
+        failed_phases.append(phase_record.name)
+
+    if failed_phases:
+      test_run.logger.error('Stopping execution because phases failed: %s',
+                            failed_phases)
+      return phase_descriptor.PhaseResult.STOP
+
+  return _Checkpoint