From 2a45db54ebdc8649fd882d0aabc7a866b3987be7 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Tue, 22 Jan 2019 16:26:28 -0500 Subject: [PATCH 1/6] test: enable marking of failing coverage tests Enable marking of coverage tests so that we can allow some tests to fail without blocking the generation of coverage data. This will later allow us to fail the coverage job if other kinds of errors occur and to capture which tests we believe are not running properly with coverage enabled. --- Makefile | 5 +++-- test/js-native-api/js-native-api.status | 25 +++++++++++++++++++++++++ tools/test.py | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 test/js-native-api/js-native-api.status diff --git a/Makefile b/Makefile index 36b61af399da58..c20849aeccee79 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,8 @@ coverage-test: coverage-build $(RM) out/$(BUILDTYPE)/obj.target/node_lib/gen/*.gcda $(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/*.gcda $(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/tracing/*.gcda - -NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage $(MAKE) $(COVTESTS) + NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage FLAKY_TESTS="dontcare" \ + TEST_CI_ARGS="$(TEST_CI_ARGS) --type=coverage" $(MAKE) $(COVTESTS) $(MAKE) coverage-report-js -(cd out && "../gcovr/scripts/gcovr" --gcov-exclude='.*deps' \ --gcov-exclude='.*usr' -v -r Release/obj.target \ @@ -277,7 +278,7 @@ coverage-run-js: $(RM) -r out/$(BUILDTYPE)/.coverage $(MAKE) coverage-build-js -NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage CI_SKIP_TESTS=$(COV_SKIP_TESTS) \ - $(MAKE) jstest + TEST_CI_ARGS="$(TEST_CI_ARGS) --type=coverage" $(MAKE) jstest $(MAKE) coverage-report-js .PHONY: test diff --git a/test/js-native-api/js-native-api.status b/test/js-native-api/js-native-api.status new file mode 100644 index 00000000000000..72f3f833e14f89 --- /dev/null +++ b/test/js-native-api/js-native-api.status @@ -0,0 +1,25 @@ +prefix js-native-api + +# To mark a test as flaky, list the test name in the appropriate section +# below, without ".js", followed by ": PASS,FLAKY". Example: +# sample-test : PASS,FLAKY + +[true] # This section applies to all platforms + +[$system==win32] + +[$system==linux] + +[$system==macos] + +[$arch==arm || $arch==arm64] + +[$system==solaris] # Also applies to SmartOS + +[$system==freebsd] + +[$system==aix] + +[$type==coverage] +test_function/test: PASS,FLAKY +test_general/testFinalizer: PASS,FLAKY diff --git a/tools/test.py b/tools/test.py index 633c347e74f0d7..66d4a8fa7e64e3 100755 --- a/tools/test.py +++ b/tools/test.py @@ -1388,7 +1388,7 @@ def BuildOptions(): help='Send SIGABRT instead of SIGTERM to kill processes that time out', default=False, action="store_true", dest="abort_on_timeout") result.add_option("--type", - help="Type of build (simple, fips)", + help="Type of build (simple, fips, coverage)", default=None) return result From 58013ef5b628315e0a1a79cbd23b10403fa6157c Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Wed, 23 Jan 2019 16:50:23 -0500 Subject: [PATCH 2/6] squash: allow FAIL and CRASH to avoid yellow Allow FAIL and CRASH to avoid yellow, otherwise if we added to regular CI run we'd have perma-yellow --- test/js-native-api/js-native-api.status | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/js-native-api/js-native-api.status b/test/js-native-api/js-native-api.status index 72f3f833e14f89..c509d2f54caf84 100644 --- a/test/js-native-api/js-native-api.status +++ b/test/js-native-api/js-native-api.status @@ -21,5 +21,5 @@ prefix js-native-api [$system==aix] [$type==coverage] -test_function/test: PASS,FLAKY -test_general/testFinalizer: PASS,FLAKY +test_function/test: PASS,FAIL,CRASH +test_general/testFinalizer: PASS,FAIL,CRASH From 9d0c8988a74d9de336ca31e695e69bf86421b53e Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Wed, 23 Jan 2019 17:02:24 -0500 Subject: [PATCH 3/6] squash: allow new tests to pass until ci job added Allow new tests to fail until we get a ci job added to the main regression job so that comits don't break coverage that have to be fixed later on. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c20849aeccee79..7320e90fa1a0ff 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,7 @@ coverage-test: coverage-build $(RM) out/$(BUILDTYPE)/obj.target/node_lib/gen/*.gcda $(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/*.gcda $(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/tracing/*.gcda - NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage FLAKY_TESTS="dontcare" \ + -NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage FLAKY_TESTS="dontcare" \ TEST_CI_ARGS="$(TEST_CI_ARGS) --type=coverage" $(MAKE) $(COVTESTS) $(MAKE) coverage-report-js -(cd out && "../gcovr/scripts/gcovr" --gcov-exclude='.*deps' \ From c27dd17e09d41361c0a3472092d05eff90dac403 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Thu, 24 Jan 2019 16:04:25 -0500 Subject: [PATCH 4/6] squash: address comments --- test/js-native-api/js-native-api.status | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 test/js-native-api/js-native-api.status diff --git a/test/js-native-api/js-native-api.status b/test/js-native-api/js-native-api.status deleted file mode 100644 index c509d2f54caf84..00000000000000 --- a/test/js-native-api/js-native-api.status +++ /dev/null @@ -1,25 +0,0 @@ -prefix js-native-api - -# To mark a test as flaky, list the test name in the appropriate section -# below, without ".js", followed by ": PASS,FLAKY". Example: -# sample-test : PASS,FLAKY - -[true] # This section applies to all platforms - -[$system==win32] - -[$system==linux] - -[$system==macos] - -[$arch==arm || $arch==arm64] - -[$system==solaris] # Also applies to SmartOS - -[$system==freebsd] - -[$system==aix] - -[$type==coverage] -test_function/test: PASS,FAIL,CRASH -test_general/testFinalizer: PASS,FAIL,CRASH From a48bc9fde3231827c43cd5da438fe79ab26edd45 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Thu, 24 Jan 2019 16:16:57 -0500 Subject: [PATCH 5/6] squash: address comments --- Makefile | 2 +- test/root.status | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7320e90fa1a0ff..51db775bdc3eba 100644 --- a/Makefile +++ b/Makefile @@ -226,7 +226,7 @@ coverage-test: coverage-build $(RM) out/$(BUILDTYPE)/obj.target/node_lib/gen/*.gcda $(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/*.gcda $(RM) out/$(BUILDTYPE)/obj.target/node_lib/src/tracing/*.gcda - -NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage FLAKY_TESTS="dontcare" \ + -NODE_V8_COVERAGE=out/$(BUILDTYPE)/.coverage \ TEST_CI_ARGS="$(TEST_CI_ARGS) --type=coverage" $(MAKE) $(COVTESTS) $(MAKE) coverage-report-js -(cd out && "../gcovr/scripts/gcovr" --gcov-exclude='.*deps' \ diff --git a/test/root.status b/test/root.status index e5dd389cae0424..b2018d9b9438c2 100644 --- a/test/root.status +++ b/test/root.status @@ -159,3 +159,7 @@ sequential/test-inspector-port-cluster: SLOW sequential/test-net-bytes-per-incoming-chunk-overhead: SLOW sequential/test-pipe: SLOW sequential/test-util-debug: SLOW + +[$type==coverage] +test_function/test: PASS,FAIL,CRASH +test_general/testFinalizer: PASS,FAIL,CRASH From f7b5b67a2627bd2be676d17223869ac6eb3f64a1 Mon Sep 17 00:00:00 2001 From: Michael Dawson Date: Thu, 24 Jan 2019 16:20:13 -0500 Subject: [PATCH 6/6] squash: address comments --- test/root.status | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/root.status b/test/root.status index b2018d9b9438c2..6d5bc44491c310 100644 --- a/test/root.status +++ b/test/root.status @@ -161,5 +161,5 @@ sequential/test-pipe: SLOW sequential/test-util-debug: SLOW [$type==coverage] -test_function/test: PASS,FAIL,CRASH -test_general/testFinalizer: PASS,FAIL,CRASH +js-native-api/test_function/test: PASS,FAIL,CRASH +js-native-api/test_general/testFinalizer: PASS,FAIL,CRASH