tenable_io: fix flakey test and timestamp handling (#10940)

The comparison for determining whether the pagination had completed was based on the response's size and the batch size (via state.batch_size, but could also have been the response's params.size without difference in behaviour). This was not correct since the size value is the current response's result set and so may match the batch size even when the pagination is complete, in the case that total size mod batch size is zero. Instead keep a running tally of records returned throughout the pagination and compare this to the total count reported by the API. Also fix the test case so that the response matches the API docs.[1] The timestamps were being examined for their maximum by string comparison. This should approximately always work if the timestamp is formatted as RFC3339, but the timestamps are rendered as variable precision RFC3339, so make sure that the comparisons work by converting to timestamp values to obtain the maximum. [1]https://developer.tenable.com/reference/io-plugins-list
elastic · Sep 13, 2024 · 08e244c · 08e244c
1 parent 82311e2
commit 08e244c
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 32 deletions.
diff --git a/packages/tenable_io/_dev/deploy/docker/files/config.yml b/packages/tenable_io/_dev/deploy/docker/files/config.yml
@@ -27,7 +27,7 @@ rules:
                 {"id":13670,"name":"Fedora Core 1 : kernel-2.4.22-1.2140.nptl (2003-047)","attributes":{"vpr":{"score":5.5,"updated":"2018-07-19T00:00:00Z","drivers":{"age_of_vuln":{"lower_bound":366,"upper_bound":730},"threat_recency":{"lower_bound":366,"upper_bound":730},"exploit_code_maturity":"UNPROVEN","cvss_impact_score_predicted":false,"cvss3_impact_score":5.9,"threat_intensity_last28":"VERY_LOW","threat_sources_last28":["No recorded events"],"product_coverage":"LOW"}},"plugin_modification_date":"2018-07-19T00:00:00Z","plugin_version":"1.17","exploited_by_malware":false,"description":"Various RTC drivers had the potential to leak...","unsupported_by_vendor":false,"cvss_temporal_score":0,"patch_publication_date":"2004-01-07T00:00:00Z","see_also":["http://example.com/u?07bc9e7f"],"default_account":false,"exploit_available":false,"cve":["CVE-2003-0984"],"exploit_framework_canvas":false,"cvss_base_score":4.6,"solution":"Update the affected packages.","cvss_vector":{"raw":"AV:L/AC:L/Au:N/C:P/I:P/A:P","AccessVector":"Local-access","Availability-Impact":"Partial","Authentication":"None required","Integrity-Impact":"Partial","AccessComplexity":"Low","Confidentiality-Impact":"Partial"},"exploit_framework_exploithub":false,"cpe":["p-cpe:/a:fedoraproject:fedora:kernel-source","cpe:/o:fedoraproject:fedora_core:1","p-cpe:/a:fedoraproject:fedora:kernel-BOOT","p-cpe:/a:fedoraproject:fedora:kernel-debuginfo","p-cpe:/a:fedoraproject:fedora:kernel","p-cpe:/a:fedoraproject:fedora:kernel-doc","p-cpe:/a:fedoraproject:fedora:kernel-smp"],"plugin_publication_date":"2004-07-23T00:00:00Z","exploit_framework_core":false,"in_the_news":false,"has_patch":true,"xref":["FEDORA:2003-047"],"malware":false,"exploit_framework_d2_elliot":false,"xrefs":[{"type":"FEDORA","id":"2003-047"}],"risk_factor":"Medium","synopsis":"The remote Fedora Core host is missing a security update.","cvss3_temporal_score":0,"exploited_by_nessus":false,"cvss3_base_score":0,"exploit_framework_metasploit":false,"plugin_type":"local"}},
                 {"id":32,"name":"Siemens S7-300 Series PLC CPU Firmware <= 3.2.11 DoS","attributes":{"intel_type":"SENSOR","synopsis":"A Siemens S7-300 programmable logic controller (PLC) has been detected which is vulnerable to a Denial of Service (DoS) attack vector.","description":"Siemens S7-300 PLC central processing units (CPUs) contain an unspecified flaw that may allow a remote attacker to use a specially crafted packet to cause the device to enter defect mode until a cold restart is performed.","solution":"Upgrade the firmware to version 3.2.12 or later.","see_also":[],"plugin_publication_date":"2019-05-21T00:00:00Z","vuln_publication_date":"2016-06-08T00:00:00Z","patch_publication_date":"2016-06-08T00:00:00Z","has_patch":true,"exploit_available":false,"risk_factor":"HIGH","plugin_modification_date":"2019-09-30T00:00:00Z","always_run":false,"compliance":false,"cvss_vector":{"raw":"AV:N/AC:L/Au:N/C:N/I:N/A:C","AccessVector":"Network","AccessComplexity":"Low","Authentication":"None required","Confidentiality-Impact":"None","Integrity-Impact":"None","Availability-Impact":"Complete"},"cvss_temporal_vector":{"raw":"E:U/RL:OF/RC:C","Exploitability":"Unproven","RemediationLevel":"Official Fix","ReportConfidence":"Confirmed"},"cvss_temporal_score":5.8,"cvss3_vector":{"raw":"AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H","AttackVector":"Network","AttackComplexity":"Low","PrivilegesRequired":"None","UserInteraction":"None","Scope":"Unchanged","Confidentiality-Impact":"None","Integrity-Impact":"None","Availability-Impact":"High"},"cvss3_temporal_vector":{"raw":"E:U/RL:O/RC:C","ExploitCodeMaturity":"Unproven","RemediationLevel":"Official Fix","ReportConfidence":"Confirmed"},"cvss3_temporal_score":6.5,"cvss3_base_score":7.5,"cve":["CVE-2016-3949"],"bid":[91133,23432,234234,235],"xref":[],"xrefs":[],"vpr":{"score":3.6,"drivers":{"age_of_vuln":{"lower_bound":731},"exploit_code_maturity":"UNPROVEN","cvss_impact_score_predicted":false,"threat_intensity_last28":"VERY_LOW","threat_sources_last28":["No recorded events"],"product_coverage":"LOW"},"updated":"2021-02-07T05:31:54Z"}}}
               ]
-            },"size":1000,"params":{"page":1,"size":1000,"last_updated":""},"total_count":2}
+            },"size":2,"params":{"page":1,"size":1000,"last_updated":""},"total_count":2}
   - path: /vulns/export
     methods: ["POST"]
     responses:

diff --git a/packages/tenable_io/changelog.yml b/packages/tenable_io/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "3.2.1"
+  changes:
+    - description: Fix pagination progression logic.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10940
+    - description: Fix timestamp logic.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10940
 - version: "3.2.0"
   changes:
     - description: Improve error reporting for API request failures.

diff --git a/packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs b/packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs
@@ -38,43 +38,45 @@ program: |
     },
   }).do_request().as(resp,
     resp.StatusCode == 200 ?
-      bytes(resp.Body).decode_json().as(body, {
-        "events": has(body.?data.plugin_details) ? body.data.plugin_details.map(e, { "message": e.encode_json() }) : [{}],
-        "cursor": {
-          "last_update_time": (
-            has(body.?data.plugin_details) && body.data.plugin_details.size() > 0 ?
-              (
-                has(state.?cursor.last_update_time) && body.data.plugin_details.map(e, e.attributes.plugin_modification_date).max() < state.cursor.last_update_time ?
-                  state.cursor.last_update_time
-                :
-                  body.data.plugin_details.map(e, e.attributes.plugin_modification_date).max()
-              )
+      bytes(resp.Body).decode_json().as(body,
+        (int(state.?count.orValue(0))+int(body.size)).as(count,
+          (count < int(body.total_count)).as(want_more, {
+            "events": has(body.?data.plugin_details) ?
+              body.data.plugin_details.map(e, { "message": e.encode_json() })
             :
-              state.?cursor.last_update_time.orValue(string(timestamp(0)))
-          ),
-          "first_update_time": (
-            has(state.?cursor.first_update_time) && has(body.data.plugin_details) ?
-              (
-                body.data.plugin_details.size() > 0 ?
+              [],
+            "cursor": {
+              "last_update_time": (
+                has(body.?data.plugin_details) && body.data.plugin_details.size() > 0 ?
+                  ([?state.?cursor.last_update_time] + body.data.plugin_details.map(e,
+                    e.attributes.plugin_modification_date
+                  )).map(t,
+                    timestamp(t)
+                  ).max()
+                :
+                  state.?cursor.last_update_time.orValue(string(timestamp(0)))
+              ),
+              "first_update_time": (
+                has(state.?cursor.first_update_time) && has(body.data.plugin_details) ?
                   (
-                    body.size == int(state.batch_size) && state.want_more ?
+                    want_more ?
                       state.cursor.first_update_time
                     :
                       state.cursor.last_update_time
                   )
                 :
-                  state.cursor.last_update_time
-              )
-            :
-              string(timestamp(0))
-          ),
-        },
-        "next_page": body.size == int(state.batch_size) ? int(state.next_page)+1 : 1,
-        "want_more": body.size == int(state.batch_size),
-        "access_key": state.access_key,
-        "secret_key": state.secret_key,
-        "batch_size": state.batch_size,
-      })
+                  string(timestamp(0))
+              ),
+            },
+            "next_page": want_more ? int(state.next_page)+1 : 1,
+            "want_more": want_more,
+            "count": count,
+            "access_key": state.access_key,
+            "secret_key": state.secret_key,
+            "batch_size": state.batch_size,
+          })
+        )
+      )
     :
       {
         "events": {

diff --git a/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml
@@ -21,6 +21,10 @@ processors:
   - json:
       field: event.original
       target_field: json
+  - fail:
+      tag: cel_failure
+      if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null
+      message: error message set and no data to process
   - drop:
       if: ctx.json?.data?.plugin_details != null && ctx.json.data.plugin_details.isEmpty()
   - fingerprint:

diff --git a/packages/tenable_io/manifest.yml b/packages/tenable_io/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: tenable_io
 title: Tenable Vulnerability Management
-version: "3.2.0"
+version: "3.2.1"
 description: Collect logs from Tenable Vulnerability Management with Elastic Agent.
 type: integration
 categories: