From 08e244c8870cb6848ab9db98ca2337f3415655f9 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Fri, 13 Sep 2024 17:39:14 +0930 Subject: [PATCH] tenable_io: fix flakey test and timestamp handling (#10940) The comparison for determining whether the pagination had completed was based on the response's size and the batch size (via state.batch_size, but could also have been the response's params.size without difference in behaviour). This was not correct since the size value is the current response's result set and so may match the batch size even when the pagination is complete, in the case that total size mod batch size is zero. Instead keep a running tally of records returned throughout the pagination and compare this to the total count reported by the API. Also fix the test case so that the response matches the API docs.[1] The timestamps were being examined for their maximum by string comparison. This should approximately always work if the timestamp is formatted as RFC3339, but the timestamps are rendered as variable precision RFC3339, so make sure that the comparisons work by converting to timestamp values to obtain the maximum. [1]https://developer.tenable.com/reference/io-plugins-list --- .../_dev/deploy/docker/files/config.yml | 2 +- packages/tenable_io/changelog.yml | 8 +++ .../plugin/agent/stream/cel.yml.hbs | 62 ++++++++++--------- .../elasticsearch/ingest_pipeline/default.yml | 4 ++ packages/tenable_io/manifest.yml | 2 +- 5 files changed, 46 insertions(+), 32 deletions(-) diff --git a/packages/tenable_io/_dev/deploy/docker/files/config.yml b/packages/tenable_io/_dev/deploy/docker/files/config.yml index 4daba6c56484..789b7a8e8dd7 100644 --- a/packages/tenable_io/_dev/deploy/docker/files/config.yml +++ b/packages/tenable_io/_dev/deploy/docker/files/config.yml @@ -27,7 +27,7 @@ rules: {"id":13670,"name":"Fedora Core 1 : kernel-2.4.22-1.2140.nptl (2003-047)","attributes":{"vpr":{"score":5.5,"updated":"2018-07-19T00:00:00Z","drivers":{"age_of_vuln":{"lower_bound":366,"upper_bound":730},"threat_recency":{"lower_bound":366,"upper_bound":730},"exploit_code_maturity":"UNPROVEN","cvss_impact_score_predicted":false,"cvss3_impact_score":5.9,"threat_intensity_last28":"VERY_LOW","threat_sources_last28":["No recorded events"],"product_coverage":"LOW"}},"plugin_modification_date":"2018-07-19T00:00:00Z","plugin_version":"1.17","exploited_by_malware":false,"description":"Various RTC drivers had the potential to leak...","unsupported_by_vendor":false,"cvss_temporal_score":0,"patch_publication_date":"2004-01-07T00:00:00Z","see_also":["http://example.com/u?07bc9e7f"],"default_account":false,"exploit_available":false,"cve":["CVE-2003-0984"],"exploit_framework_canvas":false,"cvss_base_score":4.6,"solution":"Update the affected packages.","cvss_vector":{"raw":"AV:L/AC:L/Au:N/C:P/I:P/A:P","AccessVector":"Local-access","Availability-Impact":"Partial","Authentication":"None required","Integrity-Impact":"Partial","AccessComplexity":"Low","Confidentiality-Impact":"Partial"},"exploit_framework_exploithub":false,"cpe":["p-cpe:/a:fedoraproject:fedora:kernel-source","cpe:/o:fedoraproject:fedora_core:1","p-cpe:/a:fedoraproject:fedora:kernel-BOOT","p-cpe:/a:fedoraproject:fedora:kernel-debuginfo","p-cpe:/a:fedoraproject:fedora:kernel","p-cpe:/a:fedoraproject:fedora:kernel-doc","p-cpe:/a:fedoraproject:fedora:kernel-smp"],"plugin_publication_date":"2004-07-23T00:00:00Z","exploit_framework_core":false,"in_the_news":false,"has_patch":true,"xref":["FEDORA:2003-047"],"malware":false,"exploit_framework_d2_elliot":false,"xrefs":[{"type":"FEDORA","id":"2003-047"}],"risk_factor":"Medium","synopsis":"The remote Fedora Core host is missing a security update.","cvss3_temporal_score":0,"exploited_by_nessus":false,"cvss3_base_score":0,"exploit_framework_metasploit":false,"plugin_type":"local"}}, {"id":32,"name":"Siemens S7-300 Series PLC CPU Firmware <= 3.2.11 DoS","attributes":{"intel_type":"SENSOR","synopsis":"A Siemens S7-300 programmable logic controller (PLC) has been detected which is vulnerable to a Denial of Service (DoS) attack vector.","description":"Siemens S7-300 PLC central processing units (CPUs) contain an unspecified flaw that may allow a remote attacker to use a specially crafted packet to cause the device to enter defect mode until a cold restart is performed.","solution":"Upgrade the firmware to version 3.2.12 or later.","see_also":[],"plugin_publication_date":"2019-05-21T00:00:00Z","vuln_publication_date":"2016-06-08T00:00:00Z","patch_publication_date":"2016-06-08T00:00:00Z","has_patch":true,"exploit_available":false,"risk_factor":"HIGH","plugin_modification_date":"2019-09-30T00:00:00Z","always_run":false,"compliance":false,"cvss_vector":{"raw":"AV:N/AC:L/Au:N/C:N/I:N/A:C","AccessVector":"Network","AccessComplexity":"Low","Authentication":"None required","Confidentiality-Impact":"None","Integrity-Impact":"None","Availability-Impact":"Complete"},"cvss_temporal_vector":{"raw":"E:U/RL:OF/RC:C","Exploitability":"Unproven","RemediationLevel":"Official Fix","ReportConfidence":"Confirmed"},"cvss_temporal_score":5.8,"cvss3_vector":{"raw":"AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H","AttackVector":"Network","AttackComplexity":"Low","PrivilegesRequired":"None","UserInteraction":"None","Scope":"Unchanged","Confidentiality-Impact":"None","Integrity-Impact":"None","Availability-Impact":"High"},"cvss3_temporal_vector":{"raw":"E:U/RL:O/RC:C","ExploitCodeMaturity":"Unproven","RemediationLevel":"Official Fix","ReportConfidence":"Confirmed"},"cvss3_temporal_score":6.5,"cvss3_base_score":7.5,"cve":["CVE-2016-3949"],"bid":[91133,23432,234234,235],"xref":[],"xrefs":[],"vpr":{"score":3.6,"drivers":{"age_of_vuln":{"lower_bound":731},"exploit_code_maturity":"UNPROVEN","cvss_impact_score_predicted":false,"threat_intensity_last28":"VERY_LOW","threat_sources_last28":["No recorded events"],"product_coverage":"LOW"},"updated":"2021-02-07T05:31:54Z"}}} ] - },"size":1000,"params":{"page":1,"size":1000,"last_updated":""},"total_count":2} + },"size":2,"params":{"page":1,"size":1000,"last_updated":""},"total_count":2} - path: /vulns/export methods: ["POST"] responses: diff --git a/packages/tenable_io/changelog.yml b/packages/tenable_io/changelog.yml index 007300fb8a28..a73d11cafef0 100644 --- a/packages/tenable_io/changelog.yml +++ b/packages/tenable_io/changelog.yml @@ -1,4 +1,12 @@ # newer versions go on top +- version: "3.2.1" + changes: + - description: Fix pagination progression logic. + type: bugfix + link: https://github.com/elastic/integrations/pull/10940 + - description: Fix timestamp logic. + type: bugfix + link: https://github.com/elastic/integrations/pull/10940 - version: "3.2.0" changes: - description: Improve error reporting for API request failures. diff --git a/packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs b/packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs index 6bfc74e3df18..4d5aebea0d41 100644 --- a/packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs +++ b/packages/tenable_io/data_stream/plugin/agent/stream/cel.yml.hbs @@ -38,43 +38,45 @@ program: | }, }).do_request().as(resp, resp.StatusCode == 200 ? - bytes(resp.Body).decode_json().as(body, { - "events": has(body.?data.plugin_details) ? body.data.plugin_details.map(e, { "message": e.encode_json() }) : [{}], - "cursor": { - "last_update_time": ( - has(body.?data.plugin_details) && body.data.plugin_details.size() > 0 ? - ( - has(state.?cursor.last_update_time) && body.data.plugin_details.map(e, e.attributes.plugin_modification_date).max() < state.cursor.last_update_time ? - state.cursor.last_update_time - : - body.data.plugin_details.map(e, e.attributes.plugin_modification_date).max() - ) + bytes(resp.Body).decode_json().as(body, + (int(state.?count.orValue(0))+int(body.size)).as(count, + (count < int(body.total_count)).as(want_more, { + "events": has(body.?data.plugin_details) ? + body.data.plugin_details.map(e, { "message": e.encode_json() }) : - state.?cursor.last_update_time.orValue(string(timestamp(0))) - ), - "first_update_time": ( - has(state.?cursor.first_update_time) && has(body.data.plugin_details) ? - ( - body.data.plugin_details.size() > 0 ? + [], + "cursor": { + "last_update_time": ( + has(body.?data.plugin_details) && body.data.plugin_details.size() > 0 ? + ([?state.?cursor.last_update_time] + body.data.plugin_details.map(e, + e.attributes.plugin_modification_date + )).map(t, + timestamp(t) + ).max() + : + state.?cursor.last_update_time.orValue(string(timestamp(0))) + ), + "first_update_time": ( + has(state.?cursor.first_update_time) && has(body.data.plugin_details) ? ( - body.size == int(state.batch_size) && state.want_more ? + want_more ? state.cursor.first_update_time : state.cursor.last_update_time ) : - state.cursor.last_update_time - ) - : - string(timestamp(0)) - ), - }, - "next_page": body.size == int(state.batch_size) ? int(state.next_page)+1 : 1, - "want_more": body.size == int(state.batch_size), - "access_key": state.access_key, - "secret_key": state.secret_key, - "batch_size": state.batch_size, - }) + string(timestamp(0)) + ), + }, + "next_page": want_more ? int(state.next_page)+1 : 1, + "want_more": want_more, + "count": count, + "access_key": state.access_key, + "secret_key": state.secret_key, + "batch_size": state.batch_size, + }) + ) + ) : { "events": { diff --git a/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml b/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml index 87086e676885..dddf802cfdaf 100644 --- a/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml +++ b/packages/tenable_io/data_stream/plugin/elasticsearch/ingest_pipeline/default.yml @@ -21,6 +21,10 @@ processors: - json: field: event.original target_field: json + - fail: + tag: cel_failure + if: ctx.error?.message != null && ctx.message == null && ctx.event?.original == null + message: error message set and no data to process - drop: if: ctx.json?.data?.plugin_details != null && ctx.json.data.plugin_details.isEmpty() - fingerprint: diff --git a/packages/tenable_io/manifest.yml b/packages/tenable_io/manifest.yml index 8d82b4ed1b4c..7e1762077f72 100644 --- a/packages/tenable_io/manifest.yml +++ b/packages/tenable_io/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: tenable_io title: Tenable Vulnerability Management -version: "3.2.0" +version: "3.2.1" description: Collect logs from Tenable Vulnerability Management with Elastic Agent. type: integration categories: