Remove event.original removal processors (integrations AWS & Azure)

packages/aws/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.25.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10888
 - version: "2.24.1"
   changes:
     - description: Fixed and refactored AWS cloudfront log parsing.

packages/aws/data_stream/apigateway_logs/elasticsearch/ingest_pipeline/default.yml

@@ -81,11 +81,6 @@ processors:
       field: aws.apigateway.status
       type: long
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
   - date:
       field: aws.apigateway.request_time
       target_field: "aws.apigateway.request_time"

packages/aws/data_stream/cloudfront_logs/elasticsearch/ingest_pipeline/default.yml

@@ -434,13 +434,6 @@ processors:
   - remove:
       field: _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !ctx.tags.contains('preserve_original_event')
-      ignore_failure: true
-      ignore_missing: true
-
-
 on_failure:
   - set:
       field: event.kind

packages/aws/data_stream/cloudtrail/elasticsearch/ingest_pipeline/default.yml

@@ -779,11 +779,6 @@ processors:
   - remove:
       field: json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

packages/aws/data_stream/ec2_logs/elasticsearch/ingest_pipeline/default.yml

@@ -30,11 +30,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/data_stream/elb_logs/elasticsearch/ingest_pipeline/default.yml

@@ -221,11 +221,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/data_stream/emr_logs/elasticsearch/ingest_pipeline/default.yml

@@ -51,11 +51,6 @@ processors:
           return false;
         }
         drop(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml

@@ -387,11 +387,6 @@ processors:
   - remove:
       field: json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: 'ctx?.tags == null || !(ctx.tags.contains("preserve_original_event"))'
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

packages/aws/data_stream/guardduty/elasticsearch/ingest_pipeline/default.yml

@@ -2451,10 +2451,6 @@ processors:
                 - _ingest._value.file.path
                 - _ingest._value.hash
               ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

packages/aws/data_stream/inspector/elasticsearch/ingest_pipeline/default.yml

@@ -701,10 +701,6 @@ processors:
             - _ingest._value.version
           if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
           ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

packages/aws/data_stream/route53_public_logs/elasticsearch/ingest_pipeline/default.yml

@@ -119,11 +119,6 @@ processors:
       field:
         - _tmp
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/data_stream/route53_resolver_logs/elasticsearch/ingest_pipeline/default.yml

@@ -285,11 +285,6 @@ processors:
         list.removeIf(v -> v == null || v == '' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0));
       }
       handleMap(ctx);
-- remove:
-    field: event.original
-    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    ignore_failure: true
-    ignore_missing: true
 on_failure:
 - set:
     field: 'error.message'

packages/aws/data_stream/s3access/elasticsearch/ingest_pipeline/default.yml

@@ -215,11 +215,6 @@ processors:
           }
         }
         handleMap(ctx);
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/data_stream/securityhub_findings/elasticsearch/ingest_pipeline/default.yml

@@ -2137,11 +2137,6 @@ processors:
           ignore_failure: true
           ignore_missing: true
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

packages/aws/data_stream/securityhub_insights/elasticsearch/ingest_pipeline/default.yml

@@ -987,11 +987,6 @@ processors:
       field:
         - json
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

packages/aws/data_stream/vpcflow/elasticsearch/ingest_pipeline/default.yml

@@ -304,11 +304,6 @@ processors:
         - aws.vpcflow.packets
         - aws.vpcflow.protocol
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/data_stream/waf/elasticsearch/ingest_pipeline/default.yml

@@ -197,11 +197,6 @@ processors:
             }
         }
         handleMap(ctx);
-- remove:
-    field: event.original
-    if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-    ignore_failure: true
-    ignore_missing: true
 on_failure:
   - set:
       field: 'error.message'

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.24.1
+version: 2.25.0
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:

packages/aws_bedrock/changelog.yml

@@ -1,3 +1,8 @@
+- version: "0.9.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10888
 - version: "0.8.0"
   changes:
     - description: Update integration name to Amazon Bedrock.

packages/aws_bedrock/data_stream/invocation/elasticsearch/ingest_pipeline/default.yml

@@ -570,11 +570,6 @@ processors:
       tag: remove_duplicate_custom_fields
       ignore_missing: true
       if: ctx.tags == null || !(ctx.tags.contains('preserve_duplicate_custom_fields'))
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Remove massive event.original if it was wanted.
       if: ctx.event?.original instanceof String && ctx.event.original.length() > 32766

packages/aws_bedrock/manifest.yml

@@ -1,9 +1,9 @@
 format_version: "3.0.2"
 name: aws_bedrock
-title: Amazon Bedrock
-description: Collect Amazon Bedrock model invocation logs and runtime metrics with Elastic Agent.
+title: AWS Bedrock
+description: Collect AWS Bedrock model invocation logs and runtime metrics with Elastic Agent.
 type: integration
-version: "0.8.0"
+version: "0.9.0"
 categories:
   - aws
 conditions:
@@ -13,8 +13,8 @@ conditions:
     subscription: basic
 policy_templates:
   - name: aws_bedrock
-    title: Amazon Bedrock
-    description: Collect Amazon Bedrock model invocation logs with Elastic Agent.
+    title: AWS Bedrock
+    description: Collect AWS Bedrock model invocation logs with Elastic Agent.
     inputs:
       - type: aws-s3
         title: Collect Logs from S3 Bucket
@@ -23,8 +23,8 @@ policy_templates:
         title: Collect Logs from CloudWatch
         description: Collect bedrock logs from CloudWatch with Elastic Agent.
       - type: aws/metrics
-        title: Collect Amazon Bedrock metrics
-        description: Collect Amazon Bedrock metrics using AWS CloudWatch.
+        title: Collect AWS Bedrock metrics
+        description: Collect AWS Bedrock metrics using AWS CloudWatch.
 screenshots:
   - src: /img/add_aws_bedrock.png
     title: Overview

packages/azure/changelog.yml

@@ -1,3 +1,8 @@
+- version: "1.16.0"
+  changes:
+    - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/10888
 - version: "1.15.1"
   changes:
     - description: Fix [client|source].geo.location ECS field mapping

packages/azure/data_stream/activitylogs/elasticsearch/ingest_pipeline/default.yml

@@ -315,11 +315,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

packages/azure/data_stream/application_gateway/elasticsearch/ingest_pipeline/default.yml

@@ -282,11 +282,6 @@ processors:
       field: observer.name
       copy_from: azure.resource.name
       ignore_empty_value: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

packages/azure/data_stream/auditlogs/elasticsearch/ingest_pipeline/default.yml

@@ -231,11 +231,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

packages/azure/data_stream/eventhub/elasticsearch/ingest_pipeline/parsed-message.yml

@@ -42,11 +42,6 @@ processors:
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
       ignore_failure: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message

packages/azure/data_stream/firewall_logs/elasticsearch/ingest_pipeline/default.yml

@@ -396,11 +396,6 @@ processors:
       field: observer.name
       copy_from: azure.resource.name
       ignore_empty_value: true
-  - remove:
-      field: event.original
-      if: ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))
-      ignore_failure: true
-      ignore_missing: true
   - script:
       description: Drops null/empty values recursively.
       lang: painless

packages/azure/data_stream/graphactivitylogs/elasticsearch/ingest_pipeline/default.yml

@@ -429,11 +429,6 @@ processors:
       field: client.as.organization_name
       target_field: client.as.organization.name
       ignore_missing: true
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_missing: true
-      tag: remove-event_original
   - script:
       tag: script_to_drop_null_values
       lang: painless

packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-rickyusers-raw.log-expected.json

@@ -37,7 +37,8 @@
             "event": {
                 "action": "Risky user",
                 "duration": 0,
-                "kind": "event"
+                "kind": "event",
+                "original": "{\"time\":\"8/22/2022 6:11:52 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"neu\",\"properties\":{\"id\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"[email protected]\",\"riskLastUpdatedDateTime\":\"2022-08-22T18:11:52.702Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"medium\",\"isGuest\":false,\"isDeleted\":false,\"isProcessing\":false}}"
             }
         },
         {
@@ -77,7 +78,8 @@
             "event": {
                 "action": "Risky user",
                 "duration": 0,
-                "kind": "event"
+                "kind": "event",
+                "original": "{\"time\":\"9/9/2022 9:59:27 AM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"Risky user\",\"operationVersion\":\"1.0\",\"category\":\"RiskyUsers\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"correlationId\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"identity\":\"joel miller\",\"Level\":4,\"location\":\"weu\",\"properties\":{\"id\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"userDisplayName\":\"Joel Miller\",\"userPrincipalName\":\"[email protected]\",\"riskLastUpdatedDateTime\":\"2022-09-09T9:59:27.958Z\",\"riskState\":\"atRisk\",\"riskDetail\":\"none\",\"riskLevel\":\"high\",\"isGuest\":false,\"isDeleted\":true,\"isProcessing\":false}}"
             }
         }
     ]

packages/azure/data_stream/identity_protection/_dev/test/pipeline/test-userriskevents-raw.log-expected.json

@@ -63,7 +63,8 @@
             "event": {
                 "action": "User Risk Detection",
                 "duration": 0,
-                "kind": "event"
+                "kind": "event",
+                "original": "{\"time\":\"8/22/2022 6:07:16 PM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"User Risk Detection\",\"operationVersion\":\"1.0\",\"category\":\"UserRiskEvents\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"callerIpAddress\":\"67.43.156.42\",\"correlationId\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"identity\":\"joe danger\",\"Level\":4,\"location\":\"de\",\"properties\":{\"id\":\"ce0ed07f9ccf5be15e4b97d2979af6569b1f67db87ddc9b88b5bb743ea091e47\",\"requestId\":\"e1b6d9d7-5fc0-4638-ae1a-e0abceb92200\",\"correlationId\":\"266133c2-fabb-492f-9ebf-bdf12317b817\",\"riskType\":\"anonymizedIPAddress\",\"riskEventType\":\"anonymizedIPAddress\",\"riskState\":\"atRisk\",\"riskLevel\":\"high\",\"riskDetail\":\"none\",\"source\":\"IdentityProtection\",\"detectionTimingType\":\"realtime\",\"activity\":\"signin\",\"ipAddress\":\"67.43.156.42\",\"location\":{\"city\":\"Dresden\",\"state\":\"Sachsen\",\"countryOrRegion\":\"DE\",\"geoCoordinates\":{\"altitude\":0,\"latitude\":51.0714,\"longitude\":13.7399}},\"activityDateTime\":\"2022-08-22T18:05:06.133Z\",\"detectedDateTime\":\"2022-08-22T18:05:06.133Z\",\"lastUpdatedDateTime\":\"2022-08-22T18:07:16.894Z\",\"userId\":\"51e26eae-d07b-44e5-bb0b-249f49569a8c\",\"userDisplayName\":\"Joe Danger\",\"userPrincipalName\":\"[email protected]\",\"additionalInfo\":\"[{\\\"Key\\\":\\\"userAgent\\\",\\\"Value\\\":\\\"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0\\\"}]\",\"tokenIssuerType\":\"AzureAD\",\"resourceTenantId\":null,\"homeTenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"userType\":\"member\",\"crossTenantAccessType\":\"none\"}}"
             },
             "source": {
                 "as": {
@@ -144,7 +145,8 @@
             "event": {
                 "action": "User Risk Detection",
                 "duration": 0,
-                "kind": "event"
+                "kind": "event",
+                "original": "{\"time\":\"9/9/2022 9:54:27 AM\",\"resourceId\":\"/tenants/5611623b-9128-461e-9d7f-a0d9c270ead2/providers/microsoft.aadiam\",\"operationName\":\"User Risk Detection\",\"operationVersion\":\"1.0\",\"category\":\"UserRiskEvents\",\"tenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"resultSignature\":\"None\",\"durationMs\":0,\"callerIpAddress\":\"67.43.156.42\",\"correlationId\":\"d93ffb8db664e4e8bd08db6e183b6390dacf00553cc5768aac83012e2dc7c2c9\",\"identity\":\"joel miller\",\"Level\":4,\"location\":\"us\",\"properties\":{\"id\":\"d93ffb8db664e4e8bd08db6e183b6390dacf00553cc5768aac83012e2dc7c2c9\",\"requestId\":\"52a6a863-695a-42b1-b299-e44121b93f00\",\"correlationId\":\"cbfd38f4-1619-45e5-ba71-71456f75541b\",\"riskType\":\"generic\",\"riskEventType\":\"passwordSpray\",\"riskState\":\"atRisk\",\"riskLevel\":\"high\",\"riskDetail\":\"none\",\"source\":\"IdentityProtection\",\"detectionTimingType\":\"offline\",\"activity\":\"signin\",\"ipAddress\":\"67.43.156.42\",\"location\":{\"city\":\"Staten Island\",\"state\":\"New York\",\"countryOrRegion\":\"US\",\"geoCoordinates\":{\"altitude\":0,\"latitude\":40.59615,\"longitude\":-74.16161}},\"activityDateTime\":\"2022-09-08T12:53:53.895Z\",\"detectedDateTime\":\"2022-09-09T8:48:50.361Z\",\"lastUpdatedDateTime\":\"2022-09-09T9:54:27.739Z\",\"userId\":\"e3b2b242-4ccb-4cf1-9b8b-004cf034a458\",\"userDisplayName\":\"Joel Miller\",\"userPrincipalName\":\"[email protected]\",\"additionalInfo\":\"[{\\\"Key\\\":\\\"userAgent\\\",\\\"Value\\\":\\\"Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0\\\"}]\",\"tokenIssuerType\":\"AzureAD\",\"resourceTenantId\":null,\"homeTenantId\":\"5611623b-9128-461e-9d7f-a0d9c270ead2\",\"userType\":\"member\",\"crossTenantAccessType\":\"none\"}}"
             },
             "source": {
                 "as": {

packages/azure/data_stream/identity_protection/elasticsearch/ingest_pipeline/default.yml

@@ -289,11 +289,6 @@ processors:
       value: event
   - pipeline:
       name: '{{ IngestPipeline "azure-shared-pipeline" }}'
-  - remove:
-      field: event.original
-      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
-      ignore_failure: true
-      ignore_missing: true
 on_failure:
   - set:
       field: error.message