run ingest pipeline tests

packages/aws/changelog.yml

@@ -1,9 +1,4 @@
 # newer versions go on top
-- version: "2.26.0"
-  changes:
-    - description: "Add aws.metrics_names_fingerprint to cloudwatch metrics"
-      type: enhancement
-      link: https://github.com/elastic/integrations/pull/11239
 - version: "2.25.0"
   changes:
     - description: "Allow @custom pipeline access to event.original without setting preserve_original_event."

packages/aws/data_stream/cloudwatch_metrics/elasticsearch/ingest_pipeline/default.yml

@@ -1,51 +1,10 @@
 ---
 description: "Pipeline for AWS CloudWatch metrics"
 processors:
-  - dot_expander:
-      field: "*"
-      ignore_failure: true
   - fingerprint:
       fields: ["aws.dimensions"]
       target_field: 'aws.dimensions_fingerprint'
       ignore_missing: true
-  - append:
-      field: "aws.metrics_names"
-      value: [ null ]
-      allow_duplicates: false
-      description: Create an empty field of the object type to temporarily store list of all metrics names
-  - script:
-      lang: painless
-      description: This script adds all metric names from 'aws.*.metrics.*.*' into aws.metrics_names field.
-      if: ctx.aws != null
-      source: >-
-          List metricNames = new ArrayList();
-          if (ctx.aws != null && ctx.aws instanceof Map) {
-              for (entry in ctx.aws.entrySet()) {
-                  String topLevelKey = entry.getKey();  // Get the top-level key (kinesis, cloudwatch, etc.)
-                  def nestedMap = entry.getValue();
-                  if (nestedMap instanceof Map) {
-                      for (nestedEntry in nestedMap.entrySet()) {
-                          String key = nestedEntry.getKey(); // Get the second-level key (metrics, namespace, etc.)
-                          if (key.equals("metrics")) {
-                              def metricsMap = nestedEntry.getValue(); // Get aws.*.metrics.*
-                              if (metricsMap instanceof Map) {
-                                  for (metricsEntry in metricsMap.entrySet()) {
-                                      metricNames.add(metricsEntry.getKey());
-                                  }
-                              }
-                          }
-                      }
-                  }
-              }
-          }
-          ctx.aws.metrics_names = metricNames;
-  - fingerprint:
-      fields: [ "aws.metrics_names" ]
-      target_field: "aws.metrics_names_fingerprint"
-      ignore_missing: true
-  - remove:
-      field: "aws.metrics_names"
-      description: Remove field as it contains a redundant information and can impact the documents size
 on_failure:
   - set:
       field: error.message

packages/aws/data_stream/cloudwatch_metrics/fields/package-fields.yml

@@ -14,11 +14,6 @@
       description: |
         Autogenerated ID representing the fingerprint of the aws.dimensions object
       dimension: true
-    - name: metrics_names_fingerprint
-      type: keyword
-      description: |
-        Autogenerated ID representing the fingerprint of the list of metrics names
-      dimension: true
 - name: 'aws.*.metrics.*.*'
   type: object
   object_type: double

packages/aws/docs/cloudwatch.md

@@ -185,7 +185,6 @@ Please refer to the following [document](https://www.elastic.co/guide/en/ecs/cur
 | aws.cloudwatch.namespace | The namespace specified when query cloudwatch api. | keyword |  |
 | aws.dimensions | Metric dimensions. | flattened |  |
 | aws.dimensions_fingerprint | Autogenerated ID representing the fingerprint of the aws.dimensions object | keyword |  |
-| aws.metrics_names_fingerprint | Autogenerated ID representing the fingerprint of the list of metrics names | keyword |  |
 | aws.tags | Tag key value pairs from aws resources. | flattened |  |
 | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword |  |
 | cloud.image.id | Image ID for the cloud instance. | keyword |  |

packages/aws/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.0
 name: aws
 title: AWS
-version: 2.26.0
+version: 2.25.0
 description: Collect logs and metrics from Amazon Web Services (AWS) with Elastic Agent.
 type: integration
 categories:

packages/awsfirehose/changelog.yml

@@ -1,9 +1,12 @@
 # newer versions go on top
-- version: "1.2.1"
+- version: "1.3.0"
   changes:
     - description: Add aws.firehose.arn and aws.firehose.request_id fields.
       type: bugfix
       link: https://github.com/elastic/integrations/pull/11239
+    - description: Add aws.metrics_names_fingerprint to cloudwatch metrics.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/11239
 - version: "1.2.0"
   changes:
     - description: Add AWS Bedrock runtime metrics to Firehose rerouting.

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-apigateway-metric.json-expected.json

@@ -2,27 +2,50 @@
     "expected": [
         {
             "@timestamp": "2024-05-01T08:24:00Z",
-            "agent.type": "firehose",
-            "aws.apigateway.metrics.4XXError": {
-                "count": 1,
-                "max": 0.0,
-                "min": 0.0,
-                "sum": 0.0
+            "agent": {
+                "type": "firehose"
             },
-            "aws.cloudwatch.namespace": "AWS/ApiGateway",
-            "aws.dimensions": {
-                "ApiId": "6am7mj7jqx"
+            "aws": {
+                "apigateway": {
+                    "metrics": {
+                        "4XXError": {
+                            "count": 1,
+                            "max": 0.0,
+                            "min": 0.0,
+                            "sum": 0.0
+                        }
+                    }
+                },
+                "cloudwatch": {
+                    "namespace": "AWS/ApiGateway"
+                },
+                "dimensions": {
+                    "ApiId": "6am7mj7jqx"
+                },
+                "exporter": {
+                    "arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics"
+                },
+                "firehose": {
+                    "arn": "arn:aws:firehose:ap-southeast-2:12345678901:deliverystream/KS-PUT-ELI-2",
+                    "parameters": {
+                        "X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1"
+                    },
+                    "request_id": "adcedd1d-00d2-4499-a917-4fba68f55427"
+                },
+                "metrics_names_fingerprint": "b3FPw421v4azg2Ho4+VG3EZXNFI="
+            },
+            "cloud": {
+                "account": {
+                    "id": "12345678901"
+                },
+                "provider": "aws",
+                "region": "ap-southeast-2"
+            },
+            "data_stream": {
+                "dataset": "aws.apigateway_metrics",
+                "namespace": "default",
+                "type": "metrics"
             },
-            "aws.exporter.arn": "arn:aws:cloudwatch:ap-southeast-2:12345678901:metric-stream/test-firehose-metrics",
-            "aws.firehose.arn": "arn:aws:firehose:ap-southeast-2:12345678901:deliverystream/KS-PUT-ELI-2",
-            "aws.firehose.parameters.X-Found-Cluster": "bb8e51259abe4c21996954f5cfe90af1",
-            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
-            "cloud.account.id": "12345678901",
-            "cloud.provider": "aws",
-            "cloud.region": "ap-southeast-2",
-            "data_stream.dataset": "aws.apigateway_metrics",
-            "data_stream.namespace": "default",
-            "data_stream.type": "metrics",
             "ecs": {
                 "version": "8.11.0"
             },

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json

@@ -21,7 +21,10 @@
                 "min": 18430775296,
                 "max": 18430775296
             },
-            "aws.firehose.parameters.X-Found-Cluster": "c335685bb4614254a283e02799f09d63"
+            "aws.firehose.parameters.X-Found-Cluster": "c335685bb4614254a283e02799f09d63",
+            "data_stream.type": "metrics",
+            "data_stream.dataset": "aws.cloudwatch",
+            "data_stream.namespace": "default"
         }
     ]
 }

packages/awsfirehose/data_stream/metrics/_dev/test/pipeline/test-rds-metrics.json-expected.json

@@ -2,25 +2,46 @@
     "expected": [
         {
             "@timestamp": "2024-06-12T22:00:00.000Z",
-            "agent.type": "firehose",
-            "aws.cloudwatch.namespace": "AWS/RDS",
-            "aws.dimensions": {
-                "DBInstanceIdentifier": "metricbeat-test-019e245b"
+            "agent": {
+                "type": "firehose"
             },
-            "aws.exporter.arn": "arn:aws:cloudwatch:us-east-1:12345678901:metric-stream/test-all-metrics-ks-1",
-            "aws.firehose.arn": "arn:aws:firehose:us-east-1:12345678901:deliverystream/KS-PUT-ELI-2",
-            "aws.firehose.parameters.X-Found-Cluster": "c335685bb4614254a283e02799f09d63",
-            "aws.firehose.request_id": "adcedd1d-00d2-4499-a917-4fba68f55427",
-            "aws.rds.metrics.FreeStorageSpace": {
-                "avg": 18430775296,
-                "count": 1,
-                "max": 18430775296,
-                "min": 18430775296,
-                "sum": 18430775296
+            "aws": {
+                "cloudwatch": {
+                    "namespace": "AWS/RDS"
+                },
+                "dimensions": {
+                    "DBInstanceIdentifier": "metricbeat-test-019e245b"
+                },
+                "exporter": {
+                    "arn": "arn:aws:cloudwatch:us-east-1:12345678901:metric-stream/test-all-metrics-ks-1"
+                },
+                "firehose": {
+                    "arn": "arn:aws:firehose:us-east-1:12345678901:deliverystream/KS-PUT-ELI-2",
+                    "parameters": {
+                        "X-Found-Cluster": "c335685bb4614254a283e02799f09d63"
+                    },
+                    "request_id": "adcedd1d-00d2-4499-a917-4fba68f55427"
+                },
+                "metrics_names_fingerprint": "IpH2eh4JmKtHPgFDv5deyrfYOlk=",
+                "rds": {
+                    "metrics": {
+                        "FreeStorageSpace": {
+                            "avg": 18430775296,
+                            "count": 1,
+                            "max": 18430775296,
+                            "min": 18430775296,
+                            "sum": 18430775296
+                        }
+                    }
+                }
+            },
+            "cloud": {
+                "account": {
+                    "id": "12345678901"
+                },
+                "provider": "aws",
+                "region": "us-east-1"
             },
-            "cloud.account.id": "12345678901",
-            "cloud.provider": "aws",
-            "cloud.region": "us-east-1",
             "data_stream": {
                 "dataset": "aws.rds",
                 "namespace": "default",