diff --git a/README.md b/README.md
index 1347a65..68a41b2 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ This repo deploys complete ELK stack (actually **EFK**: **Elasticsearch, Fluentd
This repo already contains fluentd configuration example which works in most cases. It contains log modification examples, Java backtrace multiline logs processing, log parsing examples, [Kubernetes events processing](#forward-kubernetes-events-into-kibanaelasticsearch) and more.
-Kibana deployment has built-in [Kaae](https://github.com/elasticfence/kaae) plugin which allows to generate notifications on logs anomalies.
+Kibana deployment has built-in [Kaae](https://github.com/elasticfence/kaae) plugin which allows to generate notifications on logs anomalies. See [watcher example](kaae_watchers) (should be stored at https://kibana.example.com/app/kaae).
## Assumptions
diff --git a/deploy.sh b/deploy.sh
index 751d8df..0ed74a0 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -45,4 +45,6 @@ done
eval "${KUBECTL} create configmap fluentd-config --from-file=docker/fluentd/td-agent.conf --dry-run -o yaml" | eval "${KUBECTL} apply -f -"
+eval "${KUBECTL} create configmap kaae-config --from-file=kaae.json --dry-run -o yaml" | eval "${KUBECTL} apply -f -"
+
eval "${KUBECTL} get pods $@"
diff --git a/docker/fluentd/td-agent.conf b/docker/fluentd/td-agent.conf
index 1a84278..a7183f1 100644
--- a/docker/fluentd/td-agent.conf
+++ b/docker/fluentd/td-agent.conf
@@ -313,6 +313,30 @@
flush_interval 1s
+################## Process java multiline logs ##################
+
+
+ # Convert multiline logs into oneliners
+ type concat
+ # Suppress timeout warnings
+ log_level error
+ key log
+ multiline_start_regexp /^[^\s]+/
+ flush_interval 1s
+
+
+
+ # Convert multiline logs with prefix into oneliners
+ # [2016-10-25T01:15:35.018][Instance 1][Port 12345] log log log
+ # [2016-10-25T01:15:35.018][Instance 1][Port 12345] multiline
+ type concat
+ # Suppress timeout warnings
+ log_level error
+ key log
+ multiline_start_regexp /^(?:\[[^\]]+\])+\s[^\s]+/
+ flush_interval 1s
+
+
################## Strip fluentd concat logs ##################
@@ -433,7 +457,7 @@
type parser
# GET /media.jpg\n Accept: image/png, image/svg+xml, image/*;q=0.8, */*;q=0.5\n Status: 200 OK 0.00003s
- format /^(?\S+) (?[^ ]*)\n+.*\s+Status: (?[^ ]*).*/
+ format /^(?\S+) (?[^ ]*)\n+\s+(?:Params: \[(?[^\]]+)\])?(?:\n|.)*Status: (?[^ ]*).*/
time_format %d/%b/%Y:%H:%M:%S %z
key_name log
types code:integer
diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile
index eb3dd56..b71a866 100644
--- a/docker/kibana/Dockerfile
+++ b/docker/kibana/Dockerfile
@@ -20,7 +20,7 @@ RUN apk add --update curl ca-certificates sudo && \
RUN /kibana-linux-x86_64/bin/kibana plugin --install kaae -u https://github.com/elasticfence/kaae/releases/download/snapshot/kaae-latest.tar.gz
-RUN sed -i s/@timestamp/time/ /kibana-linux-x86_64/installedPlugins/kaae/kaae.json
+RUN ln -s /etc/kaae/kaae.json /etc/kaae.json
# Copy run script
COPY run.sh /
diff --git a/es-kibana.yaml b/es-kibana.yaml
index 9f4d524..bc6198e 100644
--- a/es-kibana.yaml
+++ b/es-kibana.yaml
@@ -44,3 +44,11 @@ spec:
- containerPort: 5601
name: ui
protocol: TCP
+ volumeMounts:
+ - name: kaae-config
+ mountPath: /etc/kaae
+ readOnly: true
+ volumes:
+ - name: kaae-config
+ configMap:
+ name: kaae-config
diff --git a/kaae.json b/kaae.json
new file mode 100644
index 0000000..f612d01
--- /dev/null
+++ b/kaae.json
@@ -0,0 +1,35 @@
+{
+ "es": {
+ "alarm_index": "watcher_alarms",
+ "default_index": "watcher",
+ "timefield": "time",
+ "type": "watch"
+ },
+ "kaae": {
+ "history": 20,
+ "results": 50
+ },
+ "settings": {
+ "email": {
+ "active": false,
+ "host": "smtp.server.com",
+ "password": "password",
+ "ssl": true,
+ "user": "username"
+ },
+ "pushapps": {
+ "active": false,
+ "api_key": ""
+ },
+ "report": {
+ "active": false,
+ "tmp_path": "/tmp/"
+ },
+ "slack": {
+ "active": true,
+ "channel": "#devops",
+ "hook": "https://hooks.slack.com/services/abc123",
+ "username": "KAAE"
+ }
+ }
+}
diff --git a/kaae_watchers/kubernetes_events.json b/kaae_watchers/kubernetes_events.json
new file mode 100644
index 0000000..81a5921
--- /dev/null
+++ b/kaae_watchers/kubernetes_events.json
@@ -0,0 +1,102 @@
+{
+ "_index": "watcher",
+ "_type": "watch",
+ "_id": "kubernetes_events",
+ "_score": 1,
+ "_source": {
+ "trigger": {
+ "schedule": {
+ "later": "every 1 minute"
+ }
+ },
+ "input": {
+ "search": {
+ "request": {
+ "index": [],
+ "body": {
+ "query": {
+ "filtered": {
+ "query": {
+ "query_string": {
+ "query": "kubernetes.container_name:\"kubernetes-events-printer\" AND NOT type:\"DELETED\"",
+ "analyze_wildcard": true
+ }
+ },
+ "filter": {
+ "range": {
+ "time": {
+ "from": "now-1m"
+ }
+ }
+ }
+ }
+ },
+ "highlight": {
+ "pre_tags": [
+ "@kibana-highlighted-field@"
+ ],
+ "post_tags": [
+ "@/kibana-highlighted-field@"
+ ],
+ "fields": {
+ "*": {}
+ },
+ "require_field_match": false,
+ "fragment_size": 2147483647
+ },
+ "size": 500,
+ "sort": [
+ {
+ "time": {
+ "order": "desc",
+ "unmapped_type": "boolean"
+ }
+ }
+ ],
+ "aggs": {
+ "2": {
+ "date_histogram": {
+ "field": "time",
+ "interval": "30s",
+ "time_zone": "Europe/Berlin",
+ "min_doc_count": 0,
+ "extended_bounds": {
+ "min": 1477649080605,
+ "max": 1477649980605
+ }
+ }
+ }
+ },
+ "fields": [
+ "*",
+ "_source"
+ ],
+ "script_fields": {},
+ "fielddata_fields": [
+ "object.metadata.creationTimestamp",
+ "object.firstTimestamp",
+ "object.lastTimestamp",
+ "@timestamp",
+ "time"
+ ]
+ }
+ }
+ }
+ },
+ "condition": {
+ "script": {
+ "script": "payload.hits.total > 2"
+ }
+ },
+ "transform": {},
+ "actions": {
+ "slack_admin": {
+ "throttle_period": "1m",
+ "slack": {
+ "channel": "#devops",
+ "message": "Kubernetes events have been detected {{ payload._id}}: {{ payload.hits.total }}"
+ }
+ }
+ }
+ }
+}
diff --git a/update_kaae_config.sh b/update_kaae_config.sh
new file mode 100755
index 0000000..d4474fc
--- /dev/null
+++ b/update_kaae_config.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+CDIR=$(cd `dirname "$0"` && pwd)
+cd "$CDIR"
+
+NAMESPACE=${NAMESPACE:-monitoring}
+KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
+
+eval "${KUBECTL} create configmap kaae-config --from-file=kaae.json --dry-run -o yaml" | eval "${KUBECTL} apply -f -"
+# Just remove pods and deployments will recreate new ones with updated config file
+eval "${KUBECTL} delete pods -l k8s-app=kibana-logging,version=v2"