Exclude data node by pod name, not by NODE_IP

* Added possibility to override namespace

README.md

@@ -16,6 +16,10 @@ Kibana deployment has built-in [Kaae](https://github.com/elasticfence/kaae) plug
 
 ## Assumptions
 
+### Namespace
+
+This example uses `monitoring` namespace. If you wish to use your own namespace, just export `NAMESPACE=mynamespace` environment variable.
+
 ### Insecure Elasticsearch connections
 
 This repo should not be used in production when you use insecure public network. Fluentd is configured to send logs to Elasticsearch using insecure connection.
@@ -201,6 +205,7 @@ or wait until new index will be created (in our setup new index is being created
 * `journald` logs don't show up in Kibana, probably because of the TZ issues
 * `DELETED` Kubernetes events could not be stripped for now, you have to create an exclude rule for `type:"DELETED"`, otherwise these events confuse Kibana users.
 * Kubernetes < v1.3.6 has a [bug](https://github.com/kubernetes/kubernetes/issues/35333) which stops pause container before graceful Elasticsearch pod shutdown which results in inaccessible data pod while moving out shards.
+* In some cases after rolling update each new pod gets the same IP addresse as an old one. This results in one empty node after rolling update is done. New [`docker/elasticsearch/pre-stop-hook.sh`](docker/elasticsearch/pre-stop-hook.sh) already contains a fix, but you have to manually clear the `cluster.routing.allocation.exclude._host` option: `curl -XPUT http://elk:9200/_cluster/settings -d'{ "transient" :{ "cluster.routing.allocation.exclude._host" : "" } }"'`.
 
 # TODO
 

apply_labels_on_nodes.sh

@@ -12,7 +12,7 @@ print_green() {
 }
 
 #KUBECTL_PARAMS="--context=foo"
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 NODES=$(eval "${KUBECTL} get nodes -o go-template=\"{{range .items}}{{\\\$name := .metadata.name}}{{\\\$unschedulable := .spec.unschedulable}}{{range .status.conditions}}{{if eq .reason \\\"KubeletReady\\\"}}{{if eq .status \\\"True\\\"}}{{if not \\\$unschedulable}}{{\\\$name}}{{\\\"\\\\n\\\"}}{{end}}{{end}}{{end}}{{end}}{{end}}\"")

deploy.sh

@@ -16,7 +16,7 @@ render_template() {
 }
 
 #KUBECTL_PARAMS="--context=foo"
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 eval "kubectl ${KUBECTL_PARAMS} create namespace \"${NAMESPACE}\""

docker/elasticsearch/pre-stop-hook.sh

@@ -16,7 +16,7 @@ echo "Move all data from node ${NODE_IP}"
 
 curl -s -XPUT "http://${CLIENT_ENDPOINT}/_cluster/settings" -d "{
   \"transient\" :{
-      \"cluster.routing.allocation.exclude._host\" : \"${NODE_IP}\"
+      \"cluster.routing.allocation.exclude._name\" : \"${HOSTNAME}\"
    }
 }"
 echo

undeploy.sh

@@ -12,7 +12,7 @@ print_green() {
 }
 
 #KUBECTL_PARAMS="--context=foo"
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 INSTANCES="deployment/es-client

update_es_clients.sh

@@ -4,7 +4,7 @@ CDIR=$(cd `dirname "$0"` && pwd)
 cd "$CDIR"
 
 #KUBECTL_PARAMS="--context=foo"
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 eval "${KUBECTL} apply -f es-env.yaml"

update_es_data.sh

@@ -16,7 +16,7 @@ render_template() {
 }
 
 #KUBECTL_PARAMS="--context=foo"
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 NODES=$(eval "${KUBECTL} get nodes -o go-template=\"{{range .items}}{{\\\$name := .metadata.name}}{{\\\$unschedulable := .spec.unschedulable}}{{range .status.conditions}}{{if eq .reason \\\"KubeletReady\\\"}}{{if eq .status \\\"True\\\"}}{{if not \\\$unschedulable}}{{\\\$name}}{{\\\"\\\\n\\\"}}{{end}}{{end}}{{end}}{{end}}{{end}}\"")

update_fluentd_config.sh

@@ -3,7 +3,7 @@
 CDIR=$(cd `dirname "$0"` && pwd)
 cd "$CDIR"
 
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 eval "${KUBECTL} create configmap fluentd-config --from-file=docker/fluentd/td-agent.conf --dry-run -o yaml" | eval "${KUBECTL} apply -f -"

update_fluentd_ds.sh

@@ -3,7 +3,7 @@
 CDIR=$(cd `dirname "$0"` && pwd)
 cd "$CDIR"
 
-NAMESPACE="monitoring"
+NAMESPACE=${NAMESPACE:-monitoring}
 KUBECTL="kubectl ${KUBECTL_PARAMS} --namespace=\"${NAMESPACE}\""
 
 eval "${KUBECTL} apply -f es-fluentd-ds.yaml"