Kurl util image (replicatedhq#251)

* Kurl util image * fix make task for building templates * test deploy * reverr testing Co-authored-by: Andrew Reed <[email protected]>
DeDuplicate · Feb 21, 2020 · 4d24bf4 · 4d24bf4
1 parent 6013501
commit 4d24bf4
Show file tree

Hide file tree

Showing 22 changed files with 666 additions and 50 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,16 +1,22 @@
 version: 2
 
-defaults: &defaults
-  docker:
-  - image: replicated/gitops-builder:node8-go12
-  working_directory: ~/repo
-  environment: &environment
-    PROJECT_NAME: kurl
-    ECR_REPO: 923411875752.dkr.ecr.us-east-1.amazonaws.com
-    OVERLAY_PATH: kustomize/overlays/staging
-    GITOPS_BRANCH: master
-    DOCKERFILE: deploy/Dockerfile-slim
-
+references:
+  defaults: &defaults
+    docker:
+    - image: replicated/gitops-builder:node8-go12
+    working_directory: ~/repo
+    environment: &environment
+      PROJECT_NAME: kurl
+      ECR_REPO: 923411875752.dkr.ecr.us-east-1.amazonaws.com
+      OVERLAY_PATH: kustomize/overlays/staging
+      GITOPS_BRANCH: master
+      DOCKERFILE: deploy/Dockerfile-slim
+
+  tag_filters: &tag_filters
+    tags:
+      only: /^v.*/
+    branches:
+      ignore: /.*/
 
 jobs:
   is_upstream:
@@ -29,6 +35,21 @@ jobs:
     steps:
       - checkout
       - run: make -C web test
+      - run: make -C kurl_util deps test
+
+  kurl_util_image:
+    <<: *defaults
+    steps:
+      - checkout
+      - setup_remote_docker:
+          version: 17.06.0-ce
+      - run:
+          name: build and push kurl util image
+          command: |
+            set -veuo pipefail
+            source bin/kurl-util-image-export.sh
+            echo "$DOCKERHUB_DOCKER_PASS" | docker login -u $DOCKERHUB_DOCKER_USER --password-stdin
+            make -C kurl_util build-and-push-kurl-util-image
 
   upload_staging_packages:
     <<: *defaults
@@ -43,6 +64,7 @@ jobs:
             export AWS_ACCESS_KEY_ID=$STAGING_AWS_ACCESS_KEY_ID
             export AWS_SECRET_ACCESS_KEY=$STAGING_AWS_SECRET_ACCESS_KEY
             export S3_BUCKET=kurl-sh-staging
+            source bin/kurl-util-image-export.sh
             bin/upload-dist.sh
 
   upload_production_packages:
@@ -58,6 +80,7 @@ jobs:
             export AWS_ACCESS_KEY_ID=$PRODUCTION_AWS_ACCESS_KEY_ID
             export AWS_SECRET_ACCESS_KEY=$PRODUCTION_AWS_SECRET_ACCESS_KEY
             export S3_BUCKET=kurl-sh
+            source bin/kurl-util-image-export.sh
             bin/upload-dist.sh
 
   build_staging_docker_image:
@@ -71,7 +94,8 @@ jobs:
       - run: |
           export WORKDIR=`pwd`
           export CIRCLE_PROJECT_REPONAME=$PROJECT_NAME
-          build-docker
+          source bin/kurl-util-image-export.sh
+          bin/build-docker.sh
       - deploy:
           name: push image
           command: |
@@ -94,7 +118,8 @@ jobs:
       - run: |
           export WORKDIR=`pwd`
           export CIRCLE_PROJECT_REPONAME=$PROJECT_NAME
-          build-docker
+          source bin/kurl-util-image-export.sh
+          bin/build-docker.sh
       - deploy:
           name: push image
           command: |
@@ -156,12 +181,15 @@ workflows:
                 - master
                 - beta
 
-      - upload_staging_packages:
+      - kurl_util_image:
           requires:
             - is_upstream
+      - upload_staging_packages:
+          requires:
+            - kurl_util_image
       - build_staging_docker_image:
           requires:
-            - is_upstream
+            - kurl_util_image
       - deploy_staging_eks:
           requires:
             - build_staging_docker_image
@@ -171,33 +199,26 @@ workflows:
     jobs:
       - is_upstream:
           filters:
-            tags:
-              only: /^v.*/
-            branches:
-              ignore: /.*/
+            <<: *tag_filters
 
-      - build_production_docker_image:
+      - kurl_util_image:
           filters:
-            tags:
-              only: /^v.*/
-            branches:
-              ignore: /.*/
+            <<: *tag_filters
           requires:
             - is_upstream
+      - build_production_docker_image:
+          filters:
+            <<: *tag_filters
+          requires:
+            - kurl_util_image
       - upload_production_packages:
           filters:
-            tags:
-              only: /^v.*/
-            branches:
-              ignore: /.*/
+            <<: *tag_filters
           requires:
-            - is_upstream
+            - kurl_util_image
       - deploy_production_eks:
           filters:
-            tags:
-              only: /^v.*/
-            branches:
-              ignore: /.*/
+            <<: *tag_filters
           requires:
             - upload_production_packages
             - build_production_docker_image
diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-KURL_UTIL_IMAGE := replicated/kurl-util:v2020.02.11-0
+KURL_UTIL_IMAGE ?= replicated/kurl-util:alpha
 
 GIT_TREE = $(shell git rev-parse --is-inside-work-tree 2>/dev/null)
 ifneq "$(GIT_TREE)" ""
@@ -152,6 +152,7 @@ build/templates/install.tmpl: build/install.sh
 		sed 's/^KOTSADM_VERSION=.*/KOTSADM_VERSION="{{= KOTSADM_VERSION }}"/' | \
 		sed 's/^KOTSADM_APPLICATION_SLUG=.*/KOTSADM_APPLICATION_SLUG="{{= KOTSADM_APPLICATION_SLUG }}"/' | \
 		sed 's/^INSTALLER_YAML=.*/INSTALLER_YAML="{{= INSTALLER_YAML }}"/' | \
+		sed 's/^KURL_UTIL_IMAGE=.*/KURL_UTIL_IMAGE="{{= KURL_UTIL_IMAGE }}"/' | \
 		sed 's/^FLAGS=.*/FLAGS="{{= FLAGS }}"/' \
 		> build/templates/install.tmpl
 
@@ -184,6 +185,7 @@ build/templates/join.tmpl: build/join.sh
 		sed 's/^KOTSADM_VERSION=.*/KOTSADM_VERSION="{{= KOTSADM_VERSION }}"/' | \
 		sed 's/^KOTSADM_APPLICATION_SLUG=.*/KOTSADM_APPLICATION_SLUG="{{= KOTSADM_APPLICATION_SLUG }}"/' | \
 		sed 's/^INSTALLER_YAML=.*/INSTALLER_YAML="{{= INSTALLER_YAML }}"/' | \
+		sed 's/^KURL_UTIL_IMAGE=.*/KURL_UTIL_IMAGE="{{= KURL_UTIL_IMAGE }}"/' | \
 		sed 's/^FLAGS=.*/FLAGS="{{= FLAGS }}"/' \
 		> build/templates/join.tmpl
 
@@ -216,6 +218,7 @@ build/templates/upgrade.tmpl: build/upgrade.sh
 		sed 's/^KOTSADM_VERSION=.*/KOTSADM_VERSION="{{= KOTSADM_VERSION }}"/' | \
 		sed 's/^KOTSADM_APPLICATION_SLUG=.*/KOTSADM_APPLICATION_SLUG="{{= KOTSADM_APPLICATION_SLUG }}"/' | \
 		sed 's/^INSTALLER_YAML=.*/INSTALLER_YAML="{{= INSTALLER_YAML }}"/' | \
+		sed 's/^KURL_UTIL_IMAGE=.*/KURL_UTIL_IMAGE="{{= KURL_UTIL_IMAGE }}"/' | \
 		sed 's/^FLAGS=.*/FLAGS="{{= FLAGS }}"/' \
 		> build/templates/upgrade.tmpl
 

diff --git a/addons/kotsadm/alpha/Manifest b/addons/kotsadm/alpha/Manifest
@@ -5,6 +5,5 @@ image kotsadm docker.io/kotsadm/kotsadm:alpha
 image kurl-proxy docker.io/kotsadm/kurl-proxy:alpha
 image postgres postgres:10.7
 image bcrypt docker.io/epicsoft/bcrypt:latest
-image kurl-util docker.io/replicated/kurl-util:v2020.02.11-0
 
 asset kots.tar.gz https://github.com/replicatedhq/kots/releases/download/v1.12.2/kots_linux_amd64.tar.gz
diff --git a/addons/kotsadm/alpha/api.yaml b/addons/kotsadm/alpha/api.yaml
@@ -243,8 +243,6 @@ spec:
           value: "0"
         - name: ENABLE_KURL
           value: "1"
-        - name: KURL_UTIL_IMAGE
-          value: replicated/kurl-util:v2020.02.11-0
         volumeMounts:
         - name: etcd-client-cert
           readOnly: true

diff --git a/bin/build-docker.sh b/bin/build-docker.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+function require() {
+    if [ -z "$2" ]; then
+      echo "validation failed: $1 unset"
+      exit 1
+    fi
+}
+
+require WORKDIR "${WORKDIR}"
+require DOCKERFILE "${DOCKERFILE}"
+
+set -veuo pipefail
+
+docker build -f ${DOCKERFILE} \
+ --build-arg version=${CIRCLE_SHA1:0:7} \
+ --build-arg KURL_UTIL_IMAGE \
+ -t ${CIRCLE_PROJECT_REPONAME}:${CIRCLE_SHA1:0:7} \
+ ${WORKDIR:-$HOME/repo}
diff --git a/bin/kurl-util-image-export.sh b/bin/kurl-util-image-export.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Generate and export the kurl util image reference.
+
+set -eo pipefail
+
+function image_export() {
+    local prev_setu="${-//[^u]/}"
+
+    set +u
+
+    local tag=alpha
+    if [ -n "$CIRCLE_TAG" ]; then
+        tag="$CIRCLE_TAG"
+    elif [ "$CIRCLE_BRANCH" = "beta" ]; then
+        tag="beta"
+    fi
+
+    export KURL_UTIL_IMAGE=replicated/kurl-util:$tag
+
+    if [ -n "$prev_setu" ]; then set -u; fi
+}
+
+image_export
diff --git a/deploy/Dockerfile-slim b/deploy/Dockerfile-slim
@@ -7,6 +7,9 @@ RUN DEBIAN_FRONTEND=noninteractive apt-get update -qq \
 ARG version=unknown
 ENV VERSION=$version
 
+ARG KURL_UTIL_IMAGE
+ENV KURL_UTIL_IMAGE=$KURL_UTIL_IMAGE
+
 ADD web /src
 ADD build/templates /templates
 ADD build/bin/server /bin

diff --git a/kurl_util/.gitignore b/kurl_util/.gitignore
@@ -0,0 +1,3 @@
+/bin/
+*DS_Store
+/vendor/
diff --git a/kurl_util/Makefile b/kurl_util/Makefile
@@ -0,0 +1,58 @@
+SHELL := /bin/bash
+KURL_UTIL_IMAGE ?= replicated/kurl-util:alpha
+
+export GO111MODULE=on
+
+GIT_TREE = $(shell git rev-parse --is-inside-work-tree 2>/dev/null)
+ifneq "$(GIT_TREE)" ""
+define GIT_UPDATE_INDEX_CMD
+git update-index --assume-unchanged
+endef
+define GIT_SHA
+`git rev-parse HEAD`
+endef
+else
+define GIT_UPDATE_INDEX_CMD
+echo "Not a git repo, skipping git update-index"
+endef
+define GIT_SHA
+""
+endef
+endif
+
+.PHONY: clean
+clean:
+	rm -rf ./bin
+
+.PHONY: deps
+deps:
+	go get golang.org/x/lint/golint
+
+.PHONY: lint
+lint:
+	golint -set_exit_status ./...
+
+.PHONY: vet
+vet:
+	go vet ./...
+
+.PHONY: test
+test: lint vet
+	go test ./cmd/...
+
+.PHONY: build
+build: bin/yamlutil bin/subnet
+
+bin/yamlutil:
+	go build -o bin/yamlutil cmd/yamlutil/main.go
+
+bin/subnet:
+	go build -o bin/subnet cmd/subnet/main.go
+
+.PHONY: kurl-util-image
+kurl-util-image:
+	docker build -t $(KURL_UTIL_IMAGE) -f deploy/Dockerfile --build-arg commit="${GIT_SHA}" .
+
+.PHONY: build-and-push-kurl-util-image
+build-and-push-kurl-util-image: kurl-util-image
+	docker push $(KURL_UTIL_IMAGE)
diff --git a/kurl_util/README.md b/kurl_util/README.md
@@ -0,0 +1,7 @@
+# kURL Utility Image
+
+Utility image for kURL clusters.
+
+## Test manually
+
+run `make kurl-util-image`.