Merge pull request #28 from spotahome/devops-694-use-kooper

[DEVOPS-694] Use Kooper

Gopkg.toml

@@ -1,30 +1,34 @@
 [[constraint]]
   name = "github.com/go-redis/redis"
-  version = "6.5.6"
+  version = "6.8.3"
 
 [[constraint]]
   name = "github.com/sirupsen/logrus"
+  version = "1.0.4"
 
 [[constraint]]
   name = "github.com/stretchr/testify"
+  version = "v1.2.1"
 
 [[constraint]]
+  name = "github.com/spotahome/kooper"
+  version = "v0.1.0"
+
+[[override]]
   name = "k8s.io/apimachinery"
-  branch = "release-1.7"
+  version = "kubernetes-1.9.1"
 
-[[constraint]]
+[[override]]
   name = "k8s.io/client-go"
-  version = "4.0.0"
+  version = "6.0.0"
 
-[[constraint]]
+[[override]]
   name = "k8s.io/apiextensions-apiserver"
-  branch = "release-1.7"
+  version = "kubernetes-1.9.1"
 
-# This is due to an issue with k8s.io/client-go and dep not
-# being able to lock transient dependencies
 [[override]]
-  name = "github.com/ugorji/go"
-  revision = "8c0409fcbb70099c748d71f714529204975f6c3f"
+  name = "k8s.io/api"
+  version = "kubernetes-1.9.1"
 
 [[constraint]]
   name = "github.com/prometheus/client_golang"

Makefile

@@ -1,6 +1,4 @@
-# The following are targers that do not exist in the filesystem as real files and should be always executed by make
-.PHONY: default build deps-development docker-build shell run image unit-test test generate go-generate get-deps update-deps testing
-VERSION := 0.1.6
+VERSION := 0.2.0
 
 # Name of this service/application
 SERVICE_NAME := redis-operator
@@ -17,7 +15,10 @@ SHELL := $(shell which bash)
 # Get docker path or an empty string
 DOCKER := $(shell command -v docker)
 
-# Get user ID
+# Get the main unix group for the user running make (to be used by docker-compose later)
+GID := $(shell id -g)
+
+# Get the unix user id for the user running make (to be used by docker-compose later)
 UID := $(shell id -u)
 
 # Commit hash from git
@@ -33,6 +34,8 @@ UNIT_TEST_CMD := go test `go list ./... | grep -v /vendor/` -v
 GO_GENERATE_CMD := go generate `go list ./... | grep -v /vendor/`
 GET_DEPS_CMD := dep ensure
 UPDATE_DEPS_CMD := dep ensure
+UPDATE_CODEGEN_CMD := ./hack/update-codegen.sh
+MOCKS_CMD := go generate ./mocks
 
 # environment dirs
 DEV_DIR := docker/development
@@ -42,16 +45,11 @@ APP_DIR := docker/app
 WORKDIR := /go/src/github.com/spotahome/redis-operator
 
 # The default action of this Makefile is to build the development docker image
+.PHONY: default
 default: build
 
-# Test if the dependencies we need to run this Makefile are installed
-deps-development:
-ifndef DOCKER
-	@echo "Docker is not available. Please install docker"
-	@exit 1
-endif
-
 # Run the development environment in non-daemonized mode (foreground)
+.PHONY: docker-build
 docker-build: deps-development
 	docker build \
 		--build-arg UID=$(UID) \
@@ -61,18 +59,22 @@ docker-build: deps-development
 		.
 
 # Run a shell into the development docker image
+.PHONY: shell
 shell: docker-build
 	docker run -ti --rm -v ~/.kube:/.kube:ro -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) -p $(PORT):$(PORT) $(REPOSITORY)-dev /bin/bash
 
 # Build redis-failover executable file
+.PHONY: build
 build: docker-build
 	docker run -ti --rm -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) $(REPOSITORY)-dev ./scripts/build.sh
 
 # Run the development environment in the background
+.PHONY: run
 run: docker-build
 	docker run -ti --rm -v ~/.kube:/.kube:ro -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) -p $(PORT):$(PORT) $(REPOSITORY)-dev ./scripts/run.sh
 
 # Build the production image based on the public one
+.PHONY: image
 image: deps-development
 	docker build \
 	-t $(SERVICE_NAME) \
@@ -82,35 +84,61 @@ image: deps-development
 	-f $(APP_DIR)/Dockerfile \
 	.
 
+.PHONY: testing
 testing: image
 	docker push $(REPOSITORY):$(BRANCH)
 
+.PHONY: tag
 tag:
 	git tag $(VERSION)
 
+.PHONY: publish
 publish:
 	@COMMIT_VERSION="$$(git rev-list -n 1 $(VERSION))"; \
 	docker tag $(REPOSITORY):"$$COMMIT_VERSION" $(REPOSITORY):$(VERSION)
 	docker push $(REPOSITORY):$(VERSION)
 	docker push $(REPOSITORY):latest
 
+.PHONY: release
 release: tag image publish
 
 # Test stuff in dev
+.PHONY: unit-test
 unit-test: docker-build
 	docker run -ti --rm -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) $(REPOSITORY)-dev /bin/sh -c '$(UNIT_TEST_CMD)'
+
+.PHONY: test
 test: unit-test
 
+.PHONY: go-generate
 go-generate: docker-build
 	docker run -ti --rm -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) $(REPOSITORY)-dev /bin/sh -c '$(GO_GENERATE_CMD)'
 
+.PHONY: generate
 generate: go-generate
 
+.PHONY: get-deps
 get-deps: docker-build
 	docker run -ti --rm -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) $(REPOSITORY)-dev /bin/sh -c '$(GET_DEPS_CMD)'
 
+.PHONY: update-deps
 update-deps: docker-build
 	docker run -ti --rm -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) $(REPOSITORY)-dev /bin/sh -c '$(UPDATE_DEPS_CMD)'
 
-# Custom commands
-#...
+.PHONY: mocks
+mocks: docker-build
+	docker run -ti --rm -v $(PWD):$(WORKDIR) -u $(UID):$(UID) --name $(SERVICE_NAME) $(REPOSITORY)-dev /bin/sh -c '$(MOCKS_CMD)'
+
+.PHONY: deps-development
+# Test if the dependencies we need to run this Makefile are installed
+deps-development:
+ifndef DOCKER
+	@echo "Docker is not available. Please install docker"
+	@exit 1
+endif
+
+# Generate kubernetes code for types..
+.PHONY: update-codege
+update-codegen: build
+	@echo ">> Generating code for Kubernetes CRD types..."
+	docker run --rm -v $(PWD):/go/src/github.com/spotahome/redis-operator/ $(REPOSITORY)-dev /bin/bash -c '$(UPDATE_CODEGEN_CMD)'

README.md

@@ -1,5 +1,5 @@
 # redis-operator [![Build Status](https://travis-ci.org/spotahome/redis-operator.png)](https://travis-ci.org/spotahome/redis-operator) [![Go Report Card](http://goreportcard.com/badge/spotahome/redis-operator)](http://goreportcard.com/report/spotahome/redis-operator)
-Redis Operator creates/configures/manages redis clusters atop Kubernetes.
+Redis Operator creates/configures/manages redis failovers atop Kubernetes.
 
 ## Requirements
 Redis Operator is meant to be run on Kubernetes 1.8+.
@@ -9,9 +9,6 @@ All dependecies have been vendored, so there's no need to any additional downloa
 #### Redis Operator
 [![Redis Operator Image](https://quay.io/repository/spotahome/redis-operator/status "Redis Operator Image")](https://quay.io/repository/spotahome/redis-operator)
 
-#### Redis Operator Toolkit
-[![Redis Operator Toolkit Image](https://quay.io/repository/spotahome/redis-operator-toolkit/status "Redis Operator Toolkit Image")](https://quay.io/repository/spotahome/redis-operator-toolkit)
-
 ## Operator deployment on kubernetes
 In order to create Redis failovers inside a Kubernetes cluster, the operator has to be deployed:
 ~~~~
@@ -34,7 +31,7 @@ spec:
         app: redisoperator
     spec:
       containers:
-      - image: quay.io/spotahome/redis-operator:0.1.0
+      - image: quay.io/spotahome/redis-operator:0.2.0
         imagePullPolicy: IfNotPresent
         name: app
         resources:
@@ -75,34 +72,38 @@ spec:
 ~~~~
 
 ## Creation pipeline
-The redis-operator creates a redis failover, using the following pipeline:
-
-1. Start a redis bootstrap pod, containing a redis as a master allowing other redis/sentinel to connect to it.
-2. Create a sentinel service to allow service discovery.
-3. Start a sentinel deployment with the number of replicas set by the user definition (or 3 by default). This replicas monitor the bootstrap master.
-4. Create a pod disruption budget for sentinel pods.
-5. If the redis exporter is active, create a headless redis service for discovery.
-6. Start a redis statefulset, who connects to the redis bootstrap pod as a slaves.
-7. Create a pod disruption budget for redis pods.
-8. Delete the redis bootstrap pod.
-9. From this moment, the redis-failover will check the cluster is ok. If not, it will try to fix it when possible. Normal failures will be controlled by sentinel.
+The redis-operator creates a redis failover, with all the needed pieces. It does this in two separated steps:
+* Ensure: checks that all the pieces needed are created.
+    * Redis statefulset
+    * Sentinel Deployment
+    * Sentinel service
+    * Redis service (if exporter enabled)
+* Check & Heal: checks that the failover is working and configured as spected.
+    * Number of redis is equal as the set on the RF spec
+    * Number of sentinel is equal as the set on the RF spec
+    * Only one redis working as a master
+    * All redis slaves have the same master
+    * All redis slaves are connected to the master
+    * All sentinels points to the same redis master
+    * Sentinel has not death nodes
+    * Sentinel knows the correct slave number
 
 ## Code folder structure
+* api: definition of the RedisFailover CRD.
+* client: autogenerated client to interact with redis-failovers.
 * cmd: contains the starting point of the application.
+* log: wrapper of logrus, created to be able to mock it.
+* metrics: exposer of status of the failovers created.
 * mocks: contains the mocked interfaces for testing the application.
-* pkg:
-  * clock: wrapper of time, created to be able to mock it.
-  * config: contains the constants of the application.
-  * failover: contains the logic of the application.
-  * log: wrapper of logrus, created to be able to mock it.
-  * redis: interface wich allows separate the library used and the logic.
-  * tpr: created to define the third party resource that will be registered into k8s.
+* operator: the main logic. Manages the requests from k8s and creates/updates/deletes the pieces as needed.
+* service: services/clients to interact with k8s and redises.
 * vendor: vendored packages used by the application.
 
 ## Non-code folder structure
 * charts: helm chart to deploy the TPR.
 * docker: Dockerfiles to generate redis-failover docker images.
 * example: yaml files with spec of redis-failover.
+* hack: scripts to generate the client.
 * scripts: scripts used to build and run the app.
 
 ## Development
@@ -112,6 +113,8 @@ You can do the following commands with make:
 `make docker-build`
 * Generate mocks.
 `make go-generate`
+* Generate client
+`make update-codegen`
 * Run tests.
 `make test`
 * Build the executable file.

api/redisfailover/register.go

@@ -0,0 +1,5 @@
+package redisfailover
+
+const (
+	GroupName = "storage.spotahome.com"
+)

api/redisfailover/v1alpha2/doc.go

@@ -0,0 +1,5 @@
+// +k8s:deepcopy-gen=package
+
+// Package v1alpha2 is the v1alpha2 version of the API.
+// +groupName=storage.spotahome.com
+package v1alpha2

api/redisfailover/v1alpha2/register.go

@@ -0,0 +1,55 @@
+package v1alpha2
+
+import (
+	"github.com/spotahome/redis-operator/api/redisfailover"
+
+	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+const (
+	version = "v1alpha2"
+)
+
+// Team constants
+const (
+	RFKind       = "RedisFailover"
+	RFName       = "redisfailover"
+	RFNamePlural = "redisfailovers"
+	RFScope      = apiextensionsv1beta1.NamespaceScoped
+)
+
+// SchemeGroupVersion is group version used to register these objects
+var SchemeGroupVersion = schema.GroupVersion{Group: redisfailover.GroupName, Version: version}
+
+// Kind takes an unqualified kind and returns back a Group qualified GroupKind
+func Kind(kind string) schema.GroupKind {
+	return VersionKind(kind).GroupKind()
+}
+
+// VersionKind takes an unqualified kind and returns back a Group qualified GroupVersionKind
+func VersionKind(kind string) schema.GroupVersionKind {
+	return SchemeGroupVersion.WithKind(kind)
+}
+
+// Resource takes an unqualified resource and returns a Group qualified GroupResource
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}
+
+var (
+	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
+	AddToScheme   = SchemeBuilder.AddToScheme
+)
+
+// Adds the list of known types to Scheme.
+func addKnownTypes(scheme *runtime.Scheme) error {
+	scheme.AddKnownTypes(SchemeGroupVersion,
+		&RedisFailover{},
+		&RedisFailoverList{},
+	)
+	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)
+	return nil
+}