-
Notifications
You must be signed in to change notification settings - Fork 28
/
esdc-git-update
executable file
·344 lines (291 loc) · 9.99 KB
/
esdc-git-update
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
#!/bin/bash
#
# description: Script performs update of Danube Cloud to version specified as
# input parameter.
#
# input params: $1 - version tag or git commit hash that will be checked out
# $2 - full path to SSL private key (optional)
# $3 - full path to SSL public certificate (optional)
# --verbose - enable verbose output
# --force - continue update even if the HEAD points to the
# requested version (without git checkout)
# --esdc-service-restart - restart Danube Cloud services after
# successful update
#
set -o pipefail
# #############
# Return codes
# #############
ERR_GIT_NOT_FOUND=1
ERR_PARAM_MISSING=2
ERR_NO_MATCH=3
ERR_UP_TO_DATE=4
ERR_FAILED_FETCH=5
ERR_FAILED_CHECKOUT=5
ERR_POST_DEPLOY_FAILED=6
ERR_UPGRADE_FAILED=6
ERR_LOCK_EXISTS=7
ERR_LOCK_FAILED=8
ERR_GIT_CMD=11
# #################
# Global variables
# #################
MAINDIR="$(cd "$(dirname "$0")/.." || exit 64; pwd -P)"
ERIGONES_HOME=${ERIGONES_HOME:-"${MAINDIR}"}
LOG_DIR="${ERIGONES_HOME}/var/log"
LOCK="${ERIGONES_HOME}/var/run/update.lock"
CTLSH="${ERIGONES_HOME}/bin/ctl.sh"
DEPLOY_CMD="${CTLSH} deploy --update"
SELF_CMD="${ERIGONES_HOME}/bin/$(basename "${0}")"
APPLIANCE_UPDATE_CMD="${ERIGONES_HOME}/bin/esdc-appliance-update"
NODE_UPDATE_CMD="${ERIGONES_HOME}/bin/esdc-node-update"
SERVICE_RESTART_CMD="${ERIGONES_HOME}/bin/esdc-service-control restart"
VERBOSE=""
RESTART=""
FORCE=""
VERSION=""
KEY_PATH=""
CERT_PATH=""
# #####################
# Positional arguments
# #####################
for arg in "${@}"; do
case "${arg}" in
"--verbose"|"-v")
VERBOSE="true"
;;
"--esdc-service-restart")
RESTART="true"
;;
"--force")
FORCE="true"
;;
*)
if [[ -z "${VERSION}" ]]; then
VERSION="${arg}"
elif [[ -z "${KEY_PATH}" ]]; then
KEY_PATH="${arg}"
elif [[ -z "${CERT_PATH}" ]]; then
CERT_PATH="${arg}"
else
break
fi
;;
esac
done
# ##########
# Functions
# ##########
function die() {
local exit_code="${1:-1}"
shift
local msg="$*"
[[ -n "${msg}" ]] && echo "ERROR: ${msg}" 1>&2
exit "${exit_code}"
}
function usage() {
echo "Usage: $0 <version> [private SSL key file] [X509 certificate file]"
echo ""
echo "Example 1: $0 v2.4.0"
echo "Example 2: $0 v3.0.0 /path/to/YourDCHost.key /path/to/YourDCHost.crt"
}
function rollback() {
echo "#######################################"
echo "Running emergency rollback"
if [[ -n "${DEPLOY_RUNNING_FILE}" && -f "${DEPLOY_RUNNING_FILE}" ]]; then
# Deploy post command failed -> we cannot rollback + we cannot revert the repository
rm -f "${DEPLOY_RUNNING_FILE}"
# TODO: cleanup not implemented
echo "WARNING: The post deploy command has failed and may have left the repository in an unpleasant state"
echo "WARNING: Manual cleanup is required"
else
# Deploy post command did not run yet
if [[ -n "${CURRENT_VERSION_REF}" && "${CURRENT_VERSION_REF}" != "${VERSION_REF}" ]]; then
echo "Running git checkout to commit before upgrade: ${CURRENT_VERSION_REF}"
git checkout "${CURRENT_VERSION_REF}"
fi
fi
}
# Change working directory to ERIGONES_HOME
cd "${ERIGONES_HOME}" || exit 64
function acquire_lock() {
if [[ -f "${LOCK}" ]]; then
die ${ERR_LOCK_EXISTS} "Locked! Maybe another upgrade is still running."
fi
trap 'rm -f "${LOCK}"' EXIT
if ! echo "$$" > "${LOCK}"; then
die ${ERR_LOCK_FAILED} "Could not acquire lock"
fi
}
if [[ -z "${ESDC_GIT_UPDATE_2ND_RUN}" ]]; then
# Retrieve current Git settings
if ! git --version >/dev/null; then
die ${ERR_GIT_NOT_FOUND} "Git was not found on system!"
fi
if [[ -z "${VERSION}" ]]; then
usage 1>&2
die ${ERR_PARAM_MISSING}
fi
# Git environmental variables for cloning over HTTPS
if [[ -n "${KEY_PATH}" ]]; then
if [[ ! -f "${KEY_PATH}" ]]; then
die ${ERR_PARAM_MISSING} "File does not exist: ${KEY_PATH}"
fi
export GIT_SSL_KEY=${KEY_PATH}
fi
if [[ -n "${CERT_PATH}" ]]; then
if [[ ! -f "${CERT_PATH}" ]]; then
die ${ERR_PARAM_MISSING} "File does not exist: ${CERT_PATH}"
fi
export GIT_SSL_CERT=${CERT_PATH}
fi
export GIT_SSL_NO_VERIFY="false"
# Create lock file
acquire_lock
CURRENT_VERSION_REF=$(git rev-parse HEAD || die ${ERR_GIT_CMD})
if [[ -n "${VERBOSE}" ]]; then
# Gather information about current status of esdc git repository
UPDATE_URL=$(git config --list | grep "remote.origin.url" | awk -F '=' '{print $2}')
echo "#######################################"
echo "Remote git repository URL: ${UPDATE_URL}"
echo "Current HEAD points to commit: ${CURRENT_VERSION_REF}"
echo "SSL key file: ${KEY_PATH}"
echo "SSL certificate file: ${CERT_PATH}"
echo "Target version: ${VERSION}"
echo "#######################################"
fi
if [[ "${VERSION}" == "_skip_checkout" ]]; then
echo "WARNING: Skipping git fetch and git checkout"
VERSION_REF="$(git rev-parse HEAD || die ${ERR_GIT_CMD})"
else
echo "Running git fetch origin"
# fetch most recent changes first
# MERGE will be done only if requested VERSION is among fetched objects
git fetch origin > "${LOG_DIR}/update_git_fetch.log" 2>&1 || \
die ${ERR_FAILED_FETCH} "Unable to fetch changes from git remote ($(cat "${LOG_DIR}/update_git_fetch.log"))"
AVAILABLE_TAGS=$(git show-ref --tags)
# When VERSION matches one of the tags name in the list of reference-tags
# assign the commit reference to VERSION_REF variable
# else assume that VERSION holds value of the commit reference
if echo "${AVAILABLE_TAGS}" | awk '{print $NF}' | grep "refs/tags/${VERSION}$" >/dev/null; then
# Follow the tag to the target commit hash
VERSION_REF=$(git rev-list -n1 "${VERSION}" || die ${ERR_GIT_CMD})
else
if git rev-list --all | grep "^${VERSION}$" >/dev/null; then
VERSION_REF="${VERSION}"
else
ERR_MSG="Could not match '${VERSION}' you requested with any of the commits. Aborting upgrade."
die ${ERR_NO_MATCH} "${ERR_MSG}"
fi
fi
# if we are already at the requested revision exit
if [[ "${VERSION_REF}" == "$(git rev-parse HEAD)" ]]; then
ERR_MSG="Already at requested revision: ${VERSION} with hash: ${VERSION_REF}"
if [[ -n "${FORCE}" ]]; then
echo "${ERR_MSG}"
echo "WARNING: Skipping git checkout"
else
die ${ERR_UP_TO_DATE} "${ERR_MSG}"
fi
fi
echo "Running git checkout to commit: ${VERSION_REF}"
git checkout "${VERSION_REF}" > "${LOG_DIR}/update_git_checkout.log" 2>&1 || \
die ${ERR_FAILED_CHECKOUT} "Failed to checkout commit: ${VERSION_REF} ($(cat "${LOG_DIR}/update_git_checkout.log"))"
fi
# Set and truncate the log file
UPDATE_LOG="${LOG_DIR}/update.${VERSION_REF}.log"
echo "#######################################" | tee "${UPDATE_LOG}"
echo "Repository is on commit: ${VERSION_REF}" | tee -a "${UPDATE_LOG}"
echo "Starting 2nd run of ${SELF_CMD} ${*}" | tee -a "${UPDATE_LOG}"
echo "#######################################" | tee -a "${UPDATE_LOG}"
export CURRENT_VERSION_REF
export VERSION_REF
# Setup rollback
export DEPLOY_RUNNING_FILE="/tmp/.deploy.${VERSION_REF}.running"
rm -f "${DEPLOY_RUNNING_FILE}" 2> /dev/null
trap rollback ERR
# Call our NEW self!
ESDC_GIT_UPDATE_2ND_RUN="true" "${SELF_CMD}" "${@}" 2>&1 | tee -a "${UPDATE_LOG}"
exit $?
fi
#
# 2nd run starts
#
# Upgrade from an pre-3.x version
if [[ -z "${VERSION_REF}" && -z "${CURRENT_VERSION_REF}" ]]; then
echo "WARNING: Detected upgrade from a pre-3.x version => no support for rollback"
VERSION_REF="$(git rev-parse HEAD || die ${ERR_GIT_CMD})"
CURRENT_VERSION_REF=""
DEPLOY_RUNNING_FILE="/tmp/.deploy.${VERSION_REF}.running"
fi
# Double-check current commit
# (we need to to follow the old commit because it can point to a tag)
if [[ "$(git rev-list -n1 "${VERSION_REF}" || die ${ERR_GIT_CMD})" != "$(git rev-parse HEAD || die ${ERR_GIT_CMD})" ]]; then
die ${ERR_GIT_CMD} "Git commit changed during update"
fi
# In case we've upgraded from a version without locking
# shellcheck disable=SC2244
[[ ! "${LOCK}" ]] && acquire_lock
# Deactivate our Git SSL configuration, because the upcoming commands may use git
unset GIT_SSL_KEY
unset GIT_SSL_CERT
unset GIT_SSL_NO_VERIFY
# Run db migrations to make Django work again (sync newly pulled code and DB)
"${ERIGONES_HOME}"/bin/ctl.sh db_sync --force
#
# Appliance/node upgrade
#
if [[ "$(uname -s)" == "SunOS" ]] && [[ -f /usr/bin/zonename ]] && [[ "$(/usr/bin/zonename)" == "global" ]]; then
DEPLOY_CMD="${DEPLOY_CMD} --node"
# run node upgrade
if [[ -x "${NODE_UPDATE_CMD}" ]]; then
echo "Upgrading compute node software (please wait) ..."
node_update_log="${LOG_DIR}/update_node.${VERSION_REF}.log"
if "${NODE_UPDATE_CMD}" > "${node_update_log}" 2>&1; then
echo "Compute node upgrade was successful"
else
cat "${node_update_log}" >&2
die ${ERR_UPGRADE_FAILED} "Compute node upgrade failed"
fi
fi
else
# run appliance upgrade
if [[ -x "${APPLIANCE_UPDATE_CMD}" ]]; then
echo "Upgrading appliances (please wait) ..."
appliance_update_log="${LOG_DIR}/update_vms.${VERSION_REF}.log"
if "${APPLIANCE_UPDATE_CMD}" > "${appliance_update_log}" 2>&1; then
echo "Appliance upgrade was successful"
else
cat "${appliance_update_log}" >&2
die ${ERR_UPGRADE_FAILED} "Appliance upgrade failed"
fi
fi
fi
#
# Post deploy
#
echo "#######################################"
echo "Running post deploy command: ${DEPLOY_CMD}"
# This will enable rollback
echo "$$" > "${DEPLOY_RUNNING_FILE}"
# Running ctl.sh deploy --update
deploy_log="${LOG_DIR}/update_deploy.${VERSION_REF}.log"
if ! ${DEPLOY_CMD} > "${deploy_log}" 2>&1; then
cat "${deploy_log}" >&2
die ${ERR_POST_DEPLOY_FAILED} "Post deploy command failed"
fi
# This will disable rollback
rm -f "${DEPLOY_RUNNING_FILE}"
echo "Post deploy command was successful"
echo "#######################################"
#
# Service restart
#
if [[ -n "${RESTART}" ]]; then
echo "Going to restart all Danube Cloud system services in 10 seconds..."
nohup bash -c "sleep 10 && ${SERVICE_RESTART_CMD}" > /dev/null 2> /dev/null &
else
echo "You should now restart all Danube Cloud system services"
echo "(${SERVICE_RESTART_CMD})"
fi
exit 0