Add QCOW2 build mechanism (RPi-Distro#349)

Dockerfile

@@ -8,7 +8,7 @@ RUN apt-get -y update && \
         git vim parted \
         quilt coreutils qemu-user-static debootstrap zerofree zip dosfstools \
         bsdtar libcap2-bin rsync grep udev xz-utils curl xxd file kmod bc\
-        binfmt-support ca-certificates \
+        binfmt-support ca-certificates qemu-utils kpartx \
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

README.md

@@ -14,7 +14,8 @@ To install the required dependencies for `pi-gen` you should run:
 
 ```bash
 apt-get install coreutils quilt parted qemu-user-static debootstrap zerofree zip \
-dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc
+dosfstools bsdtar libcap2-bin grep rsync xz-utils file git curl bc \
+qemu-utils kpartx
 ```
 
 The file `depends` contains a list of tools needed.  The format of this
@@ -36,7 +37,30 @@ The following environment variables are supported:
    but you should use something else for a customized version.  Export files
    in stages may add suffixes to `IMG_NAME`.
 
- * `RELEASE` (Default: buster)
+* `USE_QCOW2`(Default: `1` )
+
+    Instead of using traditional way of building the rootfs of every stage in
+    single subdirectories and copying over the previous one to the next one,
+    qcow2 based virtual disks with backing images are used in every stage.
+    This speeds up the build process and reduces overall space consumption
+    significantly.
+
+    <u>Additional optional parameters regarding qcow2 build:</u>
+
+    * `BASE_QCOW2_SIZE` (Default: 12G)
+
+        Size of the virtual qcow2 disk.
+        Note: it will not actually use that much of space at once but defines the
+        maximum size of the virtual disk. If you change the build process by adding
+        a lot of bigger packages or additional build stages, it can be necessary to
+        increase the value because the virtual disk can run out of space like a normal
+        hard drive would.
+
+    **CAUTION:**  Although the qcow2 build mechanism will run fine inside Docker, it can happen
+    that the network block device is not disconnected correctly after the Docker process has
+    ended abnormally. In that case see [Disconnect an image if something went wrong](#Disconnect-an-image-if-something-went-wrong)
+
+* `RELEASE` (Default: buster)
 
    The release version to build images against. Valid values are jessie, stretch
    buster, bullseye, and testing.
@@ -340,6 +364,71 @@ follows:
  * Once you're happy with the image you can remove the SKIP_IMAGES files and
    export your image to test
 
+# Regarding Qcow2 image building
+
+### Get infos about the image in use
+
+If you issue the two commands shown in the example below in a second command shell while a build
+is running you can find out, which network block device is currently being used and which qcow2 image
+is bound to it.
+
+Example:
+
+```bash
+root@build-machine:~/$ lsblk | grep nbd
+nbd1      43:32   0    10G  0 disk 
+├─nbd1p1  43:33   0    10G  0 part 
+└─nbd1p1 253:0    0    10G  0 part
+
+root@build-machine:~/$ ps xa | grep qemu-nbd
+ 2392 pts/6    S+     0:00 grep --color=auto qemu-nbd
+31294 ?        Ssl    0:12 qemu-nbd --discard=unmap -c /dev/nbd1 image-stage4.qcow2
+```
+
+Here you can see, that the qcow2 image `image-stage4.qcow2` is currently connected to `/dev/nbd1` with
+the associated partition map `/dev/mapper/nbd1p1`. Don't worry that `lsblk` shows two entries. It is totally fine, because the device map is accessible via `/dev/mapper/nbd1p1` and also via `/dev/dm-0`. This is all part of the device mapper functionality of the kernel. See `dmsetup` for further information.
+
+### Mount a qcow2 image
+
+If you want to examine the content of a a single stage, you can simply mount the qcow2 image found in the `WORK_DIR` directory with the tool `./imagetool.sh`.
+
+See `./imagetool.sh -h` for further details on how to use it.
+
+### Disconnect an image if something went wrong
+
+It can happen, that your build stops in case of an error. Normally `./build.sh` should handle image disconnection appropriately, but in rare cases, especially during a Docker build, this may not work as expected. If that happens, starting a new build will fail and you may have to disconnect the image and/or device yourself.
+
+A typical message indicating that there are some orphaned device mapper entries is this:
+
+```
+Failed to set NBD socket 
+Disconnect client, due to: Unexpected end-of-file before all bytes were read
+```
+
+If that happens go through the following steps:
+
+1. First, check if the image is somehow mounted to a directory entry and umount it as you would any other block device, like i.e. a hard disk or USB stick.
+
+2. Second, to disconnect an image from `qemu-nbd`, the QEMU Disk Network Block Device Server, issue the following command (be sure to change the device name to the one actually used):
+
+   ```bash
+   sudo qemu-nbd -d /dev/nbd1
+   ```
+
+   Note: if you use Docker build, normally no active `qemu-nbd` process exists anymore as it will be terminated when the Docker container stops.
+
+3. To disconnect a device partition map from the network block device, execute:
+
+   ```bash
+   sudo kpartx -d /dev/nbd1
+   or
+   sudo ./imagetool.sh --cleanup
+   ```
+
+   Note: The `imagetool.sh` command will cleanup any /dev/nbdX that is not connected to a running `qemu-nbd` daemon. Be careful if you use network block devices for other tasks utilizing NBDs on your build machine as well.
+
+Now you should be able to start a new build without running into troubles again. Most of the time, especially when using Docker build, you will only need no. 3 to get everything up and running again. 
+
 # Troubleshooting
 
 ## `64 Bit Systems`

build-docker.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -eu
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
 BUILD_OPTS="$*"
@@ -87,6 +88,9 @@ ${DOCKER} build --build-arg BASE_IMAGE=${BASE_IMAGE} -t pi-gen "${DIR}"
 if [ "${CONTAINER_EXISTS}" != "" ]; then
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}_cont' SIGINT SIGTERM
 	time ${DOCKER} run --rm --privileged \
+		--cap-add=ALL \
+		-v /dev:/dev \
+		-v /lib/modules:/lib/modules \
 		--volume "${CONFIG_FILE}":/config:ro \
 		-e "GIT_HASH=${GIT_HASH}" \
 		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
@@ -98,6 +102,9 @@ if [ "${CONTAINER_EXISTS}" != "" ]; then
 else
 	trap 'echo "got CTRL+C... please wait 5s" && ${DOCKER} stop -t 5 ${CONTAINER_NAME}' SIGINT SIGTERM
 	time ${DOCKER} run --name "${CONTAINER_NAME}" --privileged \
+		--cap-add=ALL \
+		-v /dev:/dev \
+		-v /lib/modules:/lib/modules \
 		--volume "${CONFIG_FILE}":/config:ro \
 		-e "GIT_HASH=${GIT_HASH}" \
 		pi-gen \
@@ -106,6 +113,7 @@ else
 	rsync -av work/*/build.log deploy/" &
 	wait "$!"
 fi
+
 echo "copying results from deploy/"
 ${DOCKER} cp "${CONTAINER_NAME}":/pi-gen/deploy .
 ls -lah deploy

build.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -e
+
 # shellcheck disable=SC2119
 run_sub_stage()
 {
@@ -13,7 +14,7 @@ $(cat "${i}-debconf")
 SELEOF
 EOF
 
-		log "End ${SUB_STAGE_DIR}/${i}-debconf"
+			log "End ${SUB_STAGE_DIR}/${i}-debconf"
 		fi
 		if [ -f "${i}-packages-nr" ]; then
 			log "Begin ${SUB_STAGE_DIR}/${i}-packages-nr"
@@ -22,6 +23,11 @@ EOF
 				on_chroot << EOF
 apt-get -o APT::Acquire::Retries=3 install --no-install-recommends -y $PACKAGES
 EOF
+				if [ "${USE_QCOW2}" = "1" ]; then
+					on_chroot << EOF
+apt-get clean
+EOF
+				fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages-nr"
 		fi
@@ -32,6 +38,11 @@ EOF
 				on_chroot << EOF
 apt-get -o APT::Acquire::Retries=3 install -y $PACKAGES
 EOF
+				if [ "${USE_QCOW2}" = "1" ]; then
+					on_chroot << EOF
+apt-get clean
+EOF
+				fi
 			fi
 			log "End ${SUB_STAGE_DIR}/${i}-packages"
 		fi
@@ -82,17 +93,30 @@ EOF
 run_stage(){
 	log "Begin ${STAGE_DIR}"
 	STAGE="$(basename "${STAGE_DIR}")"
+
 	pushd "${STAGE_DIR}" > /dev/null
-	unmount "${WORK_DIR}/${STAGE}"
+
 	STAGE_WORK_DIR="${WORK_DIR}/${STAGE}"
 	ROOTFS_DIR="${STAGE_WORK_DIR}"/rootfs
+
+	if [ "${USE_QCOW2}" = "1" ]; then 
+		if [ ! -f SKIP ]; then
+			load_qimage
+		fi
+	else
+		# make sure we are not umounting during export-image stage
+		if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+			unmount "${WORK_DIR}/${STAGE}"
+		fi
+	fi
+
 	if [ ! -f SKIP_IMAGES ]; then
 		if [ -f "${STAGE_DIR}/EXPORT_IMAGE" ]; then
 			EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
 		fi
 	fi
 	if [ ! -f SKIP ]; then
-		if [ "${CLEAN}" = "1" ]; then
+		if [ "${CLEAN}" = "1" ] && [ "${USE_QCOW2}" = "0" ] ; then
 			if [ -d "${ROOTFS_DIR}" ]; then
 				rm -rf "${ROOTFS_DIR}"
 			fi
@@ -103,13 +127,21 @@ run_stage(){
 			log "End ${STAGE_DIR}/prerun.sh"
 		fi
 		for SUB_STAGE_DIR in "${STAGE_DIR}"/*; do
-			if [ -d "${SUB_STAGE_DIR}" ] &&
-			   [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then
+			if [ -d "${SUB_STAGE_DIR}" ] && [ ! -f "${SUB_STAGE_DIR}/SKIP" ]; then
 				run_sub_stage
 			fi
 		done
 	fi
-	unmount "${WORK_DIR}/${STAGE}"
+
+	if [ "${USE_QCOW2}" = "1" ]; then 
+		unload_qimage
+	else
+		# make sure we are not umounting during export-image stage
+		if [ "${USE_QCOW2}" = "0" ] && [ "${NO_PRERUN_QCOW2}" = "0" ]; then
+			unmount "${WORK_DIR}/${STAGE}"
+		fi
+	fi
+
 	PREV_STAGE="${STAGE}"
 	PREV_STAGE_DIR="${STAGE_DIR}"
 	PREV_ROOTFS_DIR="${ROOTFS_DIR}"
@@ -143,6 +175,15 @@ do
 	esac
 done
 
+term() {
+	if [ "${USE_QCOW2}" = "1" ]; then
+		log "Unloading image"
+		unload_qimage
+	fi
+}
+
+trap term EXIT INT TERM
+
 export PI_GEN=${PI_GEN:-pi-gen}
 export PI_GEN_REPO=${PI_GEN_REPO:-https://github.com/RPi-Distro/pi-gen}
 
@@ -211,6 +252,18 @@ source "${SCRIPT_DIR}/common"
 # shellcheck source=scripts/dependencies_check
 source "${SCRIPT_DIR}/dependencies_check"
 
+export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
+export USE_QCOW2="${USE_QCOW2:-1}"
+export BASE_QCOW2_SIZE=${BASE_QCOW2_SIZE:-12G}
+source "${SCRIPT_DIR}/qcow2_handling"
+if [ "${USE_QCOW2}" = "1" ]; then
+	NO_PRERUN_QCOW2=1
+else
+	NO_PRERUN_QCOW2=0
+fi
+
+export NO_PRERUN_QCOW2="${NO_PRERUN_QCOW2:-1}"
+
 dependencies_check "${BASE_DIR}/depends"
 
 #check username is valid
@@ -250,22 +303,98 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	# shellcheck source=/dev/null
 	source "${EXPORT_DIR}/EXPORT_IMAGE"
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename "${EXPORT_DIR}")/rootfs
-	run_stage
+	if [ "${USE_QCOW2}" = "1" ]; then
+		USE_QCOW2=0
+		EXPORT_NAME="${IMG_FILENAME}${IMG_SUFFIX}"
+		echo "------------------------------------------------------------------------"
+		echo "Running export stage for ${EXPORT_NAME}"
+		rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.img" || true
+		rm -f "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" || true
+		rm -f "${WORK_DIR}/${EXPORT_NAME}.img" || true
+		rm -f "${WORK_DIR}/${EXPORT_NAME}.qcow2" || true
+		EXPORT_STAGE=$(basename "${EXPORT_DIR}")
+		for s in $STAGE_LIST; do
+			TMP_LIST=${TMP_LIST:+$TMP_LIST }$(basename "${s}")
+		done
+		FIRST_STAGE=${TMP_LIST%% *}
+		FIRST_IMAGE="image-${FIRST_STAGE}.qcow2"
+
+		pushd "${WORK_DIR}" > /dev/null
+		echo "Creating new base "${EXPORT_NAME}.qcow2" from ${FIRST_IMAGE}"
+		cp "./${FIRST_IMAGE}" "${EXPORT_NAME}.qcow2"
+
+		ARR=($TMP_LIST)
+		# rebase stage images to new export base
+		for CURR_STAGE in "${ARR[@]}"; do
+			if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
+				PREV_IMG="${EXPORT_NAME}"
+				continue
+			fi
+		echo "Rebasing image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
+			qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
+			if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
+				break
+			fi
+			PREV_IMG="image-${CURR_STAGE}"
+		done
+
+		# commit current export stage into base export image
+		echo "Committing image-${EXPORT_STAGE}.qcow2 to ${EXPORT_NAME}.qcow2"
+		qemu-img commit -f qcow2 -p -b "${EXPORT_NAME}.qcow2" image-${EXPORT_STAGE}.qcow2
+
+		# rebase stage images back to original first stage for easy re-run
+		for CURR_STAGE in "${ARR[@]}"; do
+			if [ "${CURR_STAGE}" = "${FIRST_STAGE}" ]; then
+				PREV_IMG="image-${CURR_STAGE}"
+				continue
+			fi
+		echo "Rebasing back image-${CURR_STAGE}.qcow2 onto ${PREV_IMG}.qcow2"
+			qemu-img rebase -f qcow2 -u -b ${PREV_IMG}.qcow2 image-${CURR_STAGE}.qcow2
+			if [ "${CURR_STAGE}" = "${EXPORT_STAGE}" ]; then
+				break
+			fi
+			PREV_IMG="image-${CURR_STAGE}"
+		done
+		popd > /dev/null
+
+		mkdir -p "${WORK_DIR}/export-image/rootfs"
+		mv "${WORK_DIR}/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/"
+		echo "Mounting image ${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2 to rootfs ${WORK_DIR}/export-image/rootfs"
+		mount_qimage "${WORK_DIR}/export-image/${EXPORT_NAME}.qcow2" "${WORK_DIR}/export-image/rootfs"
+
+		CLEAN=0
+		run_stage
+		CLEAN=1
+		USE_QCOW2=1
+
+	else
+		run_stage
+	fi 
 	if [ "${USE_QEMU}" != "1" ]; then
 		if [ -e "${EXPORT_DIR}/EXPORT_NOOBS" ]; then
 			# shellcheck source=/dev/null
 			source "${EXPORT_DIR}/EXPORT_NOOBS"
 			STAGE_DIR="${BASE_DIR}/export-noobs"
-			run_stage
+			if [ "${USE_QCOW2}" = "1" ]; then
+				USE_QCOW2=0
+				run_stage
+				USE_QCOW2=1
+			else
+				run_stage
+			fi
 		fi
 	fi
 done
 
-if [ -x ${BASE_DIR}/postrun.sh ]; then
+if [ -x postrun.sh ]; then
 	log "Begin postrun.sh"
 	cd "${BASE_DIR}"
 	./postrun.sh
 	log "End postrun.sh"
 fi
 
+if [ "${USE_QCOW2}" = "1" ]; then
+	unload_qimage
+fi
+
 log "End ${BASE_DIR}"

depends

@@ -17,3 +17,5 @@ file
 git
 lsmod:kmod
 bc
+qemu-nbd:qemu-utils
+kpartx