vm kernel customization and improved build caching (#8)

README.md

@@ -15,9 +15,9 @@ Example use cases:
 
 Key features:
 - Pi 3, 4 and **5** support
-- 64 bit (ARMv8) Raspberry PI OS (Bookworm) included
-  - image: **[2024-07-04-raspios-bookworm-arm64-lite](https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/)**
-  - kernel: **[6.6-y](https://github.com/raspberrypi/linux/tree/rpi-6.6.y)**
+- 64 bit (ARMv8) Raspberry PI OS (24.04, Bookworm) included
+  - Image: **[2024-07-04-raspios-bookworm-arm64-lite](https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/)**
+  - Kernel: **[6.6-y](https://github.com/raspberrypi/linux/tree/rpi-6.6.y)**
 - Internet access
 - No root required
 - Safe, fully reproducible from source

dockerfile

@@ -1,41 +1,37 @@
 # PI-CI
-# Shared variables
-ARG BUILD_DIR=/build/
-
-FROM ubuntu:24.04 AS builder
-
-# Use shared build directory
-ARG BUILD_DIR
-
 # Kernel source
-ARG KERNEL_GIT=https://github.com/raspberrypi/linux.git
 ARG KERNEL_BRANCH=rpi-6.6.y
+ARG KERNEL_GIT=https://github.com/raspberrypi/linux.git
 
-# Distro download
-ARG DISTRO_FILE=2024-07-04-raspios-bookworm-arm64-lite.img
-ARG DISTRO_IMG=https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/$DISTRO_FILE.xz
+# Distro source
+ARG DISTRO_DATE=2024-07-04
+ARG DISTRO_FILE=$DISTRO_DATE-raspios-bookworm-arm64-lite.img
+ARG DISTRO_IMG=https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-$DISTRO_DATE/$DISTRO_FILE.xz
 
-# Kernel compile options
-ARG ARCH=arm64
-ARG CROSS_COMPILE=aarch64-linux-gnu-
+# Default directory and file names
+ARG BUILD_DIR=/build/
+ARG BASE_DIR=/base/
+ARG APP_DIR=/app/
+
+ARG IMAGE_FILE_NAME=distro.qcow2
+ARG KERNEL_FILE_NAME=kernel.img
+
+# --------------------------------
+FROM ubuntu:24.04 AS image-builder
 
 # Install dependencies
 ARG DEBIAN_FRONTEND="noninteractive"
 RUN apt-get update && apt install -y \
-    bc \
-    bison \
-    crossbuild-essential-arm64 \
-    flex \
-    git \
-    libc6-dev \
     libguestfs-tools \
     libssl-dev \
-    linux-image-generic \
-    make \
     wget \
     openssl \
+    linux-image-generic \
     xz-utils
 
+ARG DISTRO_FILE
+ARG DISTRO_IMG
+
 # Download raspbian distro
 RUN wget -nv -O /tmp/$DISTRO_FILE.xz $DISTRO_IMG \
  && unxz /tmp/$DISTRO_FILE.xz
@@ -44,15 +40,6 @@ RUN wget -nv -O /tmp/$DISTRO_FILE.xz $DISTRO_IMG \
 RUN mkdir /mnt/root /mnt/boot \
  && guestfish add tmp/$DISTRO_FILE : run : mount /dev/sda1 / : copy-out / /mnt/boot : umount / : mount /dev/sda2 / : copy-out / /mnt/root
 
-# Clone the RPI kernel repo
-RUN git clone --single-branch --branch $KERNEL_BRANCH $KERNEL_GIT $BUILD_DIR/linux/
-
-# Cross compile vm kernel image
-RUN make -C $BUILD_DIR/linux defconfig \
- && make -C $BUILD_DIR/linux kvm_guest.config \
- && make -C $BUILD_DIR/linux/ -j$(nproc) Image \
- && mv $BUILD_DIR/linux/arch/arm64/boot/Image $BUILD_DIR/kernel.img
-
 # Copy boot configuration
 COPY src/conf/fstab /mnt/root/etc/
 COPY src/conf/cmdline.txt /mnt/boot/
@@ -80,33 +67,56 @@ COPY src/conf/login.conf /mnt/root/etc/systemd/system/[email protected]
 RUN rm /mnt/root/usr/lib/systemd/system/userconfig.service \
  && rm /mnt/root/etc/systemd/system/multi-user.target.wants/userconfig.service
 
-# Create new distro image from modified boot and root
+ # Create new distro image from modified boot and root
+ARG BUILD_DIR
+RUN mkdir $BUILD_DIR
 RUN guestfish -N $BUILD_DIR/distro.img=bootroot:vfat:ext4:2G \
  && guestfish add $BUILD_DIR/distro.img : run : mount /dev/sda1 / : glob copy-in /mnt/boot/* / : umount / : mount /dev/sda2 / : glob copy-in /mnt/root/* / \
  && sfdisk --part-type $BUILD_DIR/distro.img 1 c
-# Convert new distro image to sparse file
+
+# Convert new distro image to sparse format
 RUN qemu-img convert -f raw -O qcow2 $BUILD_DIR/distro.img $BUILD_DIR/distro.qcow2
 
-CMD cp $BUILD_DIR/distro.qcow2 ./
 
+# ---------------------------------
+FROM ubuntu:24.04 AS kernel-builder
 
-# ---------------------------
-FROM ubuntu:24.04 AS emulator
+# Install dependencies
+ARG DEBIAN_FRONTEND="noninteractive"
+RUN apt-get update && apt install -y \
+    bc \
+    bison \
+    crossbuild-essential-arm64 \
+    flex \
+    git \
+    libssl-dev \
+    linux-image-generic \
+    make
 
-# Project build directory
+ARG KERNEL_GIT
+ARG KERNEL_BRANCH
 ARG BUILD_DIR
-# Folder containing default configuration files
-ENV BASE_DIR=/base/
-# Folder containing helper scripts
-ENV APP_DIR=/app/
 
-ENV IMAGE_FILE_NAME=distro.qcow2
-ENV KERNEL_FILE_NAME=kernel.img
+# Clone the RPI kernel repo
+RUN git clone --single-branch --branch $KERNEL_BRANCH $KERNEL_GIT $BUILD_DIR/linux/
 
-# Copy build files
-RUN mkdir $BASE_DIR
-COPY --from=0 $BUILD_DIR/$IMAGE_FILE_NAME $BASE_DIR/$IMAGE_FILE_NAME
-COPY --from=0 $BUILD_DIR/$KERNEL_FILE_NAME $BASE_DIR/$KERNEL_FILE_NAME
+# Kernel compile options
+ARG ARCH=arm64
+ARG CROSS_COMPILE=aarch64-linux-gnu-
+
+# Compile default VM guest image
+RUN make -C $BUILD_DIR/linux defconfig kvm_guest.config \
+ && make -C $BUILD_DIR/linux -j$(nproc) Image
+
+# Customize guest image
+COPY src/conf/custom.conf $BUILD_DIR/linux/kernel/configs/custom.config
+RUN make -C $BUILD_DIR/linux custom.config \
+ && make -C $BUILD_DIR/linux -j$(nproc) Image \
+ && mv $BUILD_DIR/linux/arch/arm64/boot/Image $BUILD_DIR/kernel.img
+
+
+# ---------------------------
+FROM ubuntu:24.04 AS emulator
 
 # Install packages and build essentials
 ARG DEBIAN_FRONTEND="noninteractive"
@@ -118,7 +128,9 @@ RUN apt-get update && apt install -y \
     libguestfs-tools \
     qemu-efi-aarch64
 
+
 ENV PIP_BREAK_SYSTEM_PACKAGES 1
+ARG APP_DIR
 
 # Copy and install Python dependencies
 COPY src/app/requirements.txt $APP_DIR/requirements.txt
@@ -127,6 +139,18 @@ RUN pip3 install -r $APP_DIR/requirements.txt
 # Copy helper scripts
 COPY src/app/ $APP_DIR
 
+# Copy build files
+ARG BASE_DIR
+ARG BUILD_DIR
+
+ARG IMAGE_FILE_NAME
+ARG KERNEL_FILE_NAME
+
+RUN mkdir $BASE_DIR
+
+COPY --from=image-builder $BUILD_DIR/$IMAGE_FILE_NAME $BASE_DIR/$IMAGE_FILE_NAME
+COPY --from=kernel-builder $BUILD_DIR/$KERNEL_FILE_NAME $BASE_DIR/$KERNEL_FILE_NAME
+
 # Helper script on running container
 ENTRYPOINT ["/app/run.py"]
 
@@ -135,3 +159,8 @@ ENV PYTHONDONTWRITEBYTECODE 1
 ENV DIST_DIR /dist
 ENV STORAGE_PATH /dev/mmcblk0
 ENV PORT 2222
+
+ENV BASE_DIR $BASE_DIR
+ENV APP_DIR $APP_DIR
+ENV IMAGE_FILE_NAME $IMAGE_FILE_NAME
+ENV KERNEL_FILE_NAME $KERNEL_FILE_NAME

src/conf/custom.conf

@@ -0,0 +1,98 @@
+# CUSTOM VM KERNEL PARAMETERS
+# ---------------------------
+
+# Allow docker to be started in the virtual machine,
+# in combination with { "bridge": "none" } in /etc/docker/daemon.json
+CONFIG_OVERLAY_FS=y
+CONFIG_NF_TABLES=y
+CONFIG_NF_TABLES_IPV4=y
+
+# Allow docker to load images that lsetxattr
+CONFIG_SECURITY_FILE_CAPABILITIES=y
+CONFIG_EXT4_FS=y
+CONFIG_EXT4_FS_XATTR=y
+CONFIG_EXT4_FS_SECURITY=y
+CONFIG_EXT4_FS_POSIX_ACL=y
+
+
+# POTENTIALLY USEFUL
+# ----------
+
+# CONFIG_OVERLAY_FS=y
+
+# CONFIG_NET_IP_TUNNEL=y
+# CONFIG_INET_TUNNEL=y
+
+# CONFIG_IPV6=y
+# CONFIG_INET6_TUNNEL=y
+# CONFIG_IPV6_VTI=y
+# CONFIG_IPV6_SIT=y
+
+# CONFIG_BRIDGE_NETFILTER=y
+
+# CONFIG_NETFILTER_NETLINK=y
+# CONFIG_NETFILTER_FAMILY_ARP=y
+
+# CONFIG_NF_CONNTRACK=y
+# CONFIG_NF_NAT=y
+
+# CONFIG_NF_TABLES=y
+# CONFIG_NF_TABLES_INET=y
+# CONFIG_NF_TABLES_NETDEV=y
+
+# CONFIG_NFT_MASQ=y
+# CONFIG_NFT_NAT=y
+
+# CONFIG_NETFILTER_XTABLES=y
+# CONFIG_NETFILTER_XT_TARGET_AUDIT=y
+
+# CONFIG_NETFILTER_XT_NAT=y
+# CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y
+
+# CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y
+# CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
+# CONFIG_NETFILTER_XT_MATCH_POLICY=y
+# CONFIG_NETFILTER_XT_MATCH_RECENT=y
+# CONFIG_NETFILTER_XT_MATCH_SOCKET=y
+# CONFIG_NETFILTER_XT_MATCH_STATE=y
+
+# CONFIG_NF_DEFRAG_IPV4=y
+# CONFIG_NF_SOCKET_IPV4=y
+# CONFIG_NF_TABLES_IPV4=y
+# CONFIG_NF_TABLES_ARP=y
+
+# CONFIG_IP_NF_IPTABLES=y
+# CONFIG_IP_NF_FILTER=y
+# CONFIG_IP_NF_NAT=y
+# CONFIG_IP_NF_TARGET_MASQUERADE=y
+
+# CONFIG_NF_SOCKET_IPV6=y
+# CONFIG_NF_TABLES_IPV6=y
+
+# CONFIG_IP6_NF_FILTER=y
+# CONFIG_IP6_NF_NAT=y
+# CONFIG_IP6_NF_IPTABLES=y
+# CONFIG_IP6_NF_TARGET_MASQUERADE=y
+
+# CONFIG_NF_DEFRAG_IPV6=y
+# CONFIG_NF_TABLES_BRIDGE=y
+# CONFIG_NF_CONNTRACK_BRIDGE=y
+# CONFIG_BRIDGE_NF_EBTABLES=y
+
+# CONFIG_BRIDGE_NF_EBTABLES=y
+
+# CONFIG_STP=y
+# CONFIG_BRIDGE=y
+
+# CONFIG_LLC=y
+
+# CONFIG_SECURITY=y
+# CONFIG_CRYPTO=y
+
+# CONFIG_NAMESPACES=y
+# CONFIG_USER_NS=y
+
+# CONFIG_FS_POSIX_ACL=y
+# CONFIG_FS_SECURITY=y
+# CONFIG_KEYS=y
+# CONFIG_SECURITYFS=y