From 74174b20f42a083319771e09ec6e9c810a562739 Mon Sep 17 00:00:00 2001 From: Ruud Peters Date: Sat, 24 Aug 2024 14:43:48 +0200 Subject: [PATCH] vm kernel customization and improved build caching (#8) --- README.md | 6 +-- dockerfile | 125 ++++++++++++++++++++++++++----------------- src/conf/custom.conf | 98 +++++++++++++++++++++++++++++++++ 3 files changed, 178 insertions(+), 51 deletions(-) create mode 100644 src/conf/custom.conf diff --git a/README.md b/README.md index 1d94790..086d395 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ Example use cases: Key features: - Pi 3, 4 and **5** support -- 64 bit (ARMv8) Raspberry PI OS (Bookworm) included - - image: **[2024-07-04-raspios-bookworm-arm64-lite](https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/)** - - kernel: **[6.6-y](https://github.com/raspberrypi/linux/tree/rpi-6.6.y)** +- 64 bit (ARMv8) Raspberry PI OS (24.04, Bookworm) included + - Image: **[2024-07-04-raspios-bookworm-arm64-lite](https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/)** + - Kernel: **[6.6-y](https://github.com/raspberrypi/linux/tree/rpi-6.6.y)** - Internet access - No root required - Safe, fully reproducible from source diff --git a/dockerfile b/dockerfile index 894662f..10ef995 100644 --- a/dockerfile +++ b/dockerfile @@ -1,41 +1,37 @@ # PI-CI -# Shared variables -ARG BUILD_DIR=/build/ - -FROM ubuntu:24.04 AS builder - -# Use shared build directory -ARG BUILD_DIR - # Kernel source -ARG KERNEL_GIT=https://github.com/raspberrypi/linux.git ARG KERNEL_BRANCH=rpi-6.6.y +ARG KERNEL_GIT=https://github.com/raspberrypi/linux.git -# Distro download -ARG DISTRO_FILE=2024-07-04-raspios-bookworm-arm64-lite.img -ARG DISTRO_IMG=https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-2024-07-04/$DISTRO_FILE.xz +# Distro source +ARG DISTRO_DATE=2024-07-04 +ARG DISTRO_FILE=$DISTRO_DATE-raspios-bookworm-arm64-lite.img +ARG DISTRO_IMG=https://downloads.raspberrypi.com/raspios_lite_arm64/images/raspios_lite_arm64-$DISTRO_DATE/$DISTRO_FILE.xz -# Kernel compile options -ARG ARCH=arm64 -ARG CROSS_COMPILE=aarch64-linux-gnu- +# Default directory and file names +ARG BUILD_DIR=/build/ +ARG BASE_DIR=/base/ +ARG APP_DIR=/app/ + +ARG IMAGE_FILE_NAME=distro.qcow2 +ARG KERNEL_FILE_NAME=kernel.img + +# -------------------------------- +FROM ubuntu:24.04 AS image-builder # Install dependencies ARG DEBIAN_FRONTEND="noninteractive" RUN apt-get update && apt install -y \ - bc \ - bison \ - crossbuild-essential-arm64 \ - flex \ - git \ - libc6-dev \ libguestfs-tools \ libssl-dev \ - linux-image-generic \ - make \ wget \ openssl \ + linux-image-generic \ xz-utils +ARG DISTRO_FILE +ARG DISTRO_IMG + # Download raspbian distro RUN wget -nv -O /tmp/$DISTRO_FILE.xz $DISTRO_IMG \ && unxz /tmp/$DISTRO_FILE.xz @@ -44,15 +40,6 @@ RUN wget -nv -O /tmp/$DISTRO_FILE.xz $DISTRO_IMG \ RUN mkdir /mnt/root /mnt/boot \ && guestfish add tmp/$DISTRO_FILE : run : mount /dev/sda1 / : copy-out / /mnt/boot : umount / : mount /dev/sda2 / : copy-out / /mnt/root -# Clone the RPI kernel repo -RUN git clone --single-branch --branch $KERNEL_BRANCH $KERNEL_GIT $BUILD_DIR/linux/ - -# Cross compile vm kernel image -RUN make -C $BUILD_DIR/linux defconfig \ - && make -C $BUILD_DIR/linux kvm_guest.config \ - && make -C $BUILD_DIR/linux/ -j$(nproc) Image \ - && mv $BUILD_DIR/linux/arch/arm64/boot/Image $BUILD_DIR/kernel.img - # Copy boot configuration COPY src/conf/fstab /mnt/root/etc/ COPY src/conf/cmdline.txt /mnt/boot/ @@ -80,33 +67,56 @@ COPY src/conf/login.conf /mnt/root/etc/systemd/system/serial-getty@ttyAMA0.servi RUN rm /mnt/root/usr/lib/systemd/system/userconfig.service \ && rm /mnt/root/etc/systemd/system/multi-user.target.wants/userconfig.service -# Create new distro image from modified boot and root + # Create new distro image from modified boot and root +ARG BUILD_DIR +RUN mkdir $BUILD_DIR RUN guestfish -N $BUILD_DIR/distro.img=bootroot:vfat:ext4:2G \ && guestfish add $BUILD_DIR/distro.img : run : mount /dev/sda1 / : glob copy-in /mnt/boot/* / : umount / : mount /dev/sda2 / : glob copy-in /mnt/root/* / \ && sfdisk --part-type $BUILD_DIR/distro.img 1 c -# Convert new distro image to sparse file + +# Convert new distro image to sparse format RUN qemu-img convert -f raw -O qcow2 $BUILD_DIR/distro.img $BUILD_DIR/distro.qcow2 -CMD cp $BUILD_DIR/distro.qcow2 ./ +# --------------------------------- +FROM ubuntu:24.04 AS kernel-builder -# --------------------------- -FROM ubuntu:24.04 AS emulator +# Install dependencies +ARG DEBIAN_FRONTEND="noninteractive" +RUN apt-get update && apt install -y \ + bc \ + bison \ + crossbuild-essential-arm64 \ + flex \ + git \ + libssl-dev \ + linux-image-generic \ + make -# Project build directory +ARG KERNEL_GIT +ARG KERNEL_BRANCH ARG BUILD_DIR -# Folder containing default configuration files -ENV BASE_DIR=/base/ -# Folder containing helper scripts -ENV APP_DIR=/app/ -ENV IMAGE_FILE_NAME=distro.qcow2 -ENV KERNEL_FILE_NAME=kernel.img +# Clone the RPI kernel repo +RUN git clone --single-branch --branch $KERNEL_BRANCH $KERNEL_GIT $BUILD_DIR/linux/ -# Copy build files -RUN mkdir $BASE_DIR -COPY --from=0 $BUILD_DIR/$IMAGE_FILE_NAME $BASE_DIR/$IMAGE_FILE_NAME -COPY --from=0 $BUILD_DIR/$KERNEL_FILE_NAME $BASE_DIR/$KERNEL_FILE_NAME +# Kernel compile options +ARG ARCH=arm64 +ARG CROSS_COMPILE=aarch64-linux-gnu- + +# Compile default VM guest image +RUN make -C $BUILD_DIR/linux defconfig kvm_guest.config \ + && make -C $BUILD_DIR/linux -j$(nproc) Image + +# Customize guest image +COPY src/conf/custom.conf $BUILD_DIR/linux/kernel/configs/custom.config +RUN make -C $BUILD_DIR/linux custom.config \ + && make -C $BUILD_DIR/linux -j$(nproc) Image \ + && mv $BUILD_DIR/linux/arch/arm64/boot/Image $BUILD_DIR/kernel.img + + +# --------------------------- +FROM ubuntu:24.04 AS emulator # Install packages and build essentials ARG DEBIAN_FRONTEND="noninteractive" @@ -118,7 +128,9 @@ RUN apt-get update && apt install -y \ libguestfs-tools \ qemu-efi-aarch64 + ENV PIP_BREAK_SYSTEM_PACKAGES 1 +ARG APP_DIR # Copy and install Python dependencies COPY src/app/requirements.txt $APP_DIR/requirements.txt @@ -127,6 +139,18 @@ RUN pip3 install -r $APP_DIR/requirements.txt # Copy helper scripts COPY src/app/ $APP_DIR +# Copy build files +ARG BASE_DIR +ARG BUILD_DIR + +ARG IMAGE_FILE_NAME +ARG KERNEL_FILE_NAME + +RUN mkdir $BASE_DIR + +COPY --from=image-builder $BUILD_DIR/$IMAGE_FILE_NAME $BASE_DIR/$IMAGE_FILE_NAME +COPY --from=kernel-builder $BUILD_DIR/$KERNEL_FILE_NAME $BASE_DIR/$KERNEL_FILE_NAME + # Helper script on running container ENTRYPOINT ["/app/run.py"] @@ -135,3 +159,8 @@ ENV PYTHONDONTWRITEBYTECODE 1 ENV DIST_DIR /dist ENV STORAGE_PATH /dev/mmcblk0 ENV PORT 2222 + +ENV BASE_DIR $BASE_DIR +ENV APP_DIR $APP_DIR +ENV IMAGE_FILE_NAME $IMAGE_FILE_NAME +ENV KERNEL_FILE_NAME $KERNEL_FILE_NAME diff --git a/src/conf/custom.conf b/src/conf/custom.conf new file mode 100644 index 0000000..f368249 --- /dev/null +++ b/src/conf/custom.conf @@ -0,0 +1,98 @@ +# CUSTOM VM KERNEL PARAMETERS +# --------------------------- + +# Allow docker to be started in the virtual machine, +# in combination with { "bridge": "none" } in /etc/docker/daemon.json +CONFIG_OVERLAY_FS=y +CONFIG_NF_TABLES=y +CONFIG_NF_TABLES_IPV4=y + +# Allow docker to load images that lsetxattr +CONFIG_SECURITY_FILE_CAPABILITIES=y +CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_XATTR=y +CONFIG_EXT4_FS_SECURITY=y +CONFIG_EXT4_FS_POSIX_ACL=y + + +# POTENTIALLY USEFUL +# ---------- + +# CONFIG_OVERLAY_FS=y + +# CONFIG_NET_IP_TUNNEL=y +# CONFIG_INET_TUNNEL=y + +# CONFIG_IPV6=y +# CONFIG_INET6_TUNNEL=y +# CONFIG_IPV6_VTI=y +# CONFIG_IPV6_SIT=y + +# CONFIG_BRIDGE_NETFILTER=y + +# CONFIG_NETFILTER_NETLINK=y +# CONFIG_NETFILTER_FAMILY_ARP=y + +# CONFIG_NF_CONNTRACK=y +# CONFIG_NF_NAT=y + +# CONFIG_NF_TABLES=y +# CONFIG_NF_TABLES_INET=y +# CONFIG_NF_TABLES_NETDEV=y + +# CONFIG_NFT_MASQ=y +# CONFIG_NFT_NAT=y + +# CONFIG_NETFILTER_XTABLES=y +# CONFIG_NETFILTER_XT_TARGET_AUDIT=y + +# CONFIG_NETFILTER_XT_NAT=y +# CONFIG_NETFILTER_XT_TARGET_MASQUERADE=y + +# CONFIG_NETFILTER_XT_MATCH_ADDRTYPE=y +# CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y +# CONFIG_NETFILTER_XT_MATCH_POLICY=y +# CONFIG_NETFILTER_XT_MATCH_RECENT=y +# CONFIG_NETFILTER_XT_MATCH_SOCKET=y +# CONFIG_NETFILTER_XT_MATCH_STATE=y + +# CONFIG_NF_DEFRAG_IPV4=y +# CONFIG_NF_SOCKET_IPV4=y +# CONFIG_NF_TABLES_IPV4=y +# CONFIG_NF_TABLES_ARP=y + +# CONFIG_IP_NF_IPTABLES=y +# CONFIG_IP_NF_FILTER=y +# CONFIG_IP_NF_NAT=y +# CONFIG_IP_NF_TARGET_MASQUERADE=y + +# CONFIG_NF_SOCKET_IPV6=y +# CONFIG_NF_TABLES_IPV6=y + +# CONFIG_IP6_NF_FILTER=y +# CONFIG_IP6_NF_NAT=y +# CONFIG_IP6_NF_IPTABLES=y +# CONFIG_IP6_NF_TARGET_MASQUERADE=y + +# CONFIG_NF_DEFRAG_IPV6=y +# CONFIG_NF_TABLES_BRIDGE=y +# CONFIG_NF_CONNTRACK_BRIDGE=y +# CONFIG_BRIDGE_NF_EBTABLES=y + +# CONFIG_BRIDGE_NF_EBTABLES=y + +# CONFIG_STP=y +# CONFIG_BRIDGE=y + +# CONFIG_LLC=y + +# CONFIG_SECURITY=y +# CONFIG_CRYPTO=y + +# CONFIG_NAMESPACES=y +# CONFIG_USER_NS=y + +# CONFIG_FS_POSIX_ACL=y +# CONFIG_FS_SECURITY=y +# CONFIG_KEYS=y +# CONFIG_SECURITYFS=y