From 5b9195d30640df64f5b53d9a9e9414a85d1fcbeb Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Fri, 5 Sep 2025 13:22:33 -0400 Subject: [PATCH 1/3] Apply runtime kernel tweaks to memory allocation These adjustments prefer writing to disk and reclaiming cache earlier in order to avoid the frequent "out of memory error 12" we have been experiencing on firecracker VM startup. We observed very large (50%) memory caches that were not freed up in time to start the next VM, so these settings will optimize for freeing the memory sooner. Change-type: minor Signed-off-by: Kyle Harding --- docker-compose.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 866134d6..2e14ffe4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -156,6 +156,56 @@ services: labels: io.balena.features.balena-api: '1' + memory-tweaks: + image: bash:alpine3.22 + entrypoint: ["/usr/local/bin/bash", "-c"] + privileged: true + environment: + DISABLED: 0 + VERBOSE: 0 + command: + - | + set -euo pipefail + + case ${DISABLED,,} in + true|1|y|yes|on) + exit 0 + ;; + *) + ;; + esac + + case ${VERBOSE,,} in + true|1|y|yes|on) + set -x + ;; + *) + ;; + esac + + echo "Setting swappiness to 5 (prioritize cache reclaim over swap)" + echo 5 > /proc/sys/vm/swappiness + + echo "Setting vfs_cache_pressure to 250 (aggressive filesystem cache reclaim)" + echo 250 > /proc/sys/vm/vfs_cache_pressure + + echo "Setting dirty_ratio to 5 (keep dirty pages low)" + echo 5 > /proc/sys/vm/dirty_ratio + + echo "Setting dirty_background_ratio to 2 (start background writeout early)" + echo 2 > /proc/sys/vm/dirty_background_ratio + + echo "Setting overcommit_memory to 0 (heuristic overcommit)" + echo 0 > /proc/sys/vm/overcommit_memory + + echo "Setting min_free_kbytes to 1048576 (keep 1GB always free)" + echo 1048576 > /proc/sys/vm/min_free_kbytes + + while true; do + free -h + sleep 1h + done + # enable IPv6 enable-ipv6: image: bash:alpine3.14 From 3565fc4d234a483cfac851bfb1677c90379464d9 Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Tue, 14 Oct 2025 10:07:11 -0400 Subject: [PATCH 2/3] Apply balanced settings to avoid fragmentation Signed-off-by: Kyle Harding --- docker-compose.yml | 69 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2e14ffe4..f9827f80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -160,9 +160,16 @@ services: image: bash:alpine3.22 entrypoint: ["/usr/local/bin/bash", "-c"] privileged: true + labels: + io.balena.features.procfs: 1 + io.balena.features.sysfs: 1 environment: DISABLED: 0 VERBOSE: 0 + VM_COMPACTION_PROACTIVENESS: 40 + VM_MIN_FREE_KBYTES: 524288 # 512MB + MM_TRANSPARENT_HUGEPAGE_ENABLED: never + MM_TRANSPARENT_HUGEPAGE_DEFRAG: never command: - | set -euo pipefail @@ -183,27 +190,61 @@ services: ;; esac - echo "Setting swappiness to 5 (prioritize cache reclaim over swap)" - echo 5 > /proc/sys/vm/swappiness + case ${MM_TRANSPARENT_HUGEPAGE_ENABLED,,} in + always|madvise|never) + ;; + *) + echo "Invalid value for MM_TRANSPARENT_HUGEPAGE_ENABLED: ${MM_TRANSPARENT_HUGEPAGE_ENABLED}" + exit 1 + ;; + esac + + case ${MM_TRANSPARENT_HUGEPAGE_DEFRAG,,} in + always|madvise|never) + ;; + *) + echo "Invalid value for MM_TRANSPARENT_HUGEPAGE_DEFRAG: ${MM_TRANSPARENT_HUGEPAGE_DEFRAG}" + exit 1 + ;; + esac - echo "Setting vfs_cache_pressure to 250 (aggressive filesystem cache reclaim)" - echo 250 > /proc/sys/vm/vfs_cache_pressure + # Match only numeric values between 0 and 100 + case ${VM_COMPACTION_PROACTIVENESS} in + 100|[1-9][0-9]|[0-9]) + ;; + *) + echo "Invalid value for VM_COMPACTION_PROACTIVENESS: ${VM_COMPACTION_PROACTIVENESS} (must be 0-100)" + exit 1 + ;; + esac - echo "Setting dirty_ratio to 5 (keep dirty pages low)" - echo 5 > /proc/sys/vm/dirty_ratio + case ${VM_MIN_FREE_KBYTES} in + ''|*[![:digit:]]*) + echo "Invalid value for VM_MIN_FREE_KBYTES: ${VM_MIN_FREE_KBYTES} (must be numeric)" + exit 1 + ;; + *) + ;; + esac - echo "Setting dirty_background_ratio to 2 (start background writeout early)" - echo 2 > /proc/sys/vm/dirty_background_ratio + # Enable aggressive background compaction + sysctl -w vm.compaction_proactiveness=${VM_COMPACTION_PROACTIVENESS} - echo "Setting overcommit_memory to 0 (heuristic overcommit)" - echo 0 > /proc/sys/vm/overcommit_memory + # Keep more memory free for allocations + sysctl -w vm.min_free_kbytes=${VM_MIN_FREE_KBYTES} - echo "Setting min_free_kbytes to 1048576 (keep 1GB always free)" - echo 1048576 > /proc/sys/vm/min_free_kbytes + # Disable THP to reduce fragmentation + echo "${MM_TRANSPARENT_HUGEPAGE_ENABLED,,}" > /sys/kernel/mm/transparent_hugepage/enabled + echo "${MM_TRANSPARENT_HUGEPAGE_DEFRAG,,}" > /sys/kernel/mm/transparent_hugepage/defrag while true; do - free -h - sleep 1h + # Watch fragmentation improve over time + cat /proc/buddyinfo + + # Monitor compaction activity + cat /proc/vmstat | grep compact_ + + sleep 15m done # enable IPv6 From 98511b4652ad5e1a574ccba9650b7d403ad208e7 Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Tue, 14 Oct 2025 16:21:54 -0400 Subject: [PATCH 3/3] Prevent the kernel from swapping guest pages to disk Change-type: patch Signed-off-by: Kyle Harding --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index f9827f80..cf6e8652 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,6 +29,11 @@ x-runner-vm: - /dev/net/tun # Required to create TAP/TUN device sysctls: - net.ipv4.ip_forward=1 # Required for VM networking + ulimits: + # Prevents the kernel from swapping guest pages to disk + memlock: + soft: -1 # unlimited + hard: -1 # unlimited tmpfs: - /tmp networks: