stack/hw/build-iso.sh

#!/bin/bash
#
# Custom Alpine ISO builder for headless NVMe+SATA storage server
# Hardware: ASUS ROG Crosshair VIII Hero, AMD 3950X, 192GB RAM
# 6x WD Blue 3D NAND 2TB M.2 2280 SATA III (WDS200T2B0B, up to 560 MB/s) via adapters
# + 1x NVMe 2TB + Samsung Fit 128GB USB
#
set -euo pipefail

ISO_TAG="storagenode"
ALPINE_VERSION="3.21"
ARCH="x86_64"
WORKDIR="$(pwd)/build"
OUTDIR="$(pwd)/out"

MIRROR="https://dl-cdn.alpinelinux.org/alpine"
MAIN_REPO="${MIRROR}/v${ALPINE_VERSION}/main"
COMMUNITY_REPO="${MIRROR}/v${ALPINE_VERSION}/community"

echo "=== Custom Alpine ISO Builder ==="
echo "Target: headless storage server (3950X / X470 / 192GB)"
echo ""

# Clean previous builds
rm -rf "${WORKDIR}"
mkdir -p "${WORKDIR}" "${OUTDIR}"

# Clone aports for mkimage tooling
if [ ! -d "${WORKDIR}/aports" ]; then
    echo "[1/4] Cloning aports build infrastructure..."
    git clone --depth 1 --branch v${ALPINE_VERSION} \
        https://gitlab.alpinelinux.org/alpine/aports.git \
        "${WORKDIR}/aports"
fi

# Create the custom profile
echo "[2/4] Writing custom ISO profile..."
cat > "${WORKDIR}/aports/scripts/mkimg.${ISO_TAG}.sh" << 'PROFILE'
profile_storagenode() {
    title="Alpine Storage Node"
    desc="Headless server - 3950X + SATA LVM + NVMe Docker"
    profile_standard
    arch="x86_64"
    output_format="iso"
    image_ext="iso"
    kernel_flavors="lts"
    apks="$apks
        xfsprogs
        e2fsprogs
        dosfstools
        docker
        docker-cli
        docker-compose
        openssh
        chrony
        htop
        iotop
        lm-sensors
        smartmontools
        nvme-cli
        wireguard-tools
        nano
        curl
        wget
        bash
        amd-ucode
        cpufrequtils
        irqbalance
        haveged
        nftables
        sfdisk
        sgdisk
        parted
        lsblk
        util-linux
        pciutils
        usbutils
        bonnie++
        fio
        ethtool
        rsync
        logrotate
        cryptsetup
        lvm2
        yubikey-manager
        yubico-pam
        libfido2
        openssh-server-common-openrc
        cloudflared
        certbot
        openssl
    "

}
PROFILE

# -------------------------------------------------------------------
# Verify nanny.sh outputs exist
# -------------------------------------------------------------------
for required in drives.conf drive-resolver.sh yubikeys.conf authorized_keys cloudflare.conf; do
    if [ ! -f "$(pwd)/${required}" ]; then
        echo "ERROR: ${required} not found. Run ./nanny.sh first to enroll hardware."
        exit 1
    fi
done

echo "[3/5] Verifying enrolled hardware..."
SATA_ENROLLED=$(grep -c "^DRIVE_SATA_" drives.conf)
NVME_ENROLLED=$(grep -c "^DRIVE_NVME_" drives.conf)
HDD_ENROLLED=$(grep -c "^DRIVE_HDD_" drives.conf)
YK_ENROLLED=$(grep -c "^YUBIKEY_" yubikeys.conf)
SSH_KEYS=$(ls ssh-keys/*.pub 2>/dev/null | wc -l)
HAS_TUNNEL=$(grep -c "^CF_TUNNEL_TOKEN=" cloudflare.conf 2>/dev/null || echo 0)

echo "  SATA drives:  ${SATA_ENROLLED}/6"
echo "  NVMe drives:  ${NVME_ENROLLED}/1"
echo "  HDD backup:   ${HDD_ENROLLED}/1"
echo "  YubiKeys:     ${YK_ENROLLED}/2"
echo "  SSH keys:     ${SSH_KEYS}"
echo "  CF Tunnel:    $([ "$HAS_TUNNEL" -gt 0 ] && echo 'configured' || echo 'missing')"

[ "$SATA_ENROLLED" -eq 6 ] || { echo "ERROR: Need 6 SATA drives enrolled."; exit 1; }
[ "$NVME_ENROLLED" -eq 1 ] || { echo "ERROR: Need 1 NVMe drive enrolled."; exit 1; }
[ "$HDD_ENROLLED" -eq 1 ]  || { echo "ERROR: Need 1 HDD backup drive enrolled."; exit 1; }
[ "$YK_ENROLLED" -eq 2 ]   || { echo "ERROR: Need 2 YubiKeys enrolled."; exit 1; }
[ "$SSH_KEYS" -ge 2 ]      || { echo "ERROR: Need at least 2 SSH public keys."; exit 1; }

# Source cloudflare.conf for values to inject
# shellcheck source=/dev/null
. "$(pwd)/cloudflare.conf"

echo ""

# Copy in the setup automation scripts
echo "[4/5] Embedding setup automation..."
mkdir -p "${WORKDIR}/aports/scripts/storagenode-overlay"

# Auto-setup answer file
cat > "${WORKDIR}/aports/scripts/storagenode-overlay/auto-setup.conf" << 'ANSWERS'
KEYMAPOPTS="us us"
HOSTNAMEOPTS="-n storagenode"
INTERFACESOPTS="auto lo
iface lo inet loopback

auto eth0
iface eth0 inet dhcp
"
DNSOPTS="-n 1.1.1.1 1.0.0.1"
TIMEZONEOPTS="-z UTC"
PROXYOPTS="none"
SSHDOPTS="-c openssh"
NTPOPTS="-c chrony"
# Do NOT auto-install to disk — we run the custom partitioner instead
DISKOPTS="none"
LBUOPTS="none"
APKCACHEOPTS="none"
ANSWERS

# The main post-boot setup script
cat > "${WORKDIR}/aports/scripts/storagenode-overlay/storage-setup.sh" << 'SETUP'
#!/bin/sh
#
# storage-setup.sh — Run ONCE after first boot from USB to configure everything
#
# Targets:
#   /dev/sdX     = Samsung Fit 128GB USB (already booted from this)
#   /dev/sd[a-f] = 6x WD Blue SA510 2TB via M.2-to-SATA adapters (SATA6G_1-6)
#   /dev/nvme0n1 = Modern 2TB NVMe in M.2_1 slot (Docker + writes)
#
set -euo pipefail

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m'

log()  { echo -e "${GREEN}[+]${NC} $*"; }
warn() { echo -e "${YELLOW}[!]${NC} $*"; }
err()  { echo -e "${RED}[!]${NC} $*"; exit 1; }

# -------------------------------------------------------------------
# 0. Resolve drives by enrolled serial numbers
# -------------------------------------------------------------------
[ "$(id -u)" -eq 0 ] || err "Must run as root"

log "Resolving enrolled hardware by serial number..."

# Source the drive resolver (generated by nanny.sh, embedded in ISO)
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
if [ -f "${SCRIPT_DIR}/drive-resolver.sh" ]; then
    . "${SCRIPT_DIR}/drive-resolver.sh"
elif [ -f /root/drive-resolver.sh ]; then
    . /root/drive-resolver.sh
else
    err "drive-resolver.sh not found. This ISO was not built correctly."
fi

# SATA_DRIVES, DEV_NVME, DEV_HDD are now set by drive-resolver.sh
# They were matched by the exact serial numbers from nanny enrollment.

NVME_DRIVE="$DEV_NVME"
HDD_DRIVE="$DEV_HDD"

# Find USB boot drive (the one we're running from)
USB_DRIVE=$(mount | grep " / " | awk '{print $1}' | sed 's/[0-9]*$//')

echo ""
echo "=== DRIVE ASSIGNMENT (serial-matched) ==="
echo "  USB (read-only root): ${USB_DRIVE}"
echo "  SATA LVM (RustFS):    ${SATA_DRIVES}"
echo "  NVMe (Docker/writes): ${NVME_DRIVE}"
echo "  HDD (nightly backup): ${HDD_DRIVE}"
echo ""
echo "THIS WILL DESTROY ALL DATA ON SATA, NVMe, AND HDD DRIVES."
echo -n "Continue? [y/N] "
read -r confirm
[ "$confirm" = "y" ] || [ "$confirm" = "Y" ] || exit 1

# -------------------------------------------------------------------
# 1. YubiKey LUKS passphrase setup
# -------------------------------------------------------------------
log "Setting up LUKS encryption with YubiKey challenge-response..."
echo ""
echo "You need to enroll BOTH YubiKeys for LUKS unlock."
echo "This uses YubiKey challenge-response (HMAC-SHA1, slot 2)."
echo ""
echo "If you haven't configured slot 2 on your YubiKeys yet, do it now:"
echo "  ykman otp chalresp --touch --generate 2"
echo "  (repeat for second key)"
echo ""
echo -n "Are both YubiKeys programmed with challenge-response on slot 2? [y/N] "
read -r yk_confirm
[ "$yk_confirm" = "y" ] || [ "$yk_confirm" = "Y" ] || err "Program your YubiKeys first"

echo ""
echo "You also need a backup passphrase in case both YubiKeys are lost."
echo -n "Enter a strong backup passphrase: "
read -rs BACKUP_PASSPHRASE
echo ""
echo -n "Confirm backup passphrase: "
read -rs BACKUP_CONFIRM
echo ""
[ "$BACKUP_PASSPHRASE" = "$BACKUP_CONFIRM" ] || err "Passphrases don't match"
[ ${#BACKUP_PASSPHRASE} -ge 12 ] || err "Passphrase too short (min 12 chars)"

# Generate a random LUKS key derived from YubiKey challenge-response
LUKS_KEY_FILE=$(mktemp)
CHALLENGE=$(dd if=/dev/urandom bs=32 count=1 2>/dev/null | xxd -p -c 64)
trap "rm -f ${LUKS_KEY_FILE}" EXIT

echo ""
log "Touch YubiKey #1 (PRIMARY) when it blinks..."
echo -n "$CHALLENGE" | ykchalresp -2 -x - > "${LUKS_KEY_FILE}" 2>/dev/null || \
    err "YubiKey challenge-response failed. Is the key inserted and slot 2 configured?"

# Store the challenge for future unlocks
mkdir -p /etc/yubikey
echo "$CHALLENGE" > /etc/yubikey/challenge
chmod 600 /etc/yubikey/challenge

# -------------------------------------------------------------------
# 2. Partition and encrypt NVMe drive
# -------------------------------------------------------------------
log "Partitioning NVMe drive..."
sgdisk --zap-all "$NVME_DRIVE"
sgdisk -n 1:0:+1600G -t 1:8309 -c 1:"docker-crypt" "$NVME_DRIVE"
sgdisk -n 2:0:0      -t 2:8309 -c 2:"cache-crypt"   "$NVME_DRIVE"
partprobe "$NVME_DRIVE"
sleep 2

log "Encrypting NVMe partitions with LUKS2..."

# Docker partition — YubiKey key in slot 0, backup passphrase in slot 1
cryptsetup luksFormat --type luks2 \
    --cipher aes-xts-plain64 \
    --key-size 512 \
    --hash sha512 \
    --iter-time 3000 \
    --key-file "${LUKS_KEY_FILE}" \
    --batch-mode \
    "${NVME_DRIVE}p1"

echo -n "$BACKUP_PASSPHRASE" | cryptsetup luksAddKey \
    --key-file "${LUKS_KEY_FILE}" \
    "${NVME_DRIVE}p1" -

# Cache partition — same keys
cryptsetup luksFormat --type luks2 \
    --cipher aes-xts-plain64 \
    --key-size 512 \
    --hash sha512 \
    --iter-time 3000 \
    --key-file "${LUKS_KEY_FILE}" \
    --batch-mode \
    "${NVME_DRIVE}p2"

echo -n "$BACKUP_PASSPHRASE" | cryptsetup luksAddKey \
    --key-file "${LUKS_KEY_FILE}" \
    "${NVME_DRIVE}p2" -

# Open and format
cryptsetup open --type luks2 --key-file "${LUKS_KEY_FILE}" "${NVME_DRIVE}p1" docker-crypt
cryptsetup open --type luks2 --key-file "${LUKS_KEY_FILE}" "${NVME_DRIVE}p2" cache-crypt

log "Formatting encrypted NVMe partitions..."
mkfs.xfs -f -L docker /dev/mapper/docker-crypt
mkfs.xfs -f -L cache  /dev/mapper/cache-crypt

# -------------------------------------------------------------------
# 3. Build LVM volume group across SATA drives, then LUKS on top
# -------------------------------------------------------------------
# shellcheck disable=SC2086
SATA_COUNT=$(echo $SATA_DRIVES | wc -w)
log "Creating LVM volume group across ${SATA_COUNT} SATA drives..."

# Wipe existing signatures
for d in $SATA_DRIVES; do
    wipefs -a "$d" 2>/dev/null || true
    sgdisk --zap-all "$d"
    pvcreate -ff -y "$d"
done

# shellcheck disable=SC2086
vgcreate rustfs-vg $SATA_DRIVES
lvcreate -l 100%FREE -n rustfs-lv rustfs-vg

log "Encrypting LVM volume with LUKS2..."
cryptsetup luksFormat --type luks2 \
    --cipher aes-xts-plain64 \
    --key-size 512 \
    --hash sha512 \
    --iter-time 3000 \
    --key-file "${LUKS_KEY_FILE}" \
    --batch-mode \
    /dev/rustfs-vg/rustfs-lv

echo -n "$BACKUP_PASSPHRASE" | cryptsetup luksAddKey \
    --key-file "${LUKS_KEY_FILE}" \
    /dev/rustfs-vg/rustfs-lv -

cryptsetup open --type luks2 --key-file "${LUKS_KEY_FILE}" /dev/rustfs-vg/rustfs-lv rustfs-crypt

log "Formatting encrypted LVM volume as XFS..."
mkfs.xfs -f -L rustfs /dev/mapper/rustfs-crypt

# -------------------------------------------------------------------
# 3b. Encrypt and format HDD backup drive
# -------------------------------------------------------------------
log "Setting up HDD backup drive: ${HDD_DRIVE}..."
sgdisk --zap-all "$HDD_DRIVE"
# Use whole disk, no partitioning

cryptsetup luksFormat --type luks2 \
    --cipher aes-xts-plain64 \
    --key-size 512 \
    --hash sha512 \
    --iter-time 3000 \
    --key-file "${LUKS_KEY_FILE}" \
    --batch-mode \
    "$HDD_DRIVE"

echo -n "$BACKUP_PASSPHRASE" | cryptsetup luksAddKey \
    --key-file "${LUKS_KEY_FILE}" \
    "$HDD_DRIVE" -

cryptsetup open --type luks2 --key-file "${LUKS_KEY_FILE}" "$HDD_DRIVE" backup-crypt

log "Formatting encrypted HDD as XFS..."
mkfs.xfs -f -L backup /dev/mapper/backup-crypt

# -------------------------------------------------------------------
# Enroll YubiKey #2 (BACKUP) into all LUKS volumes
# -------------------------------------------------------------------
log "Now insert YubiKey #2 (BACKUP) and remove YubiKey #1..."
echo -n "Press Enter when YubiKey #2 is inserted..."
read -r

LUKS_KEY_FILE_2=$(mktemp)
log "Touch YubiKey #2 when it blinks..."
echo -n "$CHALLENGE" | ykchalresp -2 -x - > "${LUKS_KEY_FILE_2}" 2>/dev/null || \
    err "YubiKey #2 challenge-response failed"

for vol in "${NVME_DRIVE}p1" "${NVME_DRIVE}p2" /dev/rustfs-vg/rustfs-lv "$HDD_DRIVE"; do
    cryptsetup luksAddKey --key-file "${LUKS_KEY_FILE}" "$vol" "${LUKS_KEY_FILE_2}"
done
rm -f "${LUKS_KEY_FILE_2}"

log "Both YubiKeys enrolled in all LUKS volumes."
log "Backup passphrase also enrolled (3 unlock methods per volume)."

# Clean up key material from RAM
rm -f "${LUKS_KEY_FILE}"

# -------------------------------------------------------------------
# 4. Create mount points, crypttab, and fstab
# -------------------------------------------------------------------
log "Configuring crypttab and fstab..."

mkdir -p /data/rustfs /var/lib/docker /var/cache/apk /backup

# Get UUIDs of the raw LUKS containers (not the mapper devices)
UUID_DOCKER_LUKS=$(blkid -s UUID -o value "${NVME_DRIVE}p1")
UUID_CACHE_LUKS=$(blkid -s UUID -o value "${NVME_DRIVE}p2")
UUID_RUSTFS_LUKS=$(blkid -s UUID -o value /dev/rustfs-vg/rustfs-lv)
UUID_BACKUP_LUKS=$(blkid -s UUID -o value "$HDD_DRIVE")
UUID_ROOT=$(blkid -s UUID -o value "${USB_DRIVE}2" 2>/dev/null || blkid -s UUID -o value "${USB_DRIVE}1")

# crypttab — volumes won't auto-open at boot (no keyfile in crypttab)
# They must be unlocked via yubikey-unlock script after SSH login
cat > /etc/crypttab << CRYPTTAB
# <name>        <device>                         <keyfile>  <options>
docker-crypt    UUID=${UUID_DOCKER_LUKS}          none       luks,noauto
cache-crypt     UUID=${UUID_CACHE_LUKS}           none       luks,noauto
rustfs-crypt    UUID=${UUID_RUSTFS_LUKS}          none       luks,noauto
backup-crypt    UUID=${UUID_BACKUP_LUKS}          none       luks,noauto
CRYPTTAB

cat > /etc/fstab << FSTAB
# Storage Node fstab
# USB root - READ ONLY
UUID=${UUID_ROOT}    /              ext4  ro,noatime,discard,errors=remount-ro  0 1

# LUKS-encrypted volumes — mounted after yubikey-unlock
/dev/mapper/docker-crypt  /var/lib/docker  xfs  defaults,noatime,nodiratime,noauto  0 0
/dev/mapper/cache-crypt   /var/cache       xfs  defaults,noatime,nodiratime,noauto  0 0
/dev/mapper/rustfs-crypt  /data/rustfs     xfs  defaults,noatime,nodiratime,noauto  0 0
/dev/mapper/backup-crypt  /backup          xfs  defaults,noatime,nodiratime,noauto  0 0

# tmpfs - volatile storage in RAM (192GB available)
tmpfs  /tmp      tmpfs  nosuid,nodev,size=8G           0 0
tmpfs  /run      tmpfs  nosuid,nodev,mode=0755,size=2G 0 0
tmpfs  /var/log  tmpfs  nosuid,nodev,noexec,size=2G    0 0
tmpfs  /var/tmp  tmpfs  nosuid,nodev,size=2G           0 0
FSTAB

# -------------------------------------------------------------------
# 5. YubiKey unlock script (run after SSH login to decrypt volumes)
# -------------------------------------------------------------------
log "Installing YubiKey unlock scripts..."

cat > /usr/local/bin/yubikey-unlock << 'UNLOCK'
#!/bin/sh
#
# Unlock all LUKS volumes using YubiKey challenge-response
# Run this after SSH login. Requires physical YubiKey inserted.
#
set -e

RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'

if [ "$(id -u)" -ne 0 ]; then
    echo "Must run as root (use sudo)"
    exit 1
fi

# Check if already unlocked
if [ -b /dev/mapper/rustfs-crypt ] && [ -b /dev/mapper/docker-crypt ]; then
    echo -e "${GREEN}Volumes already unlocked.${NC}"
    mount | grep -E '(docker-crypt|cache-crypt|rustfs-crypt|backup-crypt)' && exit 0
    echo "Volumes open but not mounted. Mounting..."
    mount /var/lib/docker
    mount /var/cache
    mount /data/rustfs
    mount /backup
    echo -e "${GREEN}All volumes mounted.${NC}"
    exit 0
fi

CHALLENGE=$(cat /etc/yubikey/challenge)

echo "Insert YubiKey and touch when it blinks..."
KEY_FILE=$(mktemp)
trap "rm -f ${KEY_FILE}" EXIT

echo -n "$CHALLENGE" | ykchalresp -2 -x - > "$KEY_FILE" 2>/dev/null
if [ $? -ne 0 ]; then
    echo -e "${RED}YubiKey challenge-response failed.${NC}"
    echo "Falling back to passphrase..."
    echo -n "Enter backup passphrase: "
    read -rs PASSPHRASE
    echo ""
    echo -n "$PASSPHRASE" > "$KEY_FILE"
fi

echo "Activating LVM volume group..."
vgchange -ay rustfs-vg 2>/dev/null || true

echo "Unlocking volumes..."

for vol_name in docker-crypt cache-crypt rustfs-crypt backup-crypt; do
    if [ -b "/dev/mapper/${vol_name}" ]; then
        echo "  ${vol_name}: already open"
        continue
    fi

    dev=$(grep "^${vol_name}" /etc/crypttab | awk '{print $2}')
    # Resolve UUID= to device path
    if echo "$dev" | grep -q "^UUID="; then
        uuid=$(echo "$dev" | sed 's/UUID=//')
        dev=$(blkid -U "$uuid")
    fi

    cryptsetup open --type luks2 --key-file "$KEY_FILE" "$dev" "$vol_name"
    echo -e "  ${GREEN}${vol_name}: unlocked${NC}"
done

rm -f "$KEY_FILE"

echo "Mounting filesystems..."
mount /var/lib/docker
mount /var/cache
mount /data/rustfs
mount /backup

echo "Starting Docker..."
service docker start

echo "Starting stack (RustFS + Cloudflare Tunnel)..."
cd /opt/rustfs && docker compose up -d

echo "Starting nightly backup cron..."
crond 2>/dev/null || true

echo ""
echo -e "${GREEN}All volumes unlocked. Stack is running.${NC}"
echo ""
echo "Services:"
echo "  RustFS S3:      https://s3.rig.fhirworx.io"
echo "  RustFS Console: https://console.rig.fhirworx.io"
echo "  Local S3:       http://127.0.0.1:9000"
echo "  Local Console:  http://127.0.0.1:9001"
UNLOCK
chmod +x /usr/local/bin/yubikey-unlock

cat > /usr/local/bin/yubikey-lock << 'LOCK'
#!/bin/sh
#
# Lock all LUKS volumes — stops Docker, unmounts, closes LUKS
#
set -e

if [ "$(id -u)" -ne 0 ]; then
    echo "Must run as root (use sudo)"
    exit 1
fi

echo "Stopping stack..."
cd /opt/rustfs && docker compose down 2>/dev/null || true

echo "Stopping Docker..."
service docker stop 2>/dev/null || true

# Kill any remaining containers
docker kill $(docker ps -q) 2>/dev/null || true

echo "Unmounting encrypted volumes..."
umount /backup 2>/dev/null || true
umount /data/rustfs 2>/dev/null || true
umount /var/cache 2>/dev/null || true
umount /var/lib/docker 2>/dev/null || true

echo "Closing LUKS volumes..."
cryptsetup close backup-crypt 2>/dev/null || true
cryptsetup close rustfs-crypt 2>/dev/null || true
cryptsetup close cache-crypt 2>/dev/null || true
cryptsetup close docker-crypt 2>/dev/null || true

echo "Deactivating LVM..."
vgchange -an rustfs-vg 2>/dev/null || true

echo "All volumes locked. Data is encrypted at rest."
LOCK
chmod +x /usr/local/bin/yubikey-lock

# -------------------------------------------------------------------
# 4. Kernel / sysctl tuning
# -------------------------------------------------------------------
log "Applying kernel tuning..."

mkdir -p /etc/sysctl.d
cat > /etc/sysctl.d/99-storagenode.conf << 'SYSCTL'
# Minimize swapping — 192GB RAM, no swap partition
vm.swappiness=1
vm.vfs_cache_pressure=50

# Dirty page tuning for mixed SATA + NVMe
# Conservative ratios since SATA is slower
vm.dirty_ratio=15
vm.dirty_background_ratio=5
vm.dirty_expire_centisecs=3000
vm.dirty_writeback_centisecs=500

# NUMA balancing for 3950X dual-CCD
kernel.numa_balancing=1

# Network tuning for object storage serving
net.core.somaxconn=65535
net.core.netdev_max_backlog=65536
net.core.rmem_max=16777216
net.core.wmem_max=16777216
net.core.rmem_default=1048576
net.core.wmem_default=1048576
net.ipv4.tcp_rmem=4096 1048576 16777216
net.ipv4.tcp_wmem=4096 1048576 16777216
net.ipv4.tcp_max_syn_backlog=65535
net.ipv4.ip_local_port_range=1024 65535
net.ipv4.tcp_tw_reuse=1
net.ipv4.tcp_fin_timeout=15
net.ipv4.tcp_slow_start_after_idle=0

# Increase inotify for Docker
fs.inotify.max_user_watches=524288
fs.inotify.max_user_instances=512

# File descriptor limits
fs.file-max=2097152
SYSCTL

# -------------------------------------------------------------------
# 6. Service configuration
# -------------------------------------------------------------------
log "Configuring services..."

# Enable community repo
sed -i 's|#\(.*community\)|\1|' /etc/apk/repositories

# Enable services — Docker is NOT auto-started (needs LUKS unlock first)
# rc-update add docker default  — INTENTIONALLY OMITTED
rc-update add chronyd default
rc-update add sshd default
rc-update add lvm boot
rc-update add irqbalance default
rc-update add haveged boot
rc-update add nftables default
rc-update add local default

# -------------------------------------------------------------------
# 7. SSH hardening — YubiKey FIDO2 keys only
# -------------------------------------------------------------------
log "Hardening SSH for YubiKey FIDO2 authentication..."

cat > /etc/ssh/sshd_config << 'SSHD'
# Storage Node SSH — YubiKey FIDO2 only
Port 22
ListenAddress 0.0.0.0
Protocol 2

# Host keys — only Ed25519
HostKey /etc/ssh/ssh_host_ed25519_key

# Authentication — public key only, FIDO2 (ed25519-sk, ecdsa-sk)
PubkeyAuthentication yes
PubkeyAcceptedKeyTypes sk-ssh-ed25519@openssh.com,sk-ecdsa-sha2-nistp256@openssh.com,ssh-ed25519

# Disable everything else
PasswordAuthentication no
ChallengeResponseAuthentication no
KbdInteractiveAuthentication no
UsePAM no
PermitRootLogin prohibit-password
AuthenticationMethods publickey

# Security
PermitEmptyPasswords no
X11Forwarding no
AllowTcpForwarding yes
AllowAgentForwarding no
PrintMotd yes
MaxAuthTries 3
MaxSessions 5
LoginGraceTime 30

# Keepalive for headless management
ClientAliveInterval 60
ClientAliveCountMax 3

# Logging
LogLevel VERBOSE
SSHD

# Remove other host key types
rm -f /etc/ssh/ssh_host_rsa_key* /etc/ssh/ssh_host_ecdsa_key* /etc/ssh/ssh_host_dsa_key*
# Regenerate Ed25519 if missing
[ -f /etc/ssh/ssh_host_ed25519_key ] || ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N ""

# Install authorized_keys from nanny enrollment
mkdir -p /root/.ssh
chmod 700 /root/.ssh
ENROLL_AUTHKEYS
# Inject actual keys (build-iso.sh replaces this marker)
chmod 600 /root/.ssh/authorized_keys

# MOTD showing unlock status
cat > /etc/motd << 'MOTD'

  ┌──────────────────────────────────────────┐
  │  rig.fhirworx.io — LUKS Encrypted       │
  │                                          │
  │  Auth chain: YubiKey → SSH → LUKS        │
  │              YubiKey → CF Access → RustFS│
  │                                          │
  │  Volumes locked at boot.                 │
  │  Run: yubikey-unlock                     │
  │                                          │
  │  Commands:                               │
  │    yubikey-unlock  — decrypt + start all │
  │    yubikey-lock    — stop + encrypt      │
  │    rw / ro         — toggle root rw      │
  │    sys-update      — update Alpine       │
  └──────────────────────────────────────────┘

MOTD

# Docker daemon config
mkdir -p /etc/docker
cat > /etc/docker/daemon.json << 'DOCKER'
{
    "storage-driver": "overlay2",
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "10m",
        "max-file": "3"
    },
    "default-address-pools": [
        {"base": "172.17.0.0/16", "size": 24}
    ],
    "live-restore": true,
    "userland-proxy": false,
    "no-new-privileges": true
}
DOCKER

# Basic firewall
mkdir -p /etc/nftables.d
cat > /etc/nftables.d/server.nft << 'NFT'
table inet filter {
    chain input {
        type filter hook input priority 0; policy drop;

        # Established/related
        ct state established,related accept

        # Loopback
        iif lo accept

        # ICMP
        ip protocol icmp accept
        ip6 nexthdr icmpv6 accept

        # SSH
        tcp dport 22 accept

        # RustFS — LAN only (tunnel handles external access)
        # Uncomment if you need direct LAN access to S3/console:
        # tcp dport { 9000, 9001 } accept

        # Cloudflare WARP (UDP, outbound-initiated but needs return)
        # WARP uses WireGuard on UDP 2408
        udp dport 2408 accept
    }

    chain forward {
        type filter hook forward priority 0; policy accept;
        # Docker manages its own forward rules
    }

    chain output {
        type filter hook output priority 0; policy accept;
    }
}
NFT

# -------------------------------------------------------------------
# 8. Deploy RustFS + Cloudflared via Docker Compose
# -------------------------------------------------------------------
log "Writing Docker Compose stack..."

mkdir -p /opt/rustfs
mkdir -p /opt/cloudflared
mkdir -p /opt/certs

cat > /opt/rustfs/docker-compose.yml << 'COMPOSE'
services:
  rustfs:
    image: rustfs/rustfs:latest
    container_name: rustfs
    restart: always
    ports:
      - "127.0.0.1:9000:9000"
      - "127.0.0.1:9001:9001"
    volumes:
      - /data/rustfs:/data
    environment:
      RUSTFS_ROOT_USER: "admin"
      RUSTFS_ROOT_PASSWORD: "CHANGEME-use-a-real-password"
      RUSTFS_VOLUMES: "/data"
      RUSTFS_BROWSER_REDIRECT_URL: "https://console.rig.fhirworx.io"
      RUSTFS_SERVER_URL: "https://s3.rig.fhirworx.io"
    command: server /data --console-address ":9001"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 10s
    networks:
      - internal

  cloudflared:
    image: cloudflare/cloudflared:latest
    container_name: cloudflared
    restart: always
    command: tunnel run
    environment:
      TUNNEL_TOKEN: "${TUNNEL_TOKEN}"
    depends_on:
      rustfs:
        condition: service_healthy
    networks:
      - internal

  warp:
    image: caomingjun/warp:latest
    container_name: warp-svc
    restart: always
    cap_add:
      - NET_ADMIN
      - SYS_MODULE
    sysctls:
      - net.ipv6.conf.all.disable_ipv6=0
      - net.ipv4.conf.all.src_valid_mark=1
    volumes:
      - /opt/cloudflared/warp-data:/var/lib/cloudflare-warp
    ports:
      - "127.0.0.1:1080:1080"
    networks:
      - internal

networks:
  internal:
    driver: bridge
COMPOSE

# Env file for tunnel token (populated during setup)
cat > /opt/rustfs/.env << 'ENV'
# Paste your Cloudflare Tunnel token here
# Get it from: https://one.dash.cloudflare.com → Networks → Tunnels → Create
TUNNEL_TOKEN=PASTE_YOUR_TOKEN_HERE
ENV
chmod 600 /opt/rustfs/.env

# -------------------------------------------------------------------
# 8b. Cloudflare Origin Certificate setup script
# -------------------------------------------------------------------
cat > /opt/certs/setup-certs.sh << 'CERTSCRIPT'
#!/bin/sh
#
# Generate and install Cloudflare Origin CA certificate for rig.fhirworx.io
#
# This cert is used for:
#   - Cloudflare Full (Strict) SSL between edge and origin
#   - Direct HTTPS access on LAN
#
# You have THREE options (run the one that fits):
#
# ── OPTION 1: Cloudflare Origin CA (recommended with tunnel) ──────
#
# 1. Go to: https://dash.cloudflare.com → fhirworx.io → SSL/TLS → Origin Server
# 2. Click "Create Certificate"
# 3. Key type: ECDSA (or RSA 2048)
# 4. Hostnames: rig.fhirworx.io, *.rig.fhirworx.io
# 5. Validity: 15 years (origin certs are only trusted by Cloudflare)
# 6. Copy the certificate PEM → paste into /opt/certs/origin.pem
# 7. Copy the private key PEM → paste into /opt/certs/origin-key.pem
#
# Then:
#   chmod 600 /opt/certs/origin-key.pem
#   chmod 644 /opt/certs/origin.pem
#
# ── OPTION 2: Let's Encrypt via DNS challenge (for LAN HTTPS too) ─
#
# Requires Cloudflare API token with Zone:DNS:Edit permission.
#
#   export CF_API_TOKEN="your-cloudflare-api-token"
#
#   certbot certonly \
#     --dns-cloudflare \
#     --dns-cloudflare-credentials /opt/certs/cf-credentials.ini \
#     -d "rig.fhirworx.io" \
#     -d "*.rig.fhirworx.io" \
#     --preferred-challenges dns-01 \
#     --non-interactive \
#     --agree-tos \
#     -m your@email.com
#
#   # Certs land in /etc/letsencrypt/live/rig.fhirworx.io/
#   # Auto-renews via: certbot renew
#
# ── OPTION 3: No origin cert needed ──────────────────────────────
#
# If you ONLY access RustFS through Cloudflare Tunnel, you don't need
# an origin cert at all. The tunnel encrypts traffic end-to-end without
# a traditional TLS certificate on the origin. This is the simplest path.
# Cloudflare Tunnel → http://rustfs:9000 (plaintext inside Docker network)
#
# This is what the default docker-compose.yml does.
#

echo "See this script's comments for setup instructions."
echo "For most users: Option 3 (no cert needed with tunnel) is correct."
CERTSCRIPT
chmod +x /opt/certs/setup-certs.sh

# Certbot Cloudflare credentials template
cat > /opt/certs/cf-credentials.ini.template << 'CFCREDS'
# Cloudflare API token for certbot DNS challenge
# Create at: https://dash.cloudflare.com/profile/api-tokens
# Permissions needed: Zone → DNS → Edit (for fhirworx.io zone only)
dns_cloudflare_api_token = YOUR_API_TOKEN_HERE
CFCREDS

# -------------------------------------------------------------------
# 9. Nightly backup script + cron
# -------------------------------------------------------------------
log "Installing nightly backup..."

cat > /usr/local/bin/backup-nightly << 'BACKUP'
#!/bin/sh
#
# Nightly backup: RustFS data + Docker volumes → HDD
#
# Runs at 2 AM via cron. Only operates if volumes are unlocked.
# Uses rsync for incremental backups with 7-day rotation.
#
set -e

LOGFILE="/var/log/backup-nightly.log"
BACKUP_DIR="/backup"
DATE=$(date +%Y-%m-%d)
RETAIN_DAYS=7

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" >> "$LOGFILE"; }

# Check volumes are unlocked
if [ ! -b /dev/mapper/backup-crypt ]; then
    log "SKIP: backup-crypt not unlocked"
    exit 0
fi

if ! mountpoint -q "$BACKUP_DIR" 2>/dev/null; then
    log "SKIP: /backup not mounted"
    exit 0
fi

if ! mountpoint -q /data/rustfs 2>/dev/null; then
    log "SKIP: /data/rustfs not mounted"
    exit 0
fi

log "START: nightly backup"

# Create dated snapshot directory
SNAP_DIR="${BACKUP_DIR}/snapshots/${DATE}"
LATEST_LINK="${BACKUP_DIR}/latest"
mkdir -p "$SNAP_DIR"

# Backup RustFS data (incremental via hardlinks to previous backup)
log "Backing up RustFS data..."
if [ -L "$LATEST_LINK" ]; then
    rsync -a --delete \
        --link-dest="$LATEST_LINK/rustfs" \
        /data/rustfs/ \
        "${SNAP_DIR}/rustfs/"
else
    rsync -a --delete \
        /data/rustfs/ \
        "${SNAP_DIR}/rustfs/"
fi

# Backup Docker volumes (named volumes only)
log "Backing up Docker volumes..."
mkdir -p "${SNAP_DIR}/docker-volumes"
for vol in $(docker volume ls -q 2>/dev/null); do
    vol_path=$(docker volume inspect --format '{{ .Mountpoint }}' "$vol" 2>/dev/null || true)
    if [ -n "$vol_path" ] && [ -d "$vol_path" ]; then
        if [ -L "$LATEST_LINK" ]; then
            rsync -a --delete \
                --link-dest="$LATEST_LINK/docker-volumes/${vol}" \
                "${vol_path}/" \
                "${SNAP_DIR}/docker-volumes/${vol}/"
        else
            rsync -a --delete \
                "${vol_path}/" \
                "${SNAP_DIR}/docker-volumes/${vol}/"
        fi
    fi
done

# Backup compose files and configs
log "Backing up configs..."
mkdir -p "${SNAP_DIR}/config"
cp -a /opt/rustfs/ "${SNAP_DIR}/config/rustfs/" 2>/dev/null || true
vgcfgbackup -f "${SNAP_DIR}/config/rustfs-vg.cfg" rustfs-vg 2>/dev/null || true
cp /etc/fstab "${SNAP_DIR}/config/" 2>/dev/null || true
cp /etc/crypttab "${SNAP_DIR}/config/" 2>/dev/null || true

# Update latest symlink
rm -f "$LATEST_LINK"
ln -s "$SNAP_DIR" "$LATEST_LINK"

# Rotate old snapshots
log "Rotating backups older than ${RETAIN_DAYS} days..."
find "${BACKUP_DIR}/snapshots" -maxdepth 1 -type d -mtime +${RETAIN_DAYS} -exec rm -rf {} \; 2>/dev/null || true

# Disk usage
USED=$(df -h "$BACKUP_DIR" | tail -1 | awk '{print $3}')
AVAIL=$(df -h "$BACKUP_DIR" | tail -1 | awk '{print $4}')
log "DONE: backup complete (used: ${USED}, available: ${AVAIL})"
BACKUP
chmod +x /usr/local/bin/backup-nightly

# Cron job — runs at 2 AM daily (only works when volumes are unlocked)
mkdir -p /etc/crontabs
cat >> /etc/crontabs/root << 'CRON'

# Nightly backup to HDD — 2 AM
0 2 * * * /usr/local/bin/backup-nightly
CRON

# -------------------------------------------------------------------
# 10. Read-only root helper scripts
# -------------------------------------------------------------------
log "Installing helper scripts..."

# Remount helper
cat > /usr/local/bin/rw << 'RW'
#!/bin/sh
# Temporarily remount root read-write for system changes
mount -o remount,rw,discard /
echo "Root is now read-write. Run 'ro' when done."
RW
chmod +x /usr/local/bin/rw

cat > /usr/local/bin/ro << 'RO'
#!/bin/sh
# Remount root back to read-only
sync
fstrim / 2>/dev/null || true
mount -o remount,ro,discard /
echo "Root is now read-only."
RO
chmod +x /usr/local/bin/ro

# System update helper
cat > /usr/local/bin/sys-update << 'UPDATE'
#!/bin/sh
set -e
echo "Remounting root read-write..."
mount -o remount,rw /
echo "Updating packages..."
apk update && apk upgrade
echo "Rebuilding initramfs..."
update-initramfs 2>/dev/null || true
sync
echo "Remounting root read-only..."
mount -o remount,ro /
echo "Done. Reboot if kernel was updated."
UPDATE
chmod +x /usr/local/bin/sys-update

# -------------------------------------------------------------------
# 10. Finalize
# -------------------------------------------------------------------

# LUKS volumes are already open from setup — mount them
log "Mounting encrypted volumes..."
mount /var/lib/docker
mount /var/cache
mount /data/rustfs
mount /backup

log "Starting Docker..."
service docker start

log "Pulling RustFS image..."
cd /opt/rustfs && docker compose pull

log "Running initial backup..."
/usr/local/bin/backup-nightly || warn "Initial backup failed — will retry at 2 AM"

# Now lock everything — next boot requires YubiKey
log "Stopping Docker and locking volumes..."
service docker stop
umount /backup /data/rustfs /var/cache /var/lib/docker
cryptsetup close backup-crypt
cryptsetup close rustfs-crypt
cryptsetup close cache-crypt
cryptsetup close docker-crypt

log ""
log "=== SETUP COMPLETE ==="
log ""
log "IMPORTANT — Before rebooting, add your YubiKey SSH public keys:"
log ""
log "  On your WORKSTATION (not this server), with YubiKey inserted:"
log "    ssh-keygen -t ed25519-sk -C \"yubikey-1\" -f ~/.ssh/id_yubikey1"
log "    ssh-keygen -t ed25519-sk -C \"yubikey-2\" -f ~/.ssh/id_yubikey2"
log ""
log "  Then on this server (root is still rw during setup):"
log "    nano /root/.ssh/authorized_keys"
log "    # Paste both .pub key lines"
log ""
log "  Then lock it down:"
log "    ro"
log ""
log "After reboot, the boot sequence is:"
log "  1. Server boots (read-only root, no data accessible)"
log "  2. SSH in with YubiKey:  ssh -i ~/.ssh/id_yubikey1 root@<ip>"
log "  3. Unlock volumes:       sudo yubikey-unlock"
log "  4. Start RustFS:         cd /opt/rustfs && docker compose up -d"
log ""
log "Commands:"
log "  yubikey-unlock  — decrypt + mount + start Docker"
log "  yubikey-lock    — stop Docker + unmount + encrypt"
log "  rw / ro         — toggle root read-write / read-only"
log "  sys-update      — update Alpine packages"

SETUP
chmod +x "${WORKDIR}/aports/scripts/storagenode-overlay/storage-setup.sh"

# -------------------------------------------------------------------
# Inject enrolled hardware fingerprints into storage-setup.sh
# -------------------------------------------------------------------
echo "[4b/5] Injecting hardware fingerprints..."

SETUP_SCRIPT="${WORKDIR}/aports/scripts/storagenode-overlay/storage-setup.sh"

# Replace the authorized_keys marker with actual keys from nanny
AUTH_KEYS_CONTENT=$(cat "$(pwd)/authorized_keys")
sed -i "/^ENROLL_AUTHKEYS$/r /dev/stdin" "$SETUP_SCRIPT" << INJECT_KEYS
cat > /root/.ssh/authorized_keys << 'AUTHKEYS_INJECTED'
${AUTH_KEYS_CONTENT}
AUTHKEYS_INJECTED
INJECT_KEYS
sed -i '/^ENROLL_AUTHKEYS$/d' "$SETUP_SCRIPT"

# Copy drive-resolver.sh into the overlay (embedded in ISO)
cp "$(pwd)/drive-resolver.sh" "${WORKDIR}/aports/scripts/storagenode-overlay/drive-resolver.sh"
cp "$(pwd)/drives.conf" "${WORKDIR}/aports/scripts/storagenode-overlay/drives.conf"
cp "$(pwd)/yubikeys.conf" "${WORKDIR}/aports/scripts/storagenode-overlay/yubikeys.conf"
cp "$(pwd)/cloudflare.conf" "${WORKDIR}/aports/scripts/storagenode-overlay/cloudflare.conf"

# Copy SSH public keys
cp -r "$(pwd)/ssh-keys" "${WORKDIR}/aports/scripts/storagenode-overlay/ssh-keys"

# Copy certs if they exist
if [ -d "$(pwd)/certs" ]; then
    cp -r "$(pwd)/certs" "${WORKDIR}/aports/scripts/storagenode-overlay/certs"
fi

# Inject Cloudflare and RustFS values into storage-setup.sh
# Replace the placeholder password and tunnel token with real values
RUSTFS_USER_VAL=$(grep "^RUSTFS_ROOT_USER=" "$(pwd)/cloudflare.conf" | cut -d= -f2-)
RUSTFS_PASS_VAL=$(grep "^RUSTFS_ROOT_PASSWORD=" "$(pwd)/cloudflare.conf" | cut -d= -f2-)
TUNNEL_TOKEN_VAL=$(grep "^CF_TUNNEL_TOKEN=" "$(pwd)/cloudflare.conf" | cut -d= -f2-)
WARP_ENABLED=$(grep "^CF_WARP_ENABLED=" "$(pwd)/cloudflare.conf" | cut -d= -f2-)

# Inject into docker-compose section of storage-setup.sh
if [ -n "$RUSTFS_USER_VAL" ]; then
    sed -i "s|RUSTFS_ROOT_USER: \"admin\"|RUSTFS_ROOT_USER: \"${RUSTFS_USER_VAL}\"|" "$SETUP_SCRIPT"
fi
if [ -n "$RUSTFS_PASS_VAL" ]; then
    sed -i "s|RUSTFS_ROOT_PASSWORD: \"CHANGEME-use-a-real-password\"|RUSTFS_ROOT_PASSWORD: \"${RUSTFS_PASS_VAL}\"|" "$SETUP_SCRIPT"
fi
if [ -n "$TUNNEL_TOKEN_VAL" ]; then
    sed -i "s|TUNNEL_TOKEN=PASTE_YOUR_TOKEN_HERE|TUNNEL_TOKEN=${TUNNEL_TOKEN_VAL}|" "$SETUP_SCRIPT"
fi

# Remove WARP service from compose if disabled
if [ "$WARP_ENABLED" = "false" ]; then
    # Remove the warp service block from the compose file in the setup script
    sed -i '/^  warp:/,/^  [a-z]/{ /^  warp:/d; /^    /d; }' "$SETUP_SCRIPT" || true
fi

# Inject static network config
STATIC_IP=$(grep "^NET_STATIC_IP=" "$(pwd)/cloudflare.conf" 2>/dev/null | cut -d= -f2-)
GATEWAY=$(grep "^NET_GATEWAY=" "$(pwd)/cloudflare.conf" 2>/dev/null | cut -d= -f2-)
DNS_SERVER=$(grep "^NET_DNS=" "$(pwd)/cloudflare.conf" 2>/dev/null | cut -d= -f2-)

if [ -n "$STATIC_IP" ] && [ -n "$GATEWAY" ]; then
    # Append network setup to the end of storage-setup.sh (before the final SETUP marker)
    cat >> "$SETUP_SCRIPT" << NETCONF

# -------------------------------------------------------------------
# Network: Static IP configuration
# -------------------------------------------------------------------
log "Configuring static IP: ${STATIC_IP}..."

cat > /etc/network/interfaces << 'NETIF'
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet static
    address ${STATIC_IP}
    gateway ${GATEWAY}
NETIF
echo "nameserver ${DNS_SERVER:-1.1.1.1}" > /etc/resolv.conf
log "Static IP configured. Will take effect on next reboot."
NETCONF
fi

echo "  Injected $(echo "$AUTH_KEYS_CONTENT" | grep -c "^sk-\|^ssh-") SSH public keys"
echo "  Embedded drive-resolver.sh with $(grep -c "^DRIVE_" "$(pwd)/drives.conf") drive fingerprints"
echo "  Embedded yubikeys.conf with $(grep -c "^YUBIKEY_" "$(pwd)/yubikeys.conf") YubiKey fingerprints"
echo "  Embedded cloudflare.conf with tunnel token and credentials"
[ -n "$STATIC_IP" ] && echo "  Injected static IP: ${STATIC_IP} via ${GATEWAY}"

# -------------------------------------------------------------------
# Build the ISO
# -------------------------------------------------------------------
echo "[5/5] Building ISO..."
cd "${WORKDIR}/aports/scripts"

sh mkimage.sh \
    --tag "${ISO_TAG}" \
    --outdir "${OUTDIR}" \
    --arch "${ARCH}" \
    --repository "${MAIN_REPO}" \
    --repository "${COMMUNITY_REPO}" \
    --profile storagenode

echo ""
echo "=== ISO built ==="
echo "Output: ${OUTDIR}/"
ls -lh "${OUTDIR}"/*.iso 2>/dev/null || echo "(check ${OUTDIR} for output)"
echo ""
echo "Write to USB with:"
echo "  dd if=${OUTDIR}/alpine-storagenode-*.iso of=/dev/sdX bs=4M status=progress"
echo ""
echo "After booting:"
echo "  1. Run: setup-alpine -f auto-setup.conf"
echo "  2. Reboot into installed system"
echo "  3. Run: storage-setup.sh"