#!/usr/bin/env bash
# =============================================================================
# fermihdi-configure — build a cluster.yaml entry for one host
# =============================================================================
# Asks what this host should run, works out CPU placement from its real
# topology, and writes a host block for cluster.yaml.
#
# One host per run. Multi-host installs run this on each host (the Ansible
# layer does that for you) and concatenate the results.
#
# Usage:
#   ./fermihdi-configure                    interactive, writes ./cluster.yaml
#   ./fermihdi-configure -o host.yaml       write elsewhere
#   ./fermihdi-configure --facts f.tsv      use saved facts instead of probing
#   ./fermihdi-configure --append cluster.yaml   add this host to an existing file
#
# Changes nothing on the host. It only writes a file; fermihdi-prepare applies it.
# =============================================================================
set -uo pipefail

HERE=$(cd "$(dirname "$0")" && pwd)
FACTS_TMP=0
OUT="cluster.yaml"
FACTS=""
APPEND=""

while [ $# -gt 0 ]; do
    case "$1" in
        -o|--output) OUT="${2:-}"; shift 2 ;;
        --facts)     FACTS="${2:-}"; shift 2 ;;
        --append)    APPEND="${2:-}"; OUT="${2:-}"; shift 2 ;;
        -h|--help)   sed -n '2,20p' "$0" | sed 's/^# \?//'; exit 0 ;;
        *) echo "unknown argument: $1" >&2; exit 2 ;;
    esac
done

# ── Core floors ──────────────────────────────────────────────────────────────
# Below these the resource controller refuses to initialise and the node exits
# at startup, so they are enforced here rather than discovered in production.
#   scc  6  = transport 2 + kv 2 + hash 2
#   sn   9  = storage + kv shared, plus transport, hash and filter
floor_for() {
    case "$1" in
        scc) echo 6 ;; sn) echo 9 ;; ingester) echo 4 ;;
        graphql) echo 2 ;; mcp) echo 2 ;; proxy) echo 1 ;; dms) echo 1 ;;
        *) echo 1 ;;
    esac
}
# Engine roles need 2 MB pages; the SCC and SN also need 1 GB.
needs_2m() { case "$1" in scc|sn|ingester|mcp|graphql) return 0 ;; *) return 1 ;; esac; }
needs_1g() { case "$1" in scc|sn) return 0 ;; *) return 1 ;; esac; }
dpdk_capable() { case "$1" in scc|sn|ingester|graphql|mcp) return 0 ;; *) return 1 ;; esac; }
takes_storage() { [ "$1" = sn ]; }

# Prompts go to stderr so stdout stays clean. Input comes from the terminal when
# there is one and from stdin otherwise, so the script can be driven by a here-doc
# in a test or by Ansible without a pseudo-tty.
if [ -r /dev/tty ] && exec 9<>/dev/tty 2>/dev/null; then TTY_IN=9; else TTY_IN=0; fi

# ── Memory model ──────────────────────────────────────────────────────────────
#
# The two hugepage pools have different consumers and must be sized separately.
# Sizing one from the other (an earlier bug here) over-reserves 1 GB pages that
# only SPDK can use and starves the engine pool that everything else needs.
#
#   2 MB pool  Engine heap and the DPDK mbuf pools. Every DPDK-capable role
#              draws from it. libfermihdi_transport's DpdkEngine defaults
#              FERMIHDI_SEASTAR_MEMORY to "512M" -- 256 pages -- and the SN is
#              the only role that overrides it: sn_hot_path multiplies
#              FAKE_SN_TX_MEMORY_PER_SHARD_MB (4096) by the transmit shard
#              count. Two shards is the common case, hence 8192.
#
#   1 GB pool  SPDK NVMe DMA only, and pinned to /dev/hugepages by
#              StorageEngine.cpp so EAL never touches the 2 MB networking pool.
#              StorageEngine's own diagnostic says "minimum 2, recommend 4 per
#              NVMe device" -- it scales with DEVICES, not with instances. An
#              SN whose devices are all kernel-path (SATA/SAS/FC/iSCSI via
#              bdev_aio) needs NONE: that path sets opts.no_huge and never
#              DMAs. The SCC's EAL takes -m 2048, so 2 pages.

# Engine budget in MB per instance -- the figure the 2 MB pool must cover.
engine_mb_for() {
    case "$1" in
        sn)          echo 8192 ;;   # 4096 per shard x 2 transmit shards
        scc|ingester|graphql|mcp) echo 512 ;;   # transport's "512M" default
        *)           echo 0 ;;
    esac
}
# 2 MB pages for an engine budget, plus 44 pages (88 MB) for the mbuf pools and
# EAL's own overhead. A 512M budget lands on 300, which is the figure the
# deployment notes have always quoted.
hp2m_for() { echo $(( $(engine_mb_for "$1") / 2 + 44 )); }
# 1 GB pages per instance. The SN's real need is per NVMe device and is added
# separately; this is the fixed part.
hp1g_fixed_for() {
    case "$1" in
        scc) echo 2 ;;              # EAL -m 2048
        *)   echo 0 ;;
    esac
}
HP1G_PER_NVME=4                     # StorageEngine: "recommend 4 per NVMe device"
# Kernel, page cache and everything not FermiHDI (4 GB), plus process stack and
# heap outside the engine arena (1 GB).
OS_HEADROOM_MB=5120

say()  { printf '%s\n' "$*" >&2; }
warn() { printf '  ! %s\n' "$*" >&2; }
ask() { # ask <prompt> <default> -> stdout
    local p="$1" d="${2:-}" a
    if [ -n "$d" ]; then printf '%s [%s]: ' "$p" "$d" >&2; else printf '%s: ' "$p" >&2; fi
    read -r a <&$TTY_IN || a=""
    printf '%s' "${a:-$d}"
}
ask_yn() { # ask_yn <prompt> <default y|n>
    local a; a=$(ask "$1 (y/n)" "$2"); case "$a" in [yY]*) return 0 ;; *) return 1 ;; esac
}
ask_int() {
    local a; while :; do
        a=$(ask "$1" "${2:-}")
        case "$a" in (''|*[!0-9]*) warn "enter a number" ;; (*) printf '%s' "$a"; return ;; esac
    done
}
# A path to a file that has to exist. Empty means "I do not have one", which is
# a legitimate answer to every question that uses this -- so a typo has to be
# distinguishable from a decision, and the only way to tell them apart is to
# check and ask again.
ask_file() { # ask_file <prompt> -> stdout (absolute path, or empty)
    local a; while :; do
        a=$(ask "$1")
        [ -z "$a" ] && return
        case "$a" in "~"/*) a="$HOME/${a#\~/}" ;; esac
        if [ -r "$a" ]; then
            # Stored absolute: the file is read where the deploy runs from, which
            # is not necessarily the directory this was answered in.
            printf '%s' "$(cd "$(dirname "$a")" && pwd)/$(basename "$a")"
            return
        fi
        warn "cannot read $a — enter a path, or leave it empty if you do not have one"
    done
}

# ── Facts ────────────────────────────────────────────────────────────────────
if [ -z "$FACTS" ]; then
    [ -x "$HERE/fermihdi-discover" ] || { echo "fermihdi-discover not found beside this script" >&2; exit 1; }
    FACTS=$(mktemp); FACTS_TMP=1; trap 'rm -f "$FACTS"' EXIT
    "$HERE/fermihdi-discover" --format=tsv > "$FACTS" || exit 1
fi

HOSTNAME_=$(awk -F'\t' '$1=="host"{print $2; exit}' "$FACTS")
LOGICAL=$(awk -F'\t' '$1=="cpu"{print $2; exit}' "$FACTS")
TPC=$(awk -F'\t' '$1=="cpu"{print $5; exit}' "$FACTS")
IOMMU=$(awk -F'\t' '$1=="iommu"{print $2; exit}' "$FACTS")
MEM_TOTAL_MB=$(( $(awk -F'\t' '$1=="memory"{print $2; exit}' "$FACTS" 2>/dev/null || echo 0) / 1024 ))
[ -z "$LOGICAL" ] && { echo "could not read CPU facts" >&2; exit 1; }

say ""
say "  Host        $HOSTNAME_"
say "  Logical CPUs $LOGICAL   threads/core $TPC   IOMMU ${IOMMU:-unknown}"
say "  Memory       ${MEM_TOTAL_MB} MB"
say ""

# ── CPU pool ─────────────────────────────────────────────────────────────────
# Cores 0-1 and their SMT siblings stay with the OS and the container runtime
# shim. If DPDK cores appear in the container cpuset the shim can preempt
# poll-mode threads, which is the eviction problem the SN guide documents.
OS_CORES="0-1"
if [ "${TPC:-1}" -gt 1 ]; then
    sib=$(awk -F'\t' '$1=="sibling" && ($2==0 || $2==1){print $3}' "$FACTS" | sort -u | tr '\n' ',' | sed 's/,$//')
    [ -n "$sib" ] && OS_CORES="$sib"
fi
# Expand a cpuset spec to a sorted list.
expand() { local out=() p a b; IFS=',' read -ra parts <<<"$1"
    for p in "${parts[@]}"; do
        case "$p" in *-*) a=${p%%-*}; b=${p##*-}; for ((i=a;i<=b;i++)); do out+=("$i"); done ;;
                     *) out+=("$p") ;; esac
    done; printf '%s\n' "${out[@]}"; }
# Collapse a sorted list back to ranges.
collapse() { awk 'NR==1{s=$1;p=$1;next} $1==p+1{p=$1;next} {printf "%s%s,",(s==p?s:s"-"p),"";s=$1;p=$1} END{printf "%s",(s==p?s:s"-"p)}'; }

mapfile -t OS_LIST < <(expand "$OS_CORES" | sort -n -u)
AVAIL=()
for ((c=0;c<LOGICAL;c++)); do
    skip=0; for o in "${OS_LIST[@]}"; do [ "$c" = "$o" ] && skip=1 && break; done
    [ "$skip" = 0 ] && AVAIL+=("$c")
done
say "  Reserved for the OS and container runtime: $OS_CORES"
say "  Available for FermiHDI: ${#AVAIL[@]} logical CPUs"
say ""

declare -A CLAIMED       # core -> 1 once handed to an instance
declare -A CPU_NODE      # core -> NUMA node
TAKEN=""
TAKEN_NODE=""

# Build the core -> node map. A cpuset that straddles two nodes makes every
# buffer access a cross-socket trip, and a poll-mode thread pays that on every
# iteration. Instances are therefore confined to one node.
declare -A NODE_MEM
while IFS=$'\t' read -r _ node cpulist nodemem _hp2 _hp1; do
    for c in $(expand "$cpulist"); do CPU_NODE[$c]=$node; done
    # Memory local to this node. Older facts files have no such column, so
    # treat a missing value as unknown rather than as zero.
    NODE_MEM[$node]=${nodemem:-0}
done < <(awk -F'\t' '$1=="numa"{print}' "$FACTS")
NUMA_NODES=$(awk -F'\t' '$1=="numa"{print $2}' "$FACTS" | sort -n -u | tr '\n' ' ')
[ -z "$NUMA_NODES" ] && NUMA_NODES=0

sibling_list_of() { awk -F'\t' -v n="$1" '$1=="sibling" && $2==n{print $3; exit}' "$FACTS"; }

is_available() { local c=$1 a; for a in "${AVAIL[@]}"; do [ "$a" = "$c" ] && return 0; done; return 1; }

# take_cores <count> -> sets TAKEN.
#
# Never call this inside $( ): CLAIMED must be updated in this shell or every
# instance is handed the same cores.
#
# Allocates whole physical cores. Claiming a core without its SMT sibling would
# let another instance land on the sibling, and a poll-mode thread sharing a
# physical core with other work is the stall this pinning exists to prevent.
# free_on_node <node> -> how many unclaimed cores that node still has
free_on_node() {
    local node=$1 n=0 c
    for c in "${AVAIL[@]}"; do
        [ -n "${CLAIMED[$c]:-}" ] && continue
        [ "${CPU_NODE[$c]:-0}" = "$node" ] && n=$((n+1))
    done
    echo "$n"
}

# take_cores <count> [preferred_node]
#
# Allocates whole physical cores from a SINGLE NUMA node, so no instance
# straddles a socket. When a preferred node is given — the node the instance's
# NVMe or NIC is attached to — it is used if it still has room, because a poll
# thread reaching across the interconnect to its own device is the worst case.
# Otherwise the emptiest node that can fit the request wins.
take_cores() {
    local want=$1 prefer="${2:-}" got=() c s node chosen=""
    TAKEN=""; TAKEN_NODE=""

    if [ -n "$prefer" ] && [ "$prefer" != "-1" ] && [ "$(free_on_node "$prefer")" -ge "$want" ]; then
        chosen=$prefer
    else
        local best=-1 bestfree=-1 f
        for node in $NUMA_NODES; do
            f=$(free_on_node "$node")
            [ "$f" -ge "$want" ] && [ "$f" -gt "$bestfree" ] && { best=$node; bestfree=$f; }
        done
        chosen=$best
    fi

    if [ "$chosen" = "-1" ] || [ -z "$chosen" ]; then
        # No single node can satisfy it. Refuse rather than quietly straddling:
        # a cpuset across sockets makes every buffer access a cross-interconnect
        # trip, which a poll-mode thread pays on every iteration.
        warn "cannot place this instance: it needs $want cores from ONE NUMA node."
        for node in $NUMA_NODES; do
            warn "  node $node has $(free_on_node "$node") free"
        done
        warn "Reduce the instance count on this host, or move it to a larger machine."
        warn "The cpuset is written as FIXME and validation will refuse to pass it."
        return 1
    fi
    if [ -n "$prefer" ] && [ "$prefer" != "-1" ] && [ "$chosen" != "$prefer" ]; then
        warn "device is on NUMA node $prefer but only node $chosen has room —"
        warn "this instance will reach across the interconnect for every I/O"
    fi

    for c in "${AVAIL[@]}"; do
        [ ${#got[@]} -ge "$want" ] && break
        [ -n "${CLAIMED[$c]:-}" ] && continue
        [ "${CPU_NODE[$c]:-0}" = "$chosen" ] || continue
        for s in $(expand "$(sibling_list_of "$c")" 2>/dev/null || echo "$c"); do
            [ -n "${CLAIMED[$s]:-}" ] && continue
            is_available "$s" || continue
            [ "${CPU_NODE[$s]:-0}" = "$chosen" ] || continue
            got+=("$s"); CLAIMED[$s]=1
        done
    done
    [ ${#got[@]} -lt "$want" ] && return 1
    TAKEN=$(printf '%s\n' "${got[@]}" | sort -n -u | collapse)
    TAKEN_NODE=$chosen
}

claimed_cpuset() {
    local out=() c
    for c in "${AVAIL[@]}"; do [ -n "${CLAIMED[$c]:-}" ] && out+=("$c"); done
    [ ${#out[@]} -eq 0 ] && return 0
    printf '%s\n' "${out[@]}" | sort -n -u | collapse
}

# ── Devices ──────────────────────────────────────────────────────────────────
USED_STORAGE=" "
USED_NICS=" "

# Eligible and not already assigned. Without the second half the same device is
# offered to a second instance, which the validator then rejects for a PCI
# conflict and a split IOMMU group.
list_storage() {
    awk -F'\t' -v used="$USED_STORAGE" '$1=="storage" && $10=="true" && index(used, " " $3 " ")==0 {print}' "$FACTS"
}
list_nics() {
    awk -F'\t' -v used="$USED_NICS" '$1=="nic" && $9=="true" && index(used, " " $3 " ")==0 {print}' "$FACTS"
}

add_vfio() { case "$VFIO_BIND" in *" $1 "*) ;; *) VFIO_BIND="$VFIO_BIND$1 " ;; esac; }

show_storage_menu() {
    local i=0 line
    say "  Available storage devices:"
    while IFS=$'\t' read -r _ name path size model kind pci grp drv elig why fs root nn; do
        i=$((i+1))
        local extra=""
        [ "$pci" != "-" ] && extra=" pci=$pci"
        [ "$grp" != "-" ] && extra="$extra iommu_group=$grp"
        [ "${nn:--1}" != "-1" ] && [ -n "${nn:-}" ] && extra="$extra numa=$nn"
        [ "$fs" = "true" ] && extra="$extra [HAS FILESYSTEM]"
        say "    $i) $path  $size  $model  ($kind)$extra"
        if [ "$kind" = "nvme" ] && [ "$drv" != "vfio-pci" ] && [ "$drv" != "-" ]; then
            say "        driver $drv -> vfio-pci will be required"
        fi
    done < <(list_storage)
    [ "$i" = 0 ] && say "    (none — every device is mounted or holds the root filesystem)"
}

pick_storage() { # pick_storage <instance> -> writes YAML device entries to stdout
    local iname="$1" picks n i line
    show_storage_menu
    picks=$(ask "  Devices for $iname (numbers, comma separated; blank for none)" "")
    [ -z "$picks" ] && return 0
    IFS=',' read -ra sel <<<"$picks"
    for n in "${sel[@]}"; do
        n=$(echo "$n" | tr -d ' ')
        line=$(list_storage | sed -n "${n}p")
        [ -z "$line" ] && { warn "no device $n"; continue; }
        IFS=$'\t' read -r _ name path size model kind pci grp drv elig why fs root nn <<<"$line"

        # Say it plainly, every time, before it is written down.
        say ""
        warn "$path ($size, $model) will be consumed by $iname."
        warn "The Storage Node writes its own layout over the WHOLE device."
        [ "$fs" = "true" ] && warn "This device currently has a filesystem. Its contents will be lost."
        if ! ask_yn "  Confirm $path is yours to destroy" "n"; then
            say "  skipped $path"
            continue
        fi

        if [ "$kind" = "nvme" ]; then
            # A local NVMe controller has to reach SPDK through vfio-pci.
            [ "$pci" = "-" ] && { warn "$path is NVMe but has no PCI address; skipping"; continue; }
            printf '              - { pci: "%s", driver: vfio, kind: nvme' "$pci"
            [ "$grp" != "-" ] && printf ', iommu_group: %s' "$grp"
            [ "${nn:--1}" != "-1" ] && printf ', numa_node: %s' "$nn"
            printf ' }\n'
            [ "${nn:--1}" != "-1" ] && DEV_NODE="$nn"
            add_vfio "$pci"
            NVME_COUNT=$(( NVME_COUNT + 1 ))
            INST_NVME=$(( INST_NVME + 1 ))
            [ "$drv" != "vfio-pci" ] && [ "$drv" != "-" ] && \
                say "  note: $pci will be switched from $drv to vfio-pci"
        else
            printf '              - { path: %s, driver: kernel, kind: %s }\n' "$path" "$kind"
        fi
        USED_STORAGE="$USED_STORAGE$path "
    done
}

pick_nic() { # pick_nic <instance> -> writes a YAML nic entry to stdout
    local iname="$1" i=0 line n
    say "  Available NICs for DPDK:"
    while IFS=$'\t' read -r _ ifn pci drv state addr grp isdef elig nn; do
        i=$((i+1)); say "    $i) $ifn  pci=$pci  driver=$drv  state=$state  ipv4=$addr  numa=${nn:--1}"
    done < <(list_nics)
    if [ "$i" = 0 ]; then
        say "    (none eligible — the default-route interface is never offered)"
        return 0
    fi
    n=$(ask "  NIC for $iname (number, blank for none)" "")
    [ -z "$n" ] && return 0
    line=$(list_nics | sed -n "${n}p")
    [ -z "$line" ] && { warn "no NIC $n"; return 0; }
    IFS=$'\t' read -r _ ifn pci drv state addr grp isdef elig nn <<<"$line"
    warn "$ifn ($pci) will be unbound from the kernel and given to DPDK."
    warn "It will lose its IP address and stop being usable by the OS."
    ask_yn "  Confirm $ifn is not needed by the operating system" "n" || return 0
    printf '            nic: { pci: "%s", driver: vfio, kind: nic' "$pci"
    [ "$grp" != "-" ] && printf ', iommu_group: %s' "$grp"
    [ "${nn:--1}" != "-1" ] && printf ', numa_node: %s' "$nn"
    printf ' }\n'
    [ "${nn:--1}" != "-1" ] && [ -z "$DEV_NODE" ] && DEV_NODE="$nn"
    add_vfio "$pci"
    USED_NICS="$USED_NICS$pci "
}

# ── Interview ────────────────────────────────────────────────────────────────
ROLES="scc sn ingester mcp graphql proxy dms"
declare -A COUNT DPDK

say "  Which roles should this host run? Enter 0 to skip a role."
say ""
TOTAL=0
for r in $ROLES; do
    n=$(ask_int "    $r" 0)
    COUNT[$r]=$n
    TOTAL=$((TOTAL+n))
done
[ "$TOTAL" = 0 ] && { say ""; say "  Nothing selected."; exit 0; }

say ""
for r in $ROLES; do
    [ "${COUNT[$r]}" -gt 0 ] || continue
    if dpdk_capable "$r"; then
        if ask_yn "  Use DPDK for $r? (no = POSIX networking)" "n"; then DPDK[$r]=true; else DPDK[$r]=false; fi
    else
        DPDK[$r]=false
    fi
done

# ── What the DMS is installed with ───────────────────────────────────────────
# Only asked of the host that runs the DMS, because these are the DMS's own
# materials rather than the cluster's. Paths, not contents: the files stay where
# they are and are read at deploy time, so nothing secret is written here.
#
# Every one of them is optional, and each has another way in:
#   the license      through the WebUX, which can also install the ICA and
#                    DMS-Core's identity certificate out of it -- the Day-0 flow
#   the ICA          through DMS-CA's /v1/license/apply, or that same Day-0 flow
#   the schema       through the WebUX, per deployment
# Answering here is the pre-provisioned path: everything in place before the
# first container starts.
DMS_LICENSE_FILE=""; DMS_ICA_CERT_FILE=""; DMS_ICA_KEY_FILE=""; DMS_SCHEMA_FILE=""
if [ "${COUNT[dms]:-0}" -gt 0 ]; then
    say ""
    say "  This host runs the DMS. If you have any of the following, name it and"
    say "  the install puts it in place; leave a line empty to skip it."
    say ""
    DMS_LICENSE_FILE=$(ask_file "    FermiHDI license file")
    DMS_ICA_CERT_FILE=$(ask_file "    DMS ICA certificate file (PEM)")
    if [ -n "$DMS_ICA_CERT_FILE" ]; then
        # The certificate on its own signs nothing. Asked immediately rather
        # than validated later, because the answer is on the screen in front of
        # whoever just found the certificate.
        while [ -z "$DMS_ICA_KEY_FILE" ]; do
            DMS_ICA_KEY_FILE=$(ask_file "    ... and its private key")
            [ -z "$DMS_ICA_KEY_FILE" ] && warn "the ICA certificate is unusable without its key"
        done
    fi
    DMS_SCHEMA_FILE=$(ask_file "    Dataset schema file (JSON)")
    if [ -n "$DMS_LICENSE_FILE" ] && [ -z "$DMS_ICA_CERT_FILE" ]; then
        say ""
        warn "A license alone does not give the DMS-CA an ICA to sign with."
        warn "Either name one here, or apply the license through the WebUX after"
        warn "the install -- that path extracts the ICA from the license itself."
    fi
fi

# Check the pool can satisfy the floors before asking anything else.
NEED=0
for r in $ROLES; do
    [ "${COUNT[$r]}" -gt 0 ] || continue
    f=$(floor_for "$r"); NEED=$((NEED + f * COUNT[$r]))
done
say ""
say "  Core floors require $NEED logical CPUs; ${#AVAIL[@]} are available."
if [ "$NEED" -gt "${#AVAIL[@]}" ]; then
    warn "This host cannot satisfy the floors. Reduce counts or use a larger machine."
    warn "Below a floor the resource controller throws and the node exits at startup."
    ask_yn "  Continue anyway and fix the cpusets by hand" "n" || exit 1
fi
say ""

VFIO_BIND=" "
BODY=$(mktemp)
# Only clean up the facts file if we generated it — --facts belongs to the caller.
trap 'rm -f "$BODY"; [ "${FACTS_TMP:-0}" = 1 ] && rm -f "$FACTS"' EXIT

WANT_2M=0; WANT_1G=0
NVME_COUNT=0   # drives the 1 GB pool: SPDK DMA scales per NVMe device
# Hugepages are reserved per NUMA node and a reactor allocates from its own
# node, so the requirement is tracked per node, not just as a host total.
declare -A NODE_2M NODE_1G
{
    printf '  - name: "%s"\n' "$HOSTNAME_"
    ADDRESS=$(ask "  Address Ansible should connect to" "$HOSTNAME_")
    printf '    address: "%s"\n' "$ADDRESS"
    printf '    ssh_user: "%s"\n' "$(ask "  SSH user" "root")"
    printf '    roles:\n'

    for r in $ROLES; do
        n=${COUNT[$r]}
        [ "$n" -gt 0 ] || continue
        needs_2m "$r" && WANT_2M=1
        needs_1g "$r" && WANT_1G=1

        printf '      - type: %s\n' "$r"
        printf '        count: %s\n' "$n"
        [ "${DPDK[$r]}" = true ] && printf '        dpdk: true\n'
        printf '        instances:\n'

        f=$(floor_for "$r")
        for ((k=1;k<=n;k++)); do
            iname="${HOSTNAME_}-${r}-${k}"
            say ""
            say "  ── $iname ──"

            # Devices first: they decide which NUMA node the cores should come
            # from. Allocating cores before knowing the device can strand an
            # instance on the wrong socket.
            DEV_NODE=""
            INST_NVME=0
            DEVBUF=$(mktemp)
            if takes_storage "$r"; then
                # Emit the header only when a device was actually picked. A bare
                # "storage:" key parses as null and fails the schema, which
                # masks the validator's own much clearer "SN has no storage"
                # message with a type error.
                DEVTMP=$(mktemp)
                pick_storage "$iname" > "$DEVTMP"
                if [ -s "$DEVTMP" ]; then
                    { printf '            storage:\n'; cat "$DEVTMP"; } > "$DEVBUF"
                else
                    printf '            storage: []\n' > "$DEVBUF"
                fi
                rm -f "$DEVTMP"
            fi
            if [ "${DPDK[$r]}" = true ]; then
                pick_nic "$iname" >> "$DEVBUF"
            fi

            if take_cores "$f" "$DEV_NODE"; then cs="$TAKEN"; else warn "ran out of cores for $iname"; cs="FIXME"; fi
            printf '          - name: "%s"\n' "$iname"
            printf '            cpuset: "%s"\n' "$cs"
            [ -n "$TAKEN_NODE" ] && printf '            numa_node: %s\n' "$TAKEN_NODE"
            _nd="${TAKEN_NODE:-0}"
            NODE_2M[$_nd]=$(( ${NODE_2M[$_nd]:-0} + $(hp2m_for "$r") ))
            NODE_1G[$_nd]=$(( ${NODE_1G[$_nd]:-0} + $(hp1g_fixed_for "$r") + INST_NVME * HP1G_PER_NVME ))
            printf '            container_cpuset: "%s"\n' "$OS_CORES"
            cat "$DEVBUF"; rm -f "$DEVBUF"
            say "  cpuset $cs (floor $f, NUMA node ${TAKEN_NODE:-?})"
        done
    done

    # isolcpus covers everything handed to FermiHDI, so the scheduler stops
    # placing other work on those cores.
    ISO=$(claimed_cpuset)
    # Hugepage sizing. The two pools are computed independently -- see the
    # memory model at the top of this script for who consumes which -- and each
    # is tracked per NUMA node, because that is the granularity the kernel
    # reserves at and the granularity a reactor allocates at.
    NEED_2M=0; NEED_1G=0; NEED_MB=0
    for r in $ROLES; do
        [ "${COUNT[$r]}" -gt 0 ] || continue
        NEED_MB=$(( NEED_MB + $(engine_mb_for "$r") * COUNT[$r] ))
    done
    # Sum the per-node tallies built during placement, and note the hungriest
    # node -- GRUB sizing depends on the peak, not on the total.
    MAX_1G_NODE=0; NODE_COUNT=0
    for nd in $NUMA_NODES; do
        NODE_COUNT=$(( NODE_COUNT + 1 ))
        n2=${NODE_2M[$nd]:-0}; n1=${NODE_1G[$nd]:-0}
        NEED_2M=$(( NEED_2M + n2 )); NEED_1G=$(( NEED_1G + n1 ))
        [ "$n1" -gt "$MAX_1G_NODE" ] && MAX_1G_NODE=$n1
    done
    [ "$NODE_COUNT" -eq 0 ] && NODE_COUNT=1

    # 1 GB pages can only come from GRUB, and "hugepagesz=1G hugepages=N" takes
    # a single number that the kernel spreads evenly across nodes. Reserving the
    # plain sum would leave the hungriest node short, so ask for its need on
    # every node.
    GRUB_1G=$(( MAX_1G_NODE * NODE_COUNT ))

    say ""
    say "  Engine budgets total ${NEED_MB} MB across all instances."
    if [ "$NODE_COUNT" -gt 1 ]; then
        say "  Per NUMA node:"
        for nd in $NUMA_NODES; do
            say "    node $nd: ${NODE_2M[$nd]:-0} x 2 MB, ${NODE_1G[$nd]:-0} x 1 GB"
        done
        if [ "$GRUB_1G" -gt "$NEED_1G" ]; then
            say "  GRUB spreads 1 GB pages evenly, so ${GRUB_1G} (${MAX_1G_NODE} x ${NODE_COUNT}"
            say "  nodes) is reserved to cover the hungriest node."
        fi
    fi
    if [ "$NVME_COUNT" -gt 0 ]; then
        say "  ${NVME_COUNT} NVMe device(s) x ${HP1G_PER_NVME} x 1 GB for SPDK DMA."
    elif [ "$NEED_1G" -eq 0 ]; then
        say "  No NVMe devices, so no 1 GB pages are needed (bdev_aio does not DMA)."
    fi

    # Per-node memory. A node-local shortfall is invisible in the host total:
    # two nodes of 64 GB look like 128 GB, but an instance pinned to node 1 can
    # only ever allocate from node 1.
    for nd in $NUMA_NODES; do
        nm=${NODE_MEM[$nd]:-0}
        [ "$nm" -gt 0 ] || continue
        want=$(( ${NODE_2M[$nd]:-0} * 2 + MAX_1G_NODE * 1024 ))
        [ "$want" -gt 0 ] || continue
        # Headroom is a host-wide figure; charge each node its share.
        nu=$(( nm - OS_HEADROOM_MB / NODE_COUNT ))
        if [ "$want" -gt "$nu" ]; then
            warn "NUMA node $nd needs ${want} MB of hugepages but has only ${nu} MB usable"
            warn "(${nm} MB local). An instance pinned there cannot borrow from another"
            warn "node, so this fails at startup even though the host total looks fine."
            ask_yn "  Continue anyway" "n" || exit 1
        fi
    done

    TOTAL_HP_MB=$(( NEED_2M * 2 + GRUB_1G * 1024 ))
    say "  Host has ${MEM_TOTAL_MB} MB; leaving ${OS_HEADROOM_MB} MB for the OS, stack and heap."
    USABLE_MB=$(( MEM_TOTAL_MB - OS_HEADROOM_MB ))
    if [ "$MEM_TOTAL_MB" -gt 0 ] && [ "$TOTAL_HP_MB" -gt "$USABLE_MB" ]; then
        warn "This host has too little memory: ${TOTAL_HP_MB} MB of hugepages needed, ${USABLE_MB} MB usable."
        warn "Reserving that many would leave the OS short and risk an OOM kill during"
        warn "startup, when every reactor allocates its budget at once."
        ask_yn "  Continue anyway" "n" || exit 1
    fi

    if [ "$GRUB_1G" -gt 0 ]; then
        H1G=$(ask_int "  1 GB hugepages to allocate (need $GRUB_1G)" "$GRUB_1G")
    else
        H1G=0
    fi
    if [ "$WANT_2M" = 1 ]; then
        H2M=$(ask_int "  2 MB hugepages to allocate (need $NEED_2M = $((NEED_2M*2)) MB)" "$NEED_2M")
    else
        H2M=0
    fi
    TOTAL_HP_MB=$(( H2M * 2 + H1G * 1024 ))
    if [ "$MEM_TOTAL_MB" -gt 0 ] && [ "$TOTAL_HP_MB" -gt "$USABLE_MB" ]; then
        warn "Requested hugepages total ${TOTAL_HP_MB} MB but only ${USABLE_MB} MB is usable."
        warn "Allocation will fail or the host will be left without enough free memory."
        ask_yn "  Write it anyway" "n" || exit 1
    fi

    printf '    host_prep:\n'
    printf '      memory_total_mb: %s\n' "$MEM_TOTAL_MB"
    printf '      hugepages_2m: %s\n' "$H2M"
    printf '      hugepages_1g: %s\n' "$H1G"
    # Per-node breakdown. The totals above are what GRUB and sysctl are given;
    # this is what each node actually has to end up with, which is what
    # fermihdi-prepare verifies after the reboot.
    if [ "$NODE_COUNT" -gt 1 ] || [ "$NEED_2M" -gt 0 ] || [ "$NEED_1G" -gt 0 ]; then
        printf '      numa:\n'
        for nd in $NUMA_NODES; do
            printf '        - node: %s\n' "$nd"
            printf '          hugepages_2m: %s\n' "${NODE_2M[$nd]:-0}"
            printf '          hugepages_1g: %s\n' "${NODE_1G[$nd]:-0}"
            [ "${NODE_MEM[$nd]:-0}" -gt 0 ] && printf '          memory_mb: %s\n' "${NODE_MEM[$nd]}"
        done
    fi
    # VFIO_BIND is seeded with a space so add_vfio's substring test works, so
    # test the trimmed value -- [ -n "$VFIO_BIND" ] is true even when empty and
    # would claim IOMMU is needed on a host that binds nothing.
    HAS_VFIO=0; [ -n "${VFIO_BIND// /}" ] && HAS_VFIO=1
    printf '      iommu: %s\n' "$([ "$HAS_VFIO" = 1 ] && echo true || echo false)"
    [ -n "$ISO" ] && printf '      isolcpus: "%s"\n' "$ISO"
    if [ "$HAS_VFIO" = 1 ]; then
        printf '      vfio_bind:\n'
        for p in $VFIO_BIND; do printf '        - "%s"\n' "$p"; done
    else
        # An explicit empty list, not a bare key: a bare key parses as null and
        # fails the schema's array type.
        printf '      vfio_bind: []\n'
    fi
    printf '      grub:\n'
    printf '        apply: %s\n' "$([ -n "$ISO" ] || [ "$HAS_VFIO" = 1 ] && echo true || echo false)"
} > "$BODY"

# ── Emit ─────────────────────────────────────────────────────────────────────
DMS_URL_DEFAULT="https://dms-core:6986"
[ "${COUNT[dms]:-0}" -gt 0 ] && DMS_URL_DEFAULT="https://${ADDRESS:-$HOSTNAME_}:6986"

# The dms: block's file references, in the order the schema lists them. Nothing
# is emitted for a question that was skipped: an absent key means "not provided",
# an empty one would mean "provided, and empty".
dms_file_lines() {
    [ -n "$DMS_LICENSE_FILE" ]  && printf '  license_file: "%s"\n'  "$DMS_LICENSE_FILE"
    [ -n "$DMS_ICA_CERT_FILE" ] && printf '  ica_cert_file: "%s"\n' "$DMS_ICA_CERT_FILE"
    [ -n "$DMS_ICA_KEY_FILE" ]  && printf '  ica_key_file: "%s"\n'  "$DMS_ICA_KEY_FILE"
    [ -n "$DMS_SCHEMA_FILE" ]   && printf '  schema_file: "%s"\n'   "$DMS_SCHEMA_FILE"
    return 0
}

if [ -n "$APPEND" ] && [ -f "$APPEND" ]; then
    cat "$BODY" >> "$APPEND"
    say ""; say "  Appended $HOSTNAME_ to $APPEND"
    # The dms: block is written once, when the file is created, and the host
    # that runs the DMS is often not the first one configured. So the answers go
    # into the existing block by inserting lines under it -- a text edit rather
    # than a YAML rewrite, which would reformat the file and drop its comments.
    if [ -n "$(dms_file_lines)" ]; then
        if DMS_LINES="$(dms_file_lines)" python3 - "$APPEND" <<'PYEOF'; then
import os, sys, pathlib
path = pathlib.Path(sys.argv[1])
lines = path.read_text().splitlines(keepends=True)
new = [l if l.endswith("\n") else l + "\n" for l in os.environ["DMS_LINES"].splitlines()]
keys = {l.split(":", 1)[0].strip() for l in new}
try:
    start = next(i for i, l in enumerate(lines) if l.rstrip("\n") == "dms:")
except StopIteration:
    sys.exit(1)
end = next((i for i in range(start + 1, len(lines))
            if lines[i].strip() and not lines[i][:1].isspace()), len(lines))
# Replace any key already there rather than adding a second copy of it.
kept = [l for l in lines[start + 1:end] if l.split(":", 1)[0].strip() not in keys]
path.write_text("".join(lines[:start + 1] + kept + new + lines[end:]))
PYEOF
            say "  Added the DMS file references to the dms: block in $APPEND"
        else
            say ""
            warn "$APPEND has no top-level dms: block. Add these lines to it yourself:"
            dms_file_lines | while IFS= read -r l; do say "    $l"; done
        fi
    fi
else
    {
        printf '# Generated by fermihdi-configure on %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
        printf 'version: 1\n'
        printf 'cluster_name: "%s"\n' "$(ask '  Cluster name' 'hd-1')"
        printf 'registry:\n'
        printf '  url: "%s"\n' "$(ask '  Registry URL' 'cr.fermihdi.io')"
        printf '  username: "%s"\n' "$(ask '  Registry username' 'deploy')"
        printf '  password_env: FERMIHDI_REGISTRY_PASSWORD\n'
        printf 'dms:\n'
        # Every node is handed this URL verbatim and has to reach the DMS at it,
        # so when this host is the one running the DMS its own address is a far
        # better default than a container name that resolves nowhere else.
        printf '  url: "%s"\n' "$(ask '  DMS URL' "$DMS_URL_DEFAULT")"
        printf '  bootstrap_token_env: FERMIHDI_BOOTSTRAP_TOKEN\n'
        dms_file_lines
        printf 'hosts:\n'
        cat "$BODY"
    } > "$OUT"
    say ""; say "  Wrote $OUT"
fi

say ""
say "  The registry password and bootstrap token are NOT stored in this file."
say "  Export them before running the Ansible layer:"
say "    export FERMIHDI_REGISTRY_PASSWORD=..."
say "    export FERMIHDI_BOOTSTRAP_TOKEN=..."
say ""
VALID=1
if [ -x "$HERE/validate-cluster" ]; then
    say "  Validating..."
    "$HERE/validate-cluster" "$OUT" >&2 || { VALID=0; say "  Fix the errors above before deploying."; }
fi

# Only point at the next step once the file passes. Preparing a host from a
# definition that does not validate is how a machine ends up half-configured.
if [ "$VALID" = 1 ]; then
    say ""
    say "  Next, apply the host preparation this file describes:"
    say "    sudo $HERE/fermihdi-prepare $OUT           # dry run, changes nothing"
    say "    sudo $HERE/fermihdi-prepare $OUT --apply"
    say ""
    say "  It never reboots. When one is needed it says so, and after the reboot:"
    say "    $HERE/fermihdi-prepare $OUT --verify"
fi
