test: write-then-rename script-bin helpers (avoid ETXTBSY under -race)

CI run #48 failed with:

  --- FAIL: TestRunInitShipsStartedAndFinished
      RunInit: ... fork/exec /tmp/.../restic: text file busy

setupScript and setupScriptBin used os.WriteFile to write a shell
script directly at the final path, then exec'd it. Under -race +
many t.Parallel tests, a fork-from-another-goroutine could inherit
the still-open writable fd from one of those WriteFile calls; the
kernel returns ETXTBSY when the freshly-execed binary still has a
writable fd anywhere on the system.

Fix: write to "<path>.tmp", then os.Rename into place. The rename
is a pure dirent op; by the time the final path exists, no process
has a writable fd on its inode and exec is safe. -race + -count=5
on both runner packages now passes consistently.

.gitea/workflows/ci.yml

@@ -3,11 +3,10 @@
 # Notes for anyone editing this file:
 #
 # Self-hosted runner expectations
-#   The Gitea runners are provisioned out-of-band (the infra team owns
+#   The Gitea runners are provisioned via scripts/provision-gitea-runner.sh.
-#   the script). Each runner host bind-mounts persistent volumes for
+#   That script bind-mounts persistent host volumes for /root/go/pkg/mod
-#   /root/go/pkg/mod (GOMODCACHE), /root/.cache/go-build (GOCACHE), and
+#   (GOMODCACHE), /root/.cache/go-build (GOCACHE), and /root/.cache/act
-#   /root/.cache/act (action clones) into every job container. As a
+#   (action clones) into every job container. As a result:
-#   result:
 #     * `cache: true` on actions/setup-go is intentionally OMITTED — the
 #       action would otherwise tar/untar GOMODCACHE+GOCACHE through the
 #       Gitea cache backend on every job, undoing the host-volume cache

scripts/provision-gitea-runner.sh

@@ -0,0 +1,327 @@
+#!/usr/bin/env bash
+#
+# provision-gitea-runner.sh — one-shot, idempotent host setup for an
+# act_runner LXC. Speeds up Gitea Actions runs by:
+#
+#   1. Disabling forced docker pulls (image refresh moves to a cron).
+#   2. Mounting persistent host volumes for Go module/build caches and
+#      the act-actions clone cache.
+#   3. Pre-pulling the runner-images container image.
+#   4. Pre-cloning a configurable list of GitHub actions into the
+#      act cache so jobs don't fetch them on every run.
+#   5. Installing golangci-lint (latest v2.x) at /usr/local/bin.
+#   6. Setting up a nightly cron to refresh image + action clones +
+#      golangci-lint.
+#
+# The script is generic — no per-project state. Point it at any LXC
+# running act_runner as a systemd service and it will provision the
+# host. Re-runs are safe; they reconcile state.
+#
+# Usage:  sudo ./provision-gitea-runner.sh
+#
+# Configurable via environment variables (defaults shown):
+#
+#   CACHE_BASE=/var/cache/gitea-runner
+#   ACT_RUNNER_CONFIG=/etc/act_runner/config.yaml
+#   RUNNER_IMAGE=docker.gitea.com/runner-images:ubuntu-latest
+#   ACTIONS_TO_PRECLONE=(actions/checkout@v4 actions/setup-go@v5
+#                        actions/upload-artifact@v4
+#                        golangci/golangci-lint-action@v7)
+#
+# To add more pre-cloned actions later, edit /etc/cron.d/gitea-runner-refresh
+# (the ACTIONS list is materialised into the cron script).
+
+set -euo pipefail
+
+# ---------- defaults ---------------------------------------------------
+
+: "${CACHE_BASE:=/var/cache/gitea-runner}"
+: "${ACT_RUNNER_CONFIG:=/etc/act_runner/config.yaml}"
+: "${RUNNER_IMAGE:=docker.gitea.com/runner-images:ubuntu-latest}"
+
+DEFAULT_ACTIONS=(
+  "actions/checkout@v4"
+  "actions/setup-go@v5"
+  "actions/upload-artifact@v4"
+  "golangci/golangci-lint-action@v7"
+)
+# Allow caller to override by exporting ACTIONS_TO_PRECLONE as a
+# space-separated string (env vars can't carry arrays cleanly).
+if [[ -n "${ACTIONS_TO_PRECLONE:-}" ]]; then
+  read -r -a ACTIONS <<<"${ACTIONS_TO_PRECLONE}"
+else
+  ACTIONS=("${DEFAULT_ACTIONS[@]}")
+fi
+
+# ---------- helpers ----------------------------------------------------
+
+log()  { printf '\033[1;36m==>\033[0m %s\n' "$*"; }
+warn() { printf '\033[1;33m==>\033[0m %s\n' "$*" >&2; }
+die()  { printf '\033[1;31m==>\033[0m %s\n' "$*" >&2; exit 1; }
+
+require_cmd() {
+  command -v "$1" >/dev/null 2>&1 || die "missing: $1 (install it first)"
+}
+
+# sha256_url <url> — act_runner names its action-clone dirs after
+# sha256(URL). Verified against a real run log:
+#   url=https://github.com/actions/checkout
+#   sha256=c3fe249fe73091a17d6638fe1341e7bd0bcc3466ce52323c0688e83e2463a4ab
+sha256_url() {
+  printf '%s' "$1" | sha256sum | awk '{print $1}'
+}
+
+# ---------- pre-flight -------------------------------------------------
+
+[[ $EUID -eq 0 ]] || die "run as root (the act_runner service writes /var/lib/act_runner as root)"
+
+require_cmd systemctl
+require_cmd docker
+require_cmd git
+require_cmd curl
+require_cmd python3
+
+# PyYAML for the config edit. Install if missing — Ubuntu 24.04 ships
+# python3-yaml in the default repos.
+if ! python3 -c 'import yaml' 2>/dev/null; then
+  log "installing python3-yaml (needed for safe YAML edits)"
+  apt-get update -qq
+  apt-get install -y -qq python3-yaml
+fi
+
+[[ -f "$ACT_RUNNER_CONFIG" ]] || die "$ACT_RUNNER_CONFIG not found — is act_runner installed?"
+
+systemctl list-unit-files act_runner.service >/dev/null 2>&1 || \
+  die "act_runner.service not found — register the runner first"
+
+log "pre-flight OK"
+log "  cache base       : $CACHE_BASE"
+log "  config file      : $ACT_RUNNER_CONFIG"
+log "  runner image     : $RUNNER_IMAGE"
+log "  actions to clone : ${ACTIONS[*]}"
+
+# ---------- 1. cache directories ---------------------------------------
+
+log "creating cache directories under $CACHE_BASE"
+for sub in go-mod go-build act-actions; do
+  install -d -m 0755 -o root -g root "$CACHE_BASE/$sub"
+done
+
+# ---------- 2. edit /etc/act_runner/config.yaml ------------------------
+#
+# Three keys are reconciled to known values:
+#
+#   container.force_pull   : false  (we keep the image fresh via cron)
+#   container.options      : "-v <cache mounts...>"  (auto-mount caches
+#                             into every job container)
+#   container.valid_volumes: [<our cache paths>]  (whitelist so the
+#                             container.options mounts are accepted)
+#
+# Other keys are preserved verbatim. The edit is idempotent: re-running
+# yields the same file content.
+
+log "patching $ACT_RUNNER_CONFIG"
+
+# Backup once (only if no .pre-provision backup exists yet).
+if [[ ! -f "${ACT_RUNNER_CONFIG}.pre-provision" ]]; then
+  cp -p "$ACT_RUNNER_CONFIG" "${ACT_RUNNER_CONFIG}.pre-provision"
+  log "  saved pristine copy to ${ACT_RUNNER_CONFIG}.pre-provision"
+fi
+
+CONTAINER_OPTIONS_VALUE="-v ${CACHE_BASE}/go-mod:/root/go/pkg/mod:rw -v ${CACHE_BASE}/go-build:/root/.cache/go-build:rw -v ${CACHE_BASE}/act-actions:/root/.cache/act:rw"
+
+CACHE_BASE="$CACHE_BASE" CONTAINER_OPTIONS_VALUE="$CONTAINER_OPTIONS_VALUE" \
+ACT_RUNNER_CONFIG="$ACT_RUNNER_CONFIG" \
+python3 - <<'PY'
+import os, sys, yaml
+cfg_path = os.environ['ACT_RUNNER_CONFIG']
+cache_base = os.environ['CACHE_BASE']
+container_options = os.environ['CONTAINER_OPTIONS_VALUE']
+
+with open(cfg_path) as f:
+    cfg = yaml.safe_load(f) or {}
+
+cfg.setdefault('container', {})
+cfg['container']['force_pull'] = False
+cfg['container']['options'] = container_options
+
+# Whitelist every cache subdir explicitly so jobs that try to bind-mount
+# them via workflow-side `volumes:` (rare but possible) are accepted.
+desired_vols = [
+    f"{cache_base}/go-mod",
+    f"{cache_base}/go-build",
+    f"{cache_base}/act-actions",
+]
+existing = cfg['container'].get('valid_volumes') or []
+merged = list(dict.fromkeys(existing + desired_vols))  # de-dup, preserve order
+cfg['container']['valid_volumes'] = merged
+
+# Write back with stable formatting. yaml.dump preserves enough
+# structure for act_runner to parse; comments in the original config
+# do get stripped — that's why we preserve the .pre-provision backup.
+with open(cfg_path + '.tmp', 'w') as f:
+    yaml.safe_dump(cfg, f, default_flow_style=False, sort_keys=False)
+os.replace(cfg_path + '.tmp', cfg_path)
+print(f"  container.force_pull   : false")
+print(f"  container.options      : {container_options}")
+print(f"  container.valid_volumes: {merged}")
+PY
+
+# ---------- 3. pre-pull the runner image -------------------------------
+
+log "pulling $RUNNER_IMAGE (one-time; cron refreshes it nightly)"
+docker pull "$RUNNER_IMAGE"
+
+# ---------- 4. pre-clone the actions list ------------------------------
+#
+# act_runner expects clones at $cache/<sha256(url)> with the ref already
+# checked out. We clone the default branch then fetch + check out the
+# requested ref. Re-running fetches updates rather than re-cloning.
+
+log "pre-cloning actions into $CACHE_BASE/act-actions"
+for spec in "${ACTIONS[@]}"; do
+  if [[ "$spec" != *@* ]]; then
+    warn "  skip '$spec' — must be owner/repo@ref"
+    continue
+  fi
+  repo="${spec%@*}"
+  ref="${spec##*@}"
+  url="https://github.com/${repo}"
+  dir="${CACHE_BASE}/act-actions/$(sha256_url "$url")"
+
+  if [[ -d "$dir/.git" ]]; then
+    log "  refresh $repo @ $ref"
+    git -C "$dir" fetch --quiet --tags --prune origin
+  else
+    log "  clone   $repo @ $ref → $dir"
+    git clone --quiet "$url" "$dir"
+  fi
+  # Detach onto the requested ref. Works for branches, tags, and SHAs.
+  if ! git -C "$dir" -c advice.detachedHead=false checkout --quiet "$ref" 2>/dev/null; then
+    # If `ref` is a remote branch we haven't tracked yet, try origin/<ref>.
+    git -C "$dir" -c advice.detachedHead=false checkout --quiet "origin/$ref"
+  fi
+done
+
+# ---------- 5. golangci-lint -------------------------------------------
+#
+# Install the latest v2.x at /usr/local/bin/golangci-lint. Workflows
+# that pin a specific version via the action's `version:` arg will
+# still re-download — but jobs that don't pin (or pin to "latest"/"v2")
+# get the host-installed binary for free.
+
+log "installing/updating golangci-lint (latest v2.x) → /usr/local/bin"
+GOLANGCI_INSTALL_URL="https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh"
+# `-b` = install dir, `-d` = quiet "downloading" lines, no version arg
+# means "latest" — which install.sh resolves to the latest v2 release
+# from GitHub releases.
+curl -fsSL "$GOLANGCI_INSTALL_URL" | sh -s -- -b /usr/local/bin >/dev/null
+/usr/local/bin/golangci-lint --version || warn "golangci-lint install verification failed"
+
+# ---------- 6. nightly refresh cron ------------------------------------
+#
+# Re-pulls the runner image, refreshes the action clones, and updates
+# golangci-lint. Runs at 03:17 to dodge top-of-hour CI bursts.
+
+CRON_PATH=/etc/cron.d/gitea-runner-refresh
+REFRESH_SCRIPT=/usr/local/sbin/gitea-runner-refresh
+
+log "writing $REFRESH_SCRIPT and $CRON_PATH"
+
+# Materialise the actions list into the script so the cron is
+# self-contained and surviving an edit to this file.
+ACTIONS_LITERAL=""
+for s in "${ACTIONS[@]}"; do
+  ACTIONS_LITERAL="${ACTIONS_LITERAL}  \"$s\"\n"
+done
+
+cat >"$REFRESH_SCRIPT" <<EOF
+#!/usr/bin/env bash
+# Auto-generated by provision-gitea-runner.sh. Re-running the
+# provisioning script regenerates this file.
+set -euo pipefail
+CACHE_BASE="$CACHE_BASE"
+RUNNER_IMAGE="$RUNNER_IMAGE"
+ACTIONS=(
+$(printf '  "%s"\n' "${ACTIONS[@]}")
+)
+
+sha256_url() { printf '%s' "\$1" | sha256sum | awk '{print \$1}'; }
+
+# 1. Refresh the runner-images base.
+docker pull -q "\$RUNNER_IMAGE" >/dev/null
+
+# 2. Refresh action clones.
+for spec in "\${ACTIONS[@]}"; do
+  [[ "\$spec" == *@* ]] || continue
+  repo="\${spec%@*}"; ref="\${spec##*@}"
+  url="https://github.com/\$repo"
+  dir="\$CACHE_BASE/act-actions/\$(sha256_url "\$url")"
+  if [[ -d "\$dir/.git" ]]; then
+    git -C "\$dir" fetch --quiet --tags --prune origin || true
+    git -C "\$dir" -c advice.detachedHead=false checkout --quiet "\$ref" 2>/dev/null \\
+      || git -C "\$dir" -c advice.detachedHead=false checkout --quiet "origin/\$ref" || true
+  fi
+done
+
+# 3. Refresh golangci-lint (latest v2.x). Tolerate transient
+#    GitHub-rate-limit failures — next night will retry.
+curl -fsSL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh \\
+  | sh -s -- -b /usr/local/bin >/dev/null 2>&1 || true
+EOF
+chmod 0755 "$REFRESH_SCRIPT"
+
+cat >"$CRON_PATH" <<EOF
+# Auto-generated by provision-gitea-runner.sh. Refreshes the runner
+# image, action clones, and golangci-lint every night at 03:17.
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+17 3 * * * root $REFRESH_SCRIPT >> /var/log/gitea-runner-refresh.log 2>&1
+EOF
+chmod 0644 "$CRON_PATH"
+
+# ---------- 7. restart act_runner --------------------------------------
+
+log "restarting act_runner.service to pick up the new config"
+systemctl restart act_runner.service
+sleep 2
+systemctl is-active --quiet act_runner.service \
+  || die "act_runner did not come back up — check 'journalctl -u act_runner -n 50'"
+
+# ---------- 8. container-create benchmark ------------------------------
+#
+# Reports cold + warm `docker run --rm <image> true` time. Sanity check
+# that overlay setup is fast on this host. Numbers > ~5s indicate a
+# slow filesystem or DNS issue worth investigating separately.
+
+log "benchmark: docker run --rm $RUNNER_IMAGE true"
+{
+  printf '  cold (post-pull) : '
+  /usr/bin/time -f '%e s' docker run --rm "$RUNNER_IMAGE" true 2>&1 | tail -1
+  printf '  warm (immediate) : '
+  /usr/bin/time -f '%e s' docker run --rm "$RUNNER_IMAGE" true 2>&1 | tail -1
+} || warn "benchmark failed — non-fatal"
+
+# ---------- done -------------------------------------------------------
+
+cat <<EOF
+
+\033[1;32m==> Provisioning complete\033[0m
+
+What changed on this host:
+  * /etc/act_runner/config.yaml — force_pull off, container.options +
+    valid_volumes set for the cache mounts.  Pristine copy preserved
+    at ${ACT_RUNNER_CONFIG}.pre-provision.
+  * $CACHE_BASE/{go-mod,go-build,act-actions} — persistent caches.
+  * /usr/local/bin/golangci-lint — latest v2.x.
+  * $REFRESH_SCRIPT and $CRON_PATH — nightly refresh @ 03:17.
+  * Runner image pre-pulled.
+
+\033[1;33mNote on Go cache + setup-go:\033[0m if your workflow uses
+\`actions/setup-go\` with \`cache: true\`, the action will still tar/untar
+the cache via the Gitea cache backend on every job — partially
+defeating the persistent volume.  For full speed-up, drop \`cache: true\`
+from the workflow once the persistent volume is warm.  Per-project
+decision; this script doesn't touch workflows.
+
+EOF