From 3b8674a4a9981b93bdeee5d92bbfdc1228a0a3f4 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Wed, 24 Jun 2026 11:19:58 +0530
Subject: [PATCH 01/11] ci(mariadb): self-hosted fan-out CI (ARC runners,
 datadir bake, 8 shards)

---
 .github/helper/hydrate.sh                  |  74 +++++
 .github/helper/install.sh                  | 367 ++++++++++++++++++---
 .github/helper/start-db.sh                 |  75 +++++
 .github/workflows/server-tests-mariadb.yml | 200 ++++++-----
 4 files changed, 581 insertions(+), 135 deletions(-)
 create mode 100755 .github/helper/hydrate.sh
 create mode 100755 .github/helper/start-db.sh

diff --git a/.github/helper/hydrate.sh b/.github/helper/hydrate.sh
new file mode 100755
index 00000000000..4619d63dbeb
--- /dev/null
+++ b/.github/helper/hydrate.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Hydrate a test shard from the setup job's artifact.
+#
+# The bench (apps, venv, node_modules, sites) is already on disk at ~/frappe-bench — the
+# workflow untar'd it from the artifact the setup job built. So there is NO bench init, no
+# asset build, and no reinstall here: just bring the DB up and restore the dump the setup job
+# baked into the bench, then start bench so tests can run. Mirrors the DB + bench-start tail of
+# install.sh. The whole point is that the expensive work happened ONCE in the setup job.
+#
+set -e
+
+ci_user="${ERPNEXT_CI_USER:-frappe}"
+db_host="${DB_HOST:-127.0.0.1}"
+dump="${CI_BASELINE_BACKUP:-/home/$ci_user/frappe-bench/test_site-db.sql.gz}"
+
+# Re-exec as the ci user (uid 1001) so bench/cache ownership matches the artifact, same as
+# install.sh. The workflow untar'd as root with -p, so the files are already owned by ci.
+if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then
+    exec su -m "$ci_user" -s /bin/bash -c \
+        "ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' CI_BASELINE_BACKUP='$dump' bash '$0'"
+fi
+
+cd ~/frappe-bench
+
+# Start the DB on the datadir baked into the artifact. It's already populated (the setup job
+# reinstalled into this very datadir), so there is NO restore — the server comes up on the
+# existing files. This is what replaces the per-shard SQL replay.
+bash ~/frappe-bench/start-db.sh
+
+# Bring up redis (lightmode unit tests need cache + queue). In the self-hosted container we use the
+# full `bench start` (web/workers too, like install.sh). On the bare GitHub Postgres shard
+# `bench start` (honcho) lagged — it blocks the redis procs behind web/worker procs the lightmode
+# suite never uses, so the wait below burned its full timeout (~4m). There, start the two redis
+# instances directly: fast and deterministic.
+if [ "${DB:-mariadb}" = "postgres" ]; then
+    # Start redis directly as daemons — reliable and persists across steps. Do NOT route it through
+    # `bench start`: honcho tears the whole process group down if any one Procfile proc dies on the
+    # bare shard, which took redis with it (redis @ 13000 refused in Run Tests). Keeping redis
+    # independent is what makes it survive. The web server (for PDF tests) is NOT started here — a
+    # backgrounded server doesn't survive into the next step; it's started inside the Run Tests step.
+    for conf in redis_cache redis_queue; do
+        [ -f ~/frappe-bench/config/$conf.conf ] && redis-server ~/frappe-bench/config/$conf.conf --daemonize yes
+    done
+else
+    bench start >> ~/frappe-bench/bench_start.log 2>&1 &
+fi
+
+# Wait for redis, failing fast instead of silently burning minutes if it never comes up.
+cfg=~/frappe-bench/sites/common_site_config.json
+if [ -f "$cfg" ]; then
+    ports=$(python - "$cfg" <<'PY'
+import json, re, sys
+try:
+    cfg = json.load(open(sys.argv[1]))
+except Exception:
+    sys.exit(0)
+for key in ("redis_cache", "redis_queue"):
+    m = re.search(r":(\d+)", str(cfg.get(key, "")))
+    if m:
+        print(m.group(1))
+PY
+)
+    for port in $ports; do
+        up=0
+        for _ in $(seq 1 60); do
+            if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then exec 3>&- 3<&-; up=1; break; fi
+            sleep 1
+        done
+        [ "$up" = "1" ] || { echo "redis did not come up on port $port"; exit 1; }
+    done
+fi
+
+echo "Hydrated: DB up on baked datadir, redis up — ready for tests."
diff --git a/.github/helper/install.sh b/.github/helper/install.sh
index b5f9b9e364b..74330abca16 100644
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@@ -7,21 +7,106 @@ cd ~ || exit
 githubbranch=${GITHUB_BASE_REF:-${GITHUB_REF##*/}}
 frappeuser=${FRAPPE_USER:-"frappe"}
 frappecommitish=${FRAPPE_BRANCH:-$githubbranch}
+db_host=${DB_HOST:-"127.0.0.1"}
+db_user_host=${DB_USER_HOST:-"localhost"}
+wkhtmltox_deb=${WKHTMLTOX_DEB:-"/tmp/wkhtmltox.deb"}
+bench_cache_dir=${BENCH_CACHE_DIR:-}
+
+run_as_ci_user_if_needed() {
+    if [ "$(id -u)" != "0" ] || [ "${SKIP_SYSTEM_SETUP:-0}" != "1" ] || [ "${ERPNEXT_CI_NON_ROOT:-0}" = "1" ]; then
+        return
+    fi
+
+    local missing_packages=()
+    if ! command -v pkg-config >/dev/null 2>&1; then
+        missing_packages+=("pkg-config")
+    fi
+    if ! command -v mariadb_config >/dev/null 2>&1 && ! command -v mysql_config >/dev/null 2>&1; then
+        missing_packages+=("libmariadb-dev")
+    fi
+    if ! command -v crontab >/dev/null 2>&1; then
+        missing_packages+=("cron")
+    fi
+
+    if [ "${#missing_packages[@]}" -gt 0 ]; then
+        apt-get update
+        apt-get install -y --no-install-recommends "${missing_packages[@]}"
+    fi
+
+    local ci_user="${ERPNEXT_CI_USER:-frappe}"
+
+    if ! id "$ci_user" >/dev/null 2>&1; then
+        useradd --home-dir "$HOME" --no-create-home --shell /bin/bash "$ci_user"
+    fi
+
+    rm -rf ~/frappe ~/frappe-bench
+
+    local ci_dirs=(
+        "$HOME"
+        "$GITHUB_WORKSPACE"
+        "$HOME/.cache"
+        "${PIP_CACHE_DIR:-$HOME/.cache/pip}"
+        "${npm_config_cache:-$HOME/.npm}"
+        "${YARN_CACHE_FOLDER:-$HOME/.cache/yarn}"
+        "$HOME/.yarn"
+        "${UV_CACHE_DIR:-$HOME/.cache/uv}"
+        "$(dirname "$wkhtmltox_deb")"
+    )
+    if [ -n "$bench_cache_dir" ]; then
+        ci_dirs+=("$bench_cache_dir")
+    fi
+
+    # Create + own (non-recursively) the home/cache/workspace dirs before dropping to
+    # the ci user. We deliberately do NOT wipe the yarn/uv caches here so a persistent
+    # cache (mounted volume or baked image layer) stays warm across runs.
+    mkdir -p "${ci_dirs[@]}" "$HOME/.yarn"
+    chown "$ci_user:$ci_user" "${ci_dirs[@]}" "$HOME/.yarn"
+
+    export ERPNEXT_CI_NON_ROOT=1
+    exec su -m "$ci_user" -s /bin/bash -c "cd '$HOME' && bash '$GITHUB_WORKSPACE/.github/helper/install.sh'"
+}
+
+run_as_ci_user_if_needed
+
+run_ci_step() {
+    local label=$1
+    shift
+
+    echo "::group::${label}"
+    date -u
+    timeout --foreground "${CI_INSTALL_STEP_TIMEOUT:-600}" "$@"
+    local exit_code=$?
+    date -u
+    echo "::endgroup::"
+    return "$exit_code"
+}
+
+if [ -n "${GITHUB_WORKSPACE:-}" ]; then
+    git config --global --add safe.directory "$GITHUB_WORKSPACE" || true
+    git config --global --add safe.directory "$GITHUB_WORKSPACE/.git" || true
+fi
+
+rm -rf ~/frappe ~/frappe-bench
 
 # ---------------------------------------------------------------------------
 # Phase 1 — parallelise the three slow, independent setup steps:
 #   a) system packages   b) frappe-bench pip install   c) frappe git fetch
 # ---------------------------------------------------------------------------
 
-sudo apt update
+if [ "${SKIP_SYSTEM_SETUP:-0}" != "1" ]; then
+    sudo apt-get update
 
-# apt remove/install must run sequentially but can overlap with pip and git.
-sudo apt remove mysql-server mysql-client
-sudo apt install libcups2-dev redis-server mariadb-client libmariadb-dev &
-apt_pid=$!
+    # apt remove/install must run sequentially but can overlap with pip and git.
+    sudo apt-get remove -y mysql-server mysql-client
+    sudo apt-get install -y libcups2-dev redis-server mariadb-client libmariadb-dev &
+    apt_pid=$!
 
-pip install frappe-bench &
-pip_pid=$!
+    pip install frappe-bench &
+    pip_pid=$!
+else
+    apt_pid=
+    pip_pid=
+fi
 
 mkdir frappe
 (
@@ -32,84 +117,264 @@ mkdir frappe
 ) &
 clone_pid=$!
 
-wait $apt_pid
-wait $pip_pid
+if [ -n "$apt_pid" ]; then wait $apt_pid; fi
+if [ -n "$pip_pid" ]; then wait $pip_pid; fi
 wait $clone_pid
 
 pushd frappe
 git checkout FETCH_HEAD
 popd
+frappe_sha=$(git -C frappe rev-parse HEAD)
+
+get_bench_cache_archive() {
+    if [ -z "$bench_cache_dir" ]; then
+        return
+    fi
+
+    mkdir -p "$bench_cache_dir"
+
+    # Keyed on tool versions only (NOT the frappe SHA): any recent base bench works, because
+    # restore_warm_bench fast-forwards it to the exact live develop SHA. This is what lets a
+    # constantly-moving develop still hit the cache.
+    local cache_key
+    cache_key=$(
+        {
+            uname -m
+            python --version
+            node --version
+            bench --version
+        } | sha256sum | awk '{print $1}'
+    )
+
+    echo "${bench_cache_dir}/frappe-bench-base-${cache_key}.tar.zst"
+}
+
+restore_warm_bench() {
+    bench_cache_archive=$(get_bench_cache_archive)
+    [ -n "$bench_cache_archive" ] && [ -f "$bench_cache_archive" ] || return 1
+
+    echo "Restoring base bench from ${bench_cache_archive}"
+    tar --use-compress-program=unzstd -xf "$bench_cache_archive" -C ~ || return 1
+    [ -d ~/frappe-bench/apps/frappe/.git ] || return 1
+    mkdir -p ~/frappe-bench/sites ~/frappe-bench/logs
+    [ -f ~/frappe-bench/sites/apps.txt ] || printf "frappe\n" > ~/frappe-bench/sites/apps.txt
+    [ -f ~/frappe-bench/sites/common_site_config.json ] || printf "{}\n" > ~/frappe-bench/sites/common_site_config.json
+
+    # Fast-forward the restored frappe to the EXACT live develop SHA fetched in phase 1, then
+    # rebuild only what changed. The editable install means the venv tracks the new code with
+    # no reinstall. Any failure returns non-zero so the caller falls back to a full bench init.
+    if ! (
+        cd ~/frappe-bench/apps/frappe || exit 1
+        # Phase 1 already fetched ~/frappe to the exact live develop SHA. Fetch that commit
+        # straight from it (bench init names the remote 'upstream', not 'origin', and points
+        # it at this local clone — so a plain `git fetch origin` does not work).
+        git fetch --no-tags "$HOME/frappe" HEAD || exit 1
+        git checkout --force FETCH_HEAD || exit 1
+    ); then
+        echo "Fast-forward to ${frappe_sha} failed; falling back to full init"
+        rm -rf ~/frappe-bench
+        return 1
+    fi
+
+    # Pick up any frappe dependency changes since the base was built (cached → fast if none),
+    # so a develop commit that bumped requirements doesn't leave a stale venv.
+    if ! ~/frappe-bench/env/bin/python -m pip install -q -e ~/frappe-bench/apps/frappe; then
+        echo "frappe dependency refresh failed; falling back to full init"
+        rm -rf ~/frappe-bench
+        return 1
+    fi
+
+    ( cd ~/frappe-bench && CI=Yes bench build --app frappe ) || { rm -rf ~/frappe-bench; return 1; }
+    return 0
+}
+
+save_warm_bench() {
+    if [ -z "${bench_cache_archive:-}" ] || [ -f "$bench_cache_archive" ]; then
+        return
+    fi
+
+    if [ -n "$bench_cache_dir" ] && [ ! -w "$bench_cache_dir" ]; then
+        echo "Skipping warm bench save because ${bench_cache_dir} is not writable"
+        return
+    fi
+
+    local tmp_archive
+    tmp_archive="${bench_cache_archive}.${$}.tmp"
+
+    echo "Saving warm bench to ${bench_cache_archive}"
+    # Keep sites/common_site_config.json (the redis ports live there — dropping it makes the
+    # restore path fall back to a default redis port that bench start never bound, so reinstall
+    # fails with "redis ... connection refused"). Only the rebuildable sites/assets is excluded;
+    # restore_warm_bench runs `bench build` to regenerate it.
+    tar \
+        --use-compress-program="zstd -T0 -3" \
+        --exclude="frappe-bench/logs" \
+        --exclude="frappe-bench/sites/assets" \
+        -cf "$tmp_archive" \
+        -C ~ frappe-bench
+    mv "$tmp_archive" "$bench_cache_archive"
+}
 
 # ---------------------------------------------------------------------------
 # Phase 2 — bench init and site setup
 # ---------------------------------------------------------------------------
 
-bench init --skip-assets --frappe-path ~/frappe --python "$(which python)" frappe-bench
+install_whktml() {
+    # Re-use the .deb if the wkhtmltopdf cache step already restored it.
+    if [ ! -f "$wkhtmltox_deb" ]; then
+        wget -O "$wkhtmltox_deb" https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb
+    fi
+    sudo apt-get install -y "$wkhtmltox_deb"
+}
+if [ "${SKIP_WKHTMLTOX_SETUP:-0}" != "1" ]; then
+    install_whktml &
+    wkpid=$!
+else
+    wkpid=
+fi
 
-mkdir ~/frappe-bench/sites/test_site
+if ! restore_warm_bench; then
+    bench init --skip-assets --frappe-path ~/frappe --python "$(which python)" frappe-bench
+
+    cd ~/frappe-bench || exit
+
+    sed -i 's/watch:/# watch:/g' Procfile
+    sed -i 's/schedule:/# schedule:/g' Procfile
+    sed -i 's/socketio:/# socketio:/g' Procfile
+    sed -i 's/redis_socketio:/# redis_socketio:/g' Procfile
+
+    CI=Yes bench build --app frappe
+    save_warm_bench
+fi
+
+if [ -n "$wkpid" ]; then wait $wkpid; fi
+
+mkdir -p ~/frappe-bench/sites/test_site
 
 if [ "$DB" == "mariadb" ];then
     cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config_mariadb.json" ~/frappe-bench/sites/test_site/site_config.json
+    if [ "$db_host" != "127.0.0.1" ]; then
+        sed -i "s/\"db_host\": \"127.0.0.1\"/\"db_host\": \"${db_host}\"/" ~/frappe-bench/sites/test_site/site_config.json
+    fi
 else
     cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config_postgres.json" ~/frappe-bench/sites/test_site/site_config.json
 fi
 
 
 if [ "$DB" == "mariadb" ];then
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'"
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"
+    for _ in {1..60}; do
+        if mariadb-admin ping --host "$db_host" --port 3306 -u root -proot --silent; then
+            break
+        fi
+        sleep 1
+    done
+    mariadb-admin ping --host "$db_host" --port 3306 -u root -proot --silent
 
-    # Belt-and-suspenders: also set performance variables at runtime in case
-    # MARIADB_EXTRA_FLAGS was not honoured by the container image.
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot \
+    mariadb --host "$db_host" --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'"
+    mariadb --host "$db_host" --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'"
+
+    # Throwaway-DB durability tuning at runtime. (innodb_doublewrite is read-only on MariaDB
+    # 10.6, so it can't be disabled here — would need a server startup flag.)
+    mariadb --host "$db_host" --port 3306 -u root -proot \
         -e "SET GLOBAL innodb_flush_log_at_trx_commit=0; SET GLOBAL sync_binlog=0;"
 
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'"
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "CREATE DATABASE test_frappe"
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'"
+    # Opt-in DDL speedup: a shared tablespace avoids a create+fsync per DocType table during
+    # reinstall — a big win under disk contention. But ROW_FORMAT=DYNAMIC must be accepted in
+    # the system tablespace on this MariaDB. Enable with CI_INNODB_SHARED_TABLESPACE=1; if
+    # reinstall then errors on table creation, unset it (off by default — zero risk).
+    if [ "${CI_INNODB_SHARED_TABLESPACE:-0}" = "1" ]; then
+        mariadb --host "$db_host" --port 3306 -u root -proot -e "SET GLOBAL innodb_file_per_table=0;"
+    fi
 
-    mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "FLUSH PRIVILEGES"
+    mariadb --host "$db_host" --port 3306 -u root -proot -e "CREATE USER 'test_frappe'@'${db_user_host}' IDENTIFIED BY 'test_frappe'"
+    mariadb --host "$db_host" --port 3306 -u root -proot -e "CREATE DATABASE test_frappe"
+    mariadb --host "$db_host" --port 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'${db_user_host}'"
+
+    mariadb --host "$db_host" --port 3306 -u root -proot -e "FLUSH PRIVILEGES"
 fi
 
 if [ "$DB" == "postgres" ];then
     echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres;
     echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres;
-    # CI databases are disposable, so trade durability for speed: postgres fsyncs on every commit
-    # by default, which dominates a commit-heavy test suite. These are all reload-time settings
-    # (no restart needed). MariaDB CI is unaffected (DB != postgres).
-    echo "travis" | psql -h 127.0.0.1 -p 5432 -U postgres \
-        -c "ALTER SYSTEM SET synchronous_commit = 'off'" \
-        -c "ALTER SYSTEM SET fsync = 'off'" \
-        -c "ALTER SYSTEM SET full_page_writes = 'off'" \
-        -c "SELECT pg_reload_conf()";
 fi
 
-
-install_whktml() {
-    # Re-use the .deb if the wkhtmltopdf cache step already restored it.
-    if [ ! -f /tmp/wkhtmltox.deb ]; then
-        wget -O /tmp/wkhtmltox.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb
-    fi
-    sudo apt install /tmp/wkhtmltox.deb
-}
-install_whktml &
-wkpid=$!
-
-
 cd ~/frappe-bench || exit
 
-sed -i 's/watch:/# watch:/g' Procfile
-sed -i 's/schedule:/# schedule:/g' Procfile
-sed -i 's/socketio:/# socketio:/g' Procfile
-sed -i 's/redis_socketio:/# redis_socketio:/g' Procfile
+run_ci_step "Get payments app" bench get-app payments --branch develop
 
-bench get-app payments --branch develop
-bench get-app erpnext "${GITHUB_WORKSPACE}"
+# Opt-in: skip building erpnext's frontend assets. Server tests don't need them, but PDF
+# tests (print formats) do — they pass only if the PDF renderer ignores missing assets.
+# Enable with CI_SKIP_ERPNEXT_ASSETS=1 to test; if PDF tests fail, unset it.
+erpnext_get_app_args=()
+if [ "${CI_SKIP_ERPNEXT_ASSETS:-0}" = "1" ]; then erpnext_get_app_args=(--skip-assets); fi
+run_ci_step "Get erpnext app" bench get-app erpnext "${GITHUB_WORKSPACE}" "${erpnext_get_app_args[@]}"
 
-if [ "$TYPE" == "server" ]; then bench setup requirements --dev; fi
+if [ "$TYPE" == "server" ]; then run_ci_step "Setup dev requirements" bench setup requirements --dev; fi
 
-wait $wkpid
+bench start >> ~/frappe-bench/bench_start.log 2>&1 &
 
-bench start &>> ~/frappe-bench/bench_start.log &
-CI=Yes bench build --app frappe &
-bench --site test_site reinstall --yes
+# Under heavy concurrency, gunicorn's startup can delay redis coming up. reinstall and the
+# tests need redis, so wait for it (best-effort, bounded) instead of racing — contention
+# then slows the job rather than failing it.
+wait_for_redis() {
+    local cfg=~/frappe-bench/sites/common_site_config.json
+    [ -f "$cfg" ] || return 0
+    local ports port
+    ports=$(python - "$cfg" <<'PY'
+import json, re, sys
+try:
+    cfg = json.load(open(sys.argv[1]))
+except Exception:
+    sys.exit(0)
+for key in ("redis_cache", "redis_queue"):
+    match = re.search(r":(\d+)", str(cfg.get(key, "")))
+    if match:
+        print(match.group(1))
+PY
+)
+    for port in $ports; do
+        for _ in $(seq 1 120); do
+            if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then
+                exec 3>&- 3<&-
+                break
+            fi
+            sleep 1
+        done
+    done
+}
+wait_for_redis
+
+# Site setup. `bench reinstall` rebuilds the entire schema in Python (~1000 DocTypes) — the
+# CI bottleneck that DB tuning / tmpfs / faster cores couldn't move. Instead, restore a
+# pre-baked baseline (the DB engine loads it, no Python schema-build) and `migrate` to sync
+# only the drift since the baseline was built. The baseline is produced by
+# .github/helper/generate-ci-baseline.sh (run nightly / at image build) from a clean
+# reinstall on develop. Gated by CI_RESTORE_FROM_BACKUP so it A/Bs against plain reinstall;
+# falls back to reinstall if the baseline is missing or the restore fails.
+CI_BASELINE_BACKUP="${CI_BASELINE_BACKUP:-/opt/ci-baseline/test_site-database.sql.gz}"
+if [ "${CI_RESTORE_FROM_BACKUP:-0}" = "1" ] && [ -f "$CI_BASELINE_BACKUP" ]; then
+    if [ "$DB" == "mariadb" ]; then
+        db_root_args=(--db-root-username root --db-root-password root)
+    else
+        db_root_args=(--db-root-username postgres --db-root-password travis)
+    fi
+    if run_ci_step "Restore baseline test site" bench --site test_site --force restore "${db_root_args[@]}" "$CI_BASELINE_BACKUP"; then
+        run_ci_step "Migrate test site" bench --site test_site migrate
+    else
+        run_ci_step "Reinstall test site (baseline restore failed)" bench --site test_site reinstall --yes
+    fi
+else
+    run_ci_step "Reinstall test site" bench --site test_site reinstall --yes
+fi
+
+# Refresh the baseline backup from this freshly set-up site. Run a normal job (reinstall path)
+# with CI_GENERATE_BASELINE=1 and the baseline dir mounted read-write; install.sh captures the
+# DB dump to CI_BASELINE_BACKUP so future runs can restore it. Nightly is enough — bench migrate
+# absorbs intraday develop drift. To bake into the image instead, copy the produced .sql.gz in.
+if [ "${CI_GENERATE_BASELINE:-0}" = "1" ]; then
+    run_ci_step "Backup baseline test site" bench --site test_site backup
+    latest_backup=$(ls -t ~/frappe-bench/sites/test_site/private/backups/*-database.sql.gz | head -1)
+    mkdir -p "$(dirname "$CI_BASELINE_BACKUP")"
+    cp "$latest_backup" "$CI_BASELINE_BACKUP"
+    echo "Baseline written to $CI_BASELINE_BACKUP ($(du -h "$latest_backup" | cut -f1))"
+fi
diff --git a/.github/helper/start-db.sh b/.github/helper/start-db.sh
new file mode 100755
index 00000000000..6998adcfd91
--- /dev/null
+++ b/.github/helper/start-db.sh
@@ -0,0 +1,75 @@
+#!/bin/bash
+#
+# Run MariaDB INSIDE the runner container, on a datadir we control. Because the datadir can be
+# packaged into the bench artifact, test shards start an already-loaded server instead of
+# replaying a SQL dump (the ~60s hydrate restore). Each shard gets its own copy → isolation kept.
+#
+# CI_DB_DATADIR picks the path:
+#   - setup job:  /home/ci/db-data  (OUTSIDE the bench, so install.sh's `rm -rf ~/frappe-bench`
+#                 doesn't wipe it; it's moved into the bench just before packaging)
+#   - test shard: ~/frappe-bench/mariadb-data  (where the artifact untar'd it)
+#
+# Idempotent: inits a fresh datadir if absent (setup), else starts on the existing one (shards).
+#
+set -e
+
+ci_user="${ERPNEXT_CI_USER:-frappe}"
+
+# Re-exec as the ci user so mariadbd and the datadir are owned consistently (root mariadbd is
+# refused anyway). Mirrors install.sh's user switch.
+if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then
+    exec su -m "$ci_user" -s /bin/bash -c \
+        "ERPNEXT_CI_USER='$ci_user' CI_DB_DATADIR='${CI_DB_DATADIR:-}' bash '$0'"
+fi
+
+# --- PostgreSQL (GitHub-hosted CI): run in-runner on a PGDATA so it bakes into the artifact,
+# same idea as the mariadb datadir. Trust auth (throwaway CI) skips password setup; durability
+# off for speed. Postgres is preinstalled on ubuntu-latest under /usr/lib/postgresql/<ver>/bin.
+if [ "${DB:-mariadb}" = "postgres" ]; then
+    PG_BIN=$(ls -d /usr/lib/postgresql/*/bin 2>/dev/null | sort -V | tail -1)
+    [ -n "$PG_BIN" ] && export PATH="$PG_BIN:$PATH"
+    PGDATA="${CI_DB_DATADIR:-$HOME/frappe-bench/pgdata}"
+    if [ ! -d "$PGDATA/base" ]; then
+        initdb -D "$PGDATA" -U postgres --auth-local=trust --auth-host=trust >/dev/null
+        echo "host all all 127.0.0.1/32 trust" >> "$PGDATA/pg_hba.conf"
+    fi
+    pg_ctl -D "$PGDATA" -w -o "-p 5432 -c listen_addresses=127.0.0.1 -c unix_socket_directories=$PGDATA -c fsync=off -c synchronous_commit=off -c full_page_writes=off" start
+    echo "PostgreSQL up in-runner (pgdata=$PGDATA)"
+    exit 0
+fi
+
+# --- MariaDB ---
+DATADIR="${CI_DB_DATADIR:-$HOME/frappe-bench/mariadb-data}"
+SOCK="$DATADIR/mysqld.sock"
+fresh=0
+
+if [ ! -d "$DATADIR/mysql" ]; then
+    mkdir -p "$DATADIR"
+    mariadb-install-db --no-defaults --datadir="$DATADIR" \
+        --auth-root-authentication-method=normal --skip-test-db >/dev/null 2>&1
+    fresh=1
+fi
+
+# Throwaway-CI durability off; bind TCP 127.0.0.1:3306 so bench/install.sh connect as usual.
+mariadbd --no-defaults --datadir="$DATADIR" --socket="$SOCK" --pid-file="$DATADIR/mysqld.pid" \
+    --port=3306 --bind-address=127.0.0.1 \
+    --innodb-flush-log-at-trx-commit=0 --sync-binlog=0 --skip-log-bin \
+    > "$HOME/mariadb.log" 2>&1 &
+
+for _ in $(seq 1 60); do
+    mariadb-admin --socket="$SOCK" ping --silent 2>/dev/null && break
+    sleep 1
+done
+
+if [ "$fresh" = "1" ]; then
+    # A fresh datadir has only a password-less root@localhost. Give it the password install.sh
+    # uses, plus a TCP-reachable root@127.0.0.1, so the rest of install.sh works unchanged.
+    mariadb --no-defaults --socket="$SOCK" -u root <<'SQL'
+ALTER USER 'root'@'localhost' IDENTIFIED BY 'root';
+CREATE USER IF NOT EXISTS 'root'@'127.0.0.1' IDENTIFIED BY 'root';
+GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' WITH GRANT OPTION;
+FLUSH PRIVILEGES;
+SQL
+fi
+
+echo "MariaDB up in-container (datadir=$DATADIR, fresh=$fresh)"
diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 0a9b094bc51..fdeeaad48bd 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -31,51 +31,49 @@ on:
 
 permissions:
   contents: read
+  packages: read
 
 concurrency:
   group: server-mariadb-develop-${{ github.event_name }}-${{ github.event.number || github.event_name == 'workflow_dispatch' && github.run_id || '' }}
   cancel-in-progress: true
 
+# Shared across both jobs. Both run in the SAME CI image so the bench lives at the identical
+# path (/home/ci/frappe-bench) on the setup runner and the test shards — that's what makes the
+# packaged Python venv portable between them.
+env:
+  TZ: 'Asia/Kolkata'
+  DEBIAN_FRONTEND: noninteractive
+  NODE_ENV: "production"
+  WITH_COVERAGE: ${{ github.event_name != 'pull_request' }}
+  ERPNEXT_CI_USER: ci
+  PIP_CACHE_DIR: /home/ci/.cache/pip
+  npm_config_cache: /home/ci/.cache/npm
+  YARN_CACHE_FOLDER: /home/ci/.cache/yarn
+  UV_CACHE_DIR: /home/ci/.cache/uv
+
 jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 60
-    env:
-      TZ: 'Asia/Kolkata'
-      NODE_ENV: "production"
-      WITH_COVERAGE: ${{ github.event_name != 'pull_request' }}
-
-    strategy:
-      fail-fast: false
-
-      matrix:
-        container: [1, 2, 3, 4]
-
-    name: Python Unit Tests
-
-    services:
-      mysql:
-        image: mariadb:10.6
-        env:
-          TZ: 'Asia/Kolkata'
-          MARIADB_ROOT_PASSWORD: 'root'
-          # Disable durability guarantees that are unnecessary in a throwaway CI container.
-          # innodb_flush_log_at_trx_commit=0 avoids an fsync on every commit (biggest win).
-          # sync_binlog=0 skips binary-log syncs; innodb_doublewrite=0 skips the doublewrite buffer.
-          MARIADB_EXTRA_FLAGS: --innodb-flush-log-at-trx-commit=0 --sync-binlog=0 --innodb-doublewrite=0
-        ports:
-          - 3306:3306
-        options: --health-cmd="mariadb-admin ping" --health-interval=5s --health-timeout=2s --health-retries=3
-
+  # Build the bench (clone + pip + yarn + assets) and reinstall test_site ONCE, on a free
+  # GitHub-hosted runner, then publish the whole bench (with a DB dump baked in) as an artifact.
+  # The expensive, non-parallelisable work happens here exactly once instead of on every shard.
+  setup:
+    name: Build & reinstall (setup)
+    # Dedicated scale set (fat cpu request) so the build+reinstall runs at full speed, uncontended
+    # by the many thin test shards. Same CI image + /home/ci path + 127.0.0.1 DB as the shards,
+    # so the packaged bench (and its venv) transplants cleanly.
+    runs-on: erpnext-arc-setup
+    timeout-minutes: 40
+    container:
+      image: ghcr.io/mihir-kandoi/erpnext-ci-mariadb:py3.14-node24-arc5
+      credentials:
+        username: ${{ secrets.GHCR_USERNAME || github.actor }}
+        password: ${{ secrets.GHCR_TOKEN || github.token }}
+    defaults:
+      run:
+        shell: bash
     steps:
       - name: Clone
         uses: actions/checkout@v6
 
-      - name: Setup Python
-        uses: actions/setup-python@v6
-        with:
-          python-version: '3.14'
-
       - name: Check for valid Python & Merge Conflicts
         run: |
           python -m compileall -fq "${GITHUB_WORKSPACE}"
@@ -84,53 +82,17 @@ jobs:
               exit 1
           fi
 
-      - name: Setup Node
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-          check-latest: true
-
       - name: Add to Hosts
         run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
 
-      - name: Cache pip
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
-
-      - name: Cache node modules
-        uses: actions/cache@v4
+      # MariaDB runs in-container on a datadir OUTSIDE the bench, because install.sh's next step
+      # does `rm -rf ~/frappe-bench`. After the reinstall, the datadir is moved into the bench so
+      # it ships in the artifact — test shards then start an already-loaded server (no restore).
+      - name: Start DB
+        run: bash ${GITHUB_WORKSPACE}/.github/helper/start-db.sh
         env:
-          cache-name: cache-node-modules
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-build-${{ env.cache-name }}-
-            ${{ runner.os }}-build-
-            ${{ runner.os }}-
-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT
-
-      - uses: actions/cache@v4
-        id: yarn-cache
-        with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-yarn-
-
-      - name: Cache wkhtmltopdf
-        uses: actions/cache@v4
-        with:
-          path: /tmp/wkhtmltox.deb
-          key: wkhtmltox-0.12.6.1-2-jammy-amd64
+          SKIP_SYSTEM_SETUP: "1"
+          CI_DB_DATADIR: /home/ci/db-data
 
       - name: Install
         run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh
@@ -138,21 +100,91 @@ jobs:
           DB: mariadb
           TYPE: server
           FRAPPE_USER: ${{ github.event.inputs.user }}
-          FRAPPE_BRANCH: ${{ github.event.client_payload.sha || github.event.inputs.branch }}
+          FRAPPE_BRANCH: ${{ github.event.client_payload.sha || github.event.inputs.branch || 'develop' }}
+          DB_HOST: 127.0.0.1
+          DB_USER_HOST: '%'
+          WKHTMLTOX_DEB: /tmp/wkhtmltox.deb
+          SKIP_SYSTEM_SETUP: "1"
+          SKIP_WKHTMLTOX_SETUP: "1"
+
+      # Clean shutdown (consistent InnoDB datadir), then stage it inside the bench for packaging.
+      - name: Stop DB and stage datadir
+        run: |
+          mariadb-admin -h 127.0.0.1 -P 3306 -u root -proot shutdown || true
+          for _ in $(seq 1 30); do [ -f /home/ci/db-data/mysqld.pid ] || break; sleep 1; done
+          mv /home/ci/db-data /home/ci/frappe-bench/mariadb-data
+
+      # Package the whole bench (apps, venv, node_modules, sites, the DB dump, and hydrate.sh)
+      # into one artifact for the test shards to consume.
+      # Single-node hand-off: stage the bench on a node-local hostPath instead of round-tripping
+      # through GitHub artifact storage (~60s/shard). Setup and shards share the same disk, so
+      # the shards just untar it locally. NOTE: this assumes one node — a shard on a different
+      # node could not read this path (then you'd need GitHub artifacts or an NFS/RWX volume).
+      - name: Stage bench on node (hostPath)
+        run: |
+          cp "${GITHUB_WORKSPACE}/.github/helper/hydrate.sh" /home/ci/frappe-bench/hydrate.sh
+          cp "${GITHUB_WORKSPACE}/.github/helper/start-db.sh" /home/ci/frappe-bench/start-db.sh
+          mkdir -p /opt/ci-bench-staging
+          # self-clean: drop bench tars from runs older than 2h
+          find /opt/ci-bench-staging -maxdepth 1 -name '*.tar.gz' -mmin +120 -delete 2>/dev/null || true
+          # Exclude .git/node_modules; the mariadb-data datadir IS included (the pre-loaded DB).
+          tar czpf "/opt/ci-bench-staging/${GITHUB_RUN_ID}.tar.gz" -C /home/ci \
+            --exclude='.git' --exclude='node_modules' frappe-bench
+          ls -lh "/opt/ci-bench-staging/${GITHUB_RUN_ID}.tar.gz"
+
+  # Fan-out: each shard downloads the bench, untars it, restores the DB dump, and runs its
+  # slice of the suite. No clone, no build, no reinstall on the self-hosted boxes.
+  test:
+    name: Python Unit Tests (ARC)
+    needs: setup
+    runs-on: erpnext-arc
+    timeout-minutes: 60
+    container:
+      image: ghcr.io/mihir-kandoi/erpnext-ci-mariadb:py3.14-node24-arc5
+      credentials:
+        username: ${{ secrets.GHCR_USERNAME || github.actor }}
+        password: ${{ secrets.GHCR_TOKEN || github.token }}
+    defaults:
+      run:
+        shell: bash
+
+    strategy:
+      fail-fast: false
+      matrix:
+        container: [1, 2, 3, 4, 5, 6, 7, 8]
+
+    steps:
+      - name: Add to Hosts
+        run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts
+
+      # Read the bench straight from the node-local hostPath the setup job staged it on — no
+      # GitHub download. -p preserves the ci (uid 1001) ownership so bench runs as ci cleanly.
+      - name: Untar bench from node (hostPath)
+        run: |
+          tar xzpf "/opt/ci-bench-staging/${GITHUB_RUN_ID}.tar.gz" -C /home/ci
+          ls -ld /home/ci/frappe-bench
+
+      - name: Hydrate (DB up + restore + bench start)
+        run: bash /home/ci/frappe-bench/hydrate.sh
+        env:
+          DB_HOST: 127.0.0.1
+          SKIP_SYSTEM_SETUP: "1"
+          CI_BASELINE_BACKUP: /home/ci/frappe-bench/test_site-db.sql.gz
 
       - name: Run Tests
         run: |
+          su -m "${ERPNEXT_CI_USER:-frappe}" -s /bin/bash <<'EOF'
           cd ~/frappe-bench/
           coverage_flag=""
           if [ "$WITH_COVERAGE" = "true" ]; then coverage_flag="--with-coverage"; fi
           bench --site test_site run-parallel-tests --lightmode --app erpnext \
-            --total-builds ${{ strategy.job-total }} \
+            --total-builds 8 \
             --build-number ${{ matrix.container }} \
             $coverage_flag
+          EOF
         env:
           TYPE: server
 
-
       - name: Show bench output
         if: ${{ always() }}
         run: cat ~/frappe-bench/bench_start.log || true
@@ -162,12 +194,12 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: coverage-${{ matrix.container }}
-          path: /home/runner/frappe-bench/sites/coverage.xml
+          path: /home/ci/frappe-bench/sites/coverage.xml
 
   coverage:
     name: Coverage Wrap Up
-    needs: test
-    if: ${{ github.event_name != 'pull_request' }}
+    needs: [test]
+    if: ${{ always() && github.event_name != 'pull_request' && !cancelled() && !failure() }}
     runs-on: ubuntu-latest
     steps:
       - name: Clone

From 62c401badcdfff7ade2d767f08b40194cc0e6601 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Wed, 24 Jun 2026 11:54:50 +0530
Subject: [PATCH 02/11] ci(mariadb): revert to 4 shards + cleanup

---
 .github/helper/hydrate.sh                  |  8 ++---
 .github/helper/install.sh                  | 38 +++-------------------
 .github/workflows/server-tests-mariadb.yml |  4 +--
 3 files changed, 9 insertions(+), 41 deletions(-)

diff --git a/.github/helper/hydrate.sh b/.github/helper/hydrate.sh
index 4619d63dbeb..cadf5c0ac6d 100755
--- a/.github/helper/hydrate.sh
+++ b/.github/helper/hydrate.sh
@@ -4,21 +4,19 @@
 #
 # The bench (apps, venv, node_modules, sites) is already on disk at ~/frappe-bench — the
 # workflow untar'd it from the artifact the setup job built. So there is NO bench init, no
-# asset build, and no reinstall here: just bring the DB up and restore the dump the setup job
-# baked into the bench, then start bench so tests can run. Mirrors the DB + bench-start tail of
-# install.sh. The whole point is that the expensive work happened ONCE in the setup job.
+# asset build, and no reinstall here: just bring the DB up on the baked datadir and start redis
+# so tests can run. The whole point is that the expensive work happened ONCE in the setup job.
 #
 set -e
 
 ci_user="${ERPNEXT_CI_USER:-frappe}"
 db_host="${DB_HOST:-127.0.0.1}"
-dump="${CI_BASELINE_BACKUP:-/home/$ci_user/frappe-bench/test_site-db.sql.gz}"
 
 # Re-exec as the ci user (uid 1001) so bench/cache ownership matches the artifact, same as
 # install.sh. The workflow untar'd as root with -p, so the files are already owned by ci.
 if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then
     exec su -m "$ci_user" -s /bin/bash -c \
-        "ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' CI_BASELINE_BACKUP='$dump' bash '$0'"
+        "ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' bash '$0'"
 fi
 
 cd ~/frappe-bench
diff --git a/.github/helper/install.sh b/.github/helper/install.sh
index 74330abca16..83a6cd5c8ed 100644
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@@ -344,37 +344,7 @@ PY
 }
 wait_for_redis
 
-# Site setup. `bench reinstall` rebuilds the entire schema in Python (~1000 DocTypes) — the
-# CI bottleneck that DB tuning / tmpfs / faster cores couldn't move. Instead, restore a
-# pre-baked baseline (the DB engine loads it, no Python schema-build) and `migrate` to sync
-# only the drift since the baseline was built. The baseline is produced by
-# .github/helper/generate-ci-baseline.sh (run nightly / at image build) from a clean
-# reinstall on develop. Gated by CI_RESTORE_FROM_BACKUP so it A/Bs against plain reinstall;
-# falls back to reinstall if the baseline is missing or the restore fails.
-CI_BASELINE_BACKUP="${CI_BASELINE_BACKUP:-/opt/ci-baseline/test_site-database.sql.gz}"
-if [ "${CI_RESTORE_FROM_BACKUP:-0}" = "1" ] && [ -f "$CI_BASELINE_BACKUP" ]; then
-    if [ "$DB" == "mariadb" ]; then
-        db_root_args=(--db-root-username root --db-root-password root)
-    else
-        db_root_args=(--db-root-username postgres --db-root-password travis)
-    fi
-    if run_ci_step "Restore baseline test site" bench --site test_site --force restore "${db_root_args[@]}" "$CI_BASELINE_BACKUP"; then
-        run_ci_step "Migrate test site" bench --site test_site migrate
-    else
-        run_ci_step "Reinstall test site (baseline restore failed)" bench --site test_site reinstall --yes
-    fi
-else
-    run_ci_step "Reinstall test site" bench --site test_site reinstall --yes
-fi
-
-# Refresh the baseline backup from this freshly set-up site. Run a normal job (reinstall path)
-# with CI_GENERATE_BASELINE=1 and the baseline dir mounted read-write; install.sh captures the
-# DB dump to CI_BASELINE_BACKUP so future runs can restore it. Nightly is enough — bench migrate
-# absorbs intraday develop drift. To bake into the image instead, copy the produced .sql.gz in.
-if [ "${CI_GENERATE_BASELINE:-0}" = "1" ]; then
-    run_ci_step "Backup baseline test site" bench --site test_site backup
-    latest_backup=$(ls -t ~/frappe-bench/sites/test_site/private/backups/*-database.sql.gz | head -1)
-    mkdir -p "$(dirname "$CI_BASELINE_BACKUP")"
-    cp "$latest_backup" "$CI_BASELINE_BACKUP"
-    echo "Baseline written to $CI_BASELINE_BACKUP ($(du -h "$latest_backup" | cut -f1))"
-fi
+# Site setup: build the schema (~1000 DocTypes) into the DB. This is the single-threaded-Python
+# bottleneck, but the fan-out amortises it — it runs once here in the setup job, and the test
+# shards start the DB on the baked datadir instead of repeating the reinstall.
+run_ci_step "Reinstall test site" bench --site test_site reinstall --yes
diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index fdeeaad48bd..13de494e646 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -151,7 +151,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        container: [1, 2, 3, 4, 5, 6, 7, 8]
+        container: [1, 2, 3, 4]
 
     steps:
       - name: Add to Hosts
@@ -178,7 +178,7 @@ jobs:
           coverage_flag=""
           if [ "$WITH_COVERAGE" = "true" ]; then coverage_flag="--with-coverage"; fi
           bench --site test_site run-parallel-tests --lightmode --app erpnext \
-            --total-builds 8 \
+            --total-builds 4 \
             --build-number ${{ matrix.container }} \
             $coverage_flag
           EOF

From 69cb1121ed6750234dd0c4c4bc9b259dce1be5e0 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Wed, 24 Jun 2026 11:56:51 +0530
Subject: [PATCH 03/11] ci(mariadb): drop '(ARC)' from test job name

---
 .github/workflows/server-tests-mariadb.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 13de494e646..f4ccae2cc2a 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -135,7 +135,7 @@ jobs:
   # Fan-out: each shard downloads the bench, untars it, restores the DB dump, and runs its
   # slice of the suite. No clone, no build, no reinstall on the self-hosted boxes.
   test:
-    name: Python Unit Tests (ARC)
+    name: Python Unit Tests
     needs: setup
     runs-on: erpnext-arc
     timeout-minutes: 60

From b74abbb9c38103ba20d0bf58f3b5d62789844e81 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Wed, 24 Jun 2026 12:08:03 +0530
Subject: [PATCH 04/11] ci(mariadb): use org-owned ghcr.io/frappe image

---
 .github/workflows/server-tests-mariadb.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index f4ccae2cc2a..7e9fb72d6e9 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -63,7 +63,7 @@ jobs:
     runs-on: erpnext-arc-setup
     timeout-minutes: 40
     container:
-      image: ghcr.io/mihir-kandoi/erpnext-ci-mariadb:py3.14-node24-arc5
+      image: ghcr.io/frappe/erpnext-ci-mariadb:py3.14-node24
       credentials:
         username: ${{ secrets.GHCR_USERNAME || github.actor }}
         password: ${{ secrets.GHCR_TOKEN || github.token }}
@@ -140,7 +140,7 @@ jobs:
     runs-on: erpnext-arc
     timeout-minutes: 60
     container:
-      image: ghcr.io/mihir-kandoi/erpnext-ci-mariadb:py3.14-node24-arc5
+      image: ghcr.io/frappe/erpnext-ci-mariadb:py3.14-node24
       credentials:
         username: ${{ secrets.GHCR_USERNAME || github.actor }}
         password: ${{ secrets.GHCR_TOKEN || github.token }}

From 7cd1cca2edc1bb2b220caee48c52158a0e767493 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Wed, 24 Jun 2026 12:26:46 +0530
Subject: [PATCH 05/11] ci(mariadb): address greptile review

---
 .github/helper/start-db.sh                 | 6 +++++-
 .github/workflows/server-tests-mariadb.yml | 8 +++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/helper/start-db.sh b/.github/helper/start-db.sh
index 6998adcfd91..e99d44babe1 100755
--- a/.github/helper/start-db.sh
+++ b/.github/helper/start-db.sh
@@ -56,10 +56,14 @@ mariadbd --no-defaults --datadir="$DATADIR" --socket="$SOCK" --pid-file="$DATADI
     --innodb-flush-log-at-trx-commit=0 --sync-binlog=0 --skip-log-bin \
     > "$HOME/mariadb.log" 2>&1 &
 
+up=0
 for _ in $(seq 1 60); do
-    mariadb-admin --socket="$SOCK" ping --silent 2>/dev/null && break
+    if mariadb-admin --socket="$SOCK" ping --silent 2>/dev/null; then up=1; break; fi
     sleep 1
 done
+# Fail loudly instead of letting the loop fall through (exit 0 of the last `sleep`) into SQL that
+# would error with a vague socket-connection failure.
+[ "$up" = "1" ] || { echo "mariadbd did not come up on $SOCK"; cat "$HOME/mariadb.log" 2>/dev/null; exit 1; }
 
 if [ "$fresh" = "1" ]; then
     # A fresh datadir has only a password-less root@localhost. Give it the password install.sh
diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 7e9fb72d6e9..5c2996b9bd0 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -112,6 +112,9 @@ jobs:
         run: |
           mariadb-admin -h 127.0.0.1 -P 3306 -u root -proot shutdown || true
           for _ in $(seq 1 30); do [ -f /home/ci/db-data/mysqld.pid ] || break; sleep 1; done
+          # Don't bake a dirty datadir — fail if mariadbd didn't finish stopping, rather than ship
+          # an inconsistent datadir the shards would have to crash-recover.
+          [ -f /home/ci/db-data/mysqld.pid ] && { echo "mariadbd did not shut down cleanly"; exit 1; }
           mv /home/ci/db-data /home/ci/frappe-bench/mariadb-data
 
       # Package the whole bench (apps, venv, node_modules, sites, the DB dump, and hydrate.sh)
@@ -164,12 +167,11 @@ jobs:
           tar xzpf "/opt/ci-bench-staging/${GITHUB_RUN_ID}.tar.gz" -C /home/ci
           ls -ld /home/ci/frappe-bench
 
-      - name: Hydrate (DB up + restore + bench start)
+      - name: Hydrate (start DB on baked datadir + bench start)
         run: bash /home/ci/frappe-bench/hydrate.sh
         env:
           DB_HOST: 127.0.0.1
           SKIP_SYSTEM_SETUP: "1"
-          CI_BASELINE_BACKUP: /home/ci/frappe-bench/test_site-db.sql.gz
 
       - name: Run Tests
         run: |
@@ -178,7 +180,7 @@ jobs:
           coverage_flag=""
           if [ "$WITH_COVERAGE" = "true" ]; then coverage_flag="--with-coverage"; fi
           bench --site test_site run-parallel-tests --lightmode --app erpnext \
-            --total-builds 4 \
+            --total-builds ${{ strategy.job-total }} \
             --build-number ${{ matrix.container }} \
             $coverage_flag
           EOF

From eae05f19077a8b31d92e8430a5b2f21c9699ec1b Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Wed, 24 Jun 2026 12:37:48 +0530
Subject: [PATCH 06/11] ci: wait_for_redis fail-fast (greptile)

---
 .github/helper/install.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/helper/install.sh b/.github/helper/install.sh
index 83a6cd5c8ed..263c12555bb 100644
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@@ -333,13 +333,17 @@ for key in ("redis_cache", "redis_queue"):
 PY
 )
     for port in $ports; do
+        local up=0
         for _ in $(seq 1 120); do
             if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then
-                exec 3>&- 3<&-
+                exec 3>&- 3<&-; up=1
                 break
             fi
             sleep 1
         done
+        # Fail clearly instead of letting reinstall die later on a vague socket-connection error
+        # when redis never bound.
+        [ "$up" = "1" ] || { echo "redis did not come up on port $port"; return 1; }
     done
 }
 wait_for_redis

From dfdfcb8ca14a1cc135cde175f1214a56008f806e Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Thu, 25 Jun 2026 06:28:28 +0530
Subject: [PATCH 07/11] ci(mariadb): fix stale 'restore DB dump' fan-out
 comment

The shards start MariaDB on the baked datadir; there is no DB dump restore.
Aligns the overview comment with the corrected Hydrate step (greptile).
---
 .github/workflows/server-tests-mariadb.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 5c2996b9bd0..4b800c5ed09 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -135,8 +135,8 @@ jobs:
             --exclude='.git' --exclude='node_modules' frappe-bench
           ls -lh "/opt/ci-bench-staging/${GITHUB_RUN_ID}.tar.gz"
 
-  # Fan-out: each shard downloads the bench, untars it, restores the DB dump, and runs its
-  # slice of the suite. No clone, no build, no reinstall on the self-hosted boxes.
+  # Fan-out: each shard downloads the bench, untars it, starts MariaDB on the baked datadir, and
+  # runs its slice of the suite. No clone, no build, no reinstall, no DB dump restore on the shards.
   test:
     name: Python Unit Tests
     needs: setup

From 694f46f7f7fe94a28543c4b29f8c62d80875714c Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Thu, 25 Jun 2026 11:54:26 +0530
Subject: [PATCH 08/11] ci(mariadb): track the erpnext branch for frappe, drop
 hardcoded develop

Remove the `|| 'develop'` fallback on FRAPPE_BRANCH so install.sh resolves the
frappe framework branch from the git context (GITHUB_BASE_REF/GITHUB_REF), the
same as the Postgres workflow. This makes the workflow backportable unchanged:
on version-15-hotfix / version-16-hotfix it now clones the matching frappe
branch instead of develop.
---
 .github/workflows/server-tests-mariadb.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 4b800c5ed09..68304f5fefd 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -100,7 +100,7 @@ jobs:
           DB: mariadb
           TYPE: server
           FRAPPE_USER: ${{ github.event.inputs.user }}
-          FRAPPE_BRANCH: ${{ github.event.client_payload.sha || github.event.inputs.branch || 'develop' }}
+          FRAPPE_BRANCH: ${{ github.event.client_payload.sha || github.event.inputs.branch }}
           DB_HOST: 127.0.0.1
           DB_USER_HOST: '%'
           WKHTMLTOX_DEB: /tmp/wkhtmltox.deb

From beec05ce1c73b0c8946b17ef1a56f594e77fdeea Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Thu, 25 Jun 2026 12:03:07 +0530
Subject: [PATCH 09/11] ci: restore postgres durability-off settings in
 install.sh
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The fan-out moved fsync/synchronous_commit/full_page_writes=off into
start-db.sh startup flags, but the Postgres workflow runs a postgres:13.3
service container and calls install.sh directly — it never runs start-db.sh.
So those flags never reached the Postgres CI, regressing it to full durability
on a commit-heavy suite. Re-apply them via ALTER SYSTEM (reloadable) in the
DB == "postgres" path, where the service-container workflow executes. MariaDB
is unaffected (DB != postgres).
---
 .github/helper/install.sh | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/.github/helper/install.sh b/.github/helper/install.sh
index 263c12555bb..bb7c85d88e0 100644
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@@ -296,6 +296,15 @@ fi
 if [ "$DB" == "postgres" ];then
     echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres;
     echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres;
+
+    # Disposable CI DB: durability off for speed (postgres fsyncs every commit by default, which
+    # dominates a commit-heavy suite). All reloadable, no restart. The postgres workflow runs a
+    # service-container DB and never calls start-db.sh, so the flags must be applied here.
+    echo "travis" | psql -h 127.0.0.1 -p 5432 -U postgres \
+        -c "ALTER SYSTEM SET synchronous_commit = 'off'" \
+        -c "ALTER SYSTEM SET fsync = 'off'" \
+        -c "ALTER SYSTEM SET full_page_writes = 'off'" \
+        -c "SELECT pg_reload_conf()";
 fi
 
 cd ~/frappe-bench || exit

From 368ea75e3856258a518ff5357a0b3059d2fb5ab9 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Thu, 25 Jun 2026 13:24:15 +0530
Subject: [PATCH 10/11] ci: address self-review findings

- Wire up warm-bench: set BENCH_CACHE_DIR on the setup job so the bench-base
  cache actually activates (was inert with no dir set, so every run did a
  full bench init). Lives on the node-local bench-staging hostPath; any
  miss/failure still falls back to a full init.
- run_ci_step: capture the timeout exit code under `set -e` (the previous
  `timeout ...` + `ec=$?` aborted at the timeout line on failure, skipping
  ::endgroup:: and the exit-code return).
- Raise the per-step timeout 600s -> 1800s so a contended reinstall isn't
  killed before the 40-min job timeout.
- Propagate DB through the su re-exec in start-db.sh / hydrate.sh so a
  DB=postgres invocation can't silently fall back to the mariadb branch.
- Simplify the coverage job `if` to the equivalent plain non-PR gate.
---
 .github/helper/hydrate.sh                  | 2 +-
 .github/helper/install.sh                  | 4 ++--
 .github/helper/start-db.sh                 | 2 +-
 .github/workflows/server-tests-mariadb.yml | 3 ++-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/helper/hydrate.sh b/.github/helper/hydrate.sh
index cadf5c0ac6d..1372bc393a3 100755
--- a/.github/helper/hydrate.sh
+++ b/.github/helper/hydrate.sh
@@ -16,7 +16,7 @@ db_host="${DB_HOST:-127.0.0.1}"
 # install.sh. The workflow untar'd as root with -p, so the files are already owned by ci.
 if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then
     exec su -m "$ci_user" -s /bin/bash -c \
-        "ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' bash '$0'"
+        "ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' DB='${DB:-}' bash '$0'"
 fi
 
 cd ~/frappe-bench
diff --git a/.github/helper/install.sh b/.github/helper/install.sh
index bb7c85d88e0..34e777506c9 100644
--- a/.github/helper/install.sh
+++ b/.github/helper/install.sh
@@ -74,8 +74,8 @@ run_ci_step() {
 
     echo "::group::${label}"
     date -u
-    timeout --foreground "${CI_INSTALL_STEP_TIMEOUT:-600}" "$@"
-    local exit_code=$?
+    local exit_code=0
+    timeout --foreground "${CI_INSTALL_STEP_TIMEOUT:-1800}" "$@" || exit_code=$?
     date -u
     echo "::endgroup::"
     return "$exit_code"
diff --git a/.github/helper/start-db.sh b/.github/helper/start-db.sh
index e99d44babe1..507c33bb7e4 100755
--- a/.github/helper/start-db.sh
+++ b/.github/helper/start-db.sh
@@ -19,7 +19,7 @@ ci_user="${ERPNEXT_CI_USER:-frappe}"
 # refused anyway). Mirrors install.sh's user switch.
 if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then
     exec su -m "$ci_user" -s /bin/bash -c \
-        "ERPNEXT_CI_USER='$ci_user' CI_DB_DATADIR='${CI_DB_DATADIR:-}' bash '$0'"
+        "ERPNEXT_CI_USER='$ci_user' CI_DB_DATADIR='${CI_DB_DATADIR:-}' DB='${DB:-}' bash '$0'"
 fi
 
 # --- PostgreSQL (GitHub-hosted CI): run in-runner on a PGDATA so it bakes into the artifact,
diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 68304f5fefd..680639ab6ea 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -106,6 +106,7 @@ jobs:
           WKHTMLTOX_DEB: /tmp/wkhtmltox.deb
           SKIP_SYSTEM_SETUP: "1"
           SKIP_WKHTMLTOX_SETUP: "1"
+          BENCH_CACHE_DIR: /opt/ci-bench-staging/base-cache
 
       # Clean shutdown (consistent InnoDB datadir), then stage it inside the bench for packaging.
       - name: Stop DB and stage datadir
@@ -201,7 +202,7 @@ jobs:
   coverage:
     name: Coverage Wrap Up
     needs: [test]
-    if: ${{ always() && github.event_name != 'pull_request' && !cancelled() && !failure() }}
+    if: ${{ github.event_name != 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
       - name: Clone

From f3315ecb340f9dae77bf817709d9ca6869e3b429 Mon Sep 17 00:00:00 2001
From: Mihir Kandoi <kandoimihir@gmail.com>
Date: Thu, 25 Jun 2026 13:47:54 +0530
Subject: [PATCH 11/11] ci(mariadb): un-wire warm-bench (no measurable gain)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Measured A/B on the self-hosted setup: warm-bench restore 85s vs full
bench init 82s — no gain (slightly slower). bench init is already fast
because the uv/pip caches are mounted warm, so the cache only replaces a
~13s init while adding a ~200MB untar and still running bench build.

Drop BENCH_CACHE_DIR so warm-bench stays inert (the helper functions
remain, matching develop's install.sh).
---
 .github/workflows/server-tests-mariadb.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/server-tests-mariadb.yml b/.github/workflows/server-tests-mariadb.yml
index 680639ab6ea..5c417fb9137 100644
--- a/.github/workflows/server-tests-mariadb.yml
+++ b/.github/workflows/server-tests-mariadb.yml
@@ -106,7 +106,6 @@ jobs:
           WKHTMLTOX_DEB: /tmp/wkhtmltox.deb
           SKIP_SYSTEM_SETUP: "1"
           SKIP_WKHTMLTOX_SETUP: "1"
-          BENCH_CACHE_DIR: /opt/ci-bench-staging/base-cache
 
       # Clean shutdown (consistent InnoDB datadir), then stage it inside the bench for packaging.
       - name: Stop DB and stage datadir