From 8fd08136141d545f1fcfd53accb499b80e8bb5b2 Mon Sep 17 00:00:00 2001 From: Mihir Kandoi Date: Wed, 24 Jun 2026 10:39:44 +0530 Subject: [PATCH] ci(postgres): scheduled fan-out Postgres CI (daily 3am IST + 'postgres' label gate) --- .github/helper/hydrate.sh | 74 ++++ .github/helper/install.sh | 367 +++++++++++++++++--- .github/helper/start-db.sh | 75 ++++ .github/workflows/server-tests-postgres.yml | 260 +++++++------- 4 files changed, 592 insertions(+), 184 deletions(-) create mode 100755 .github/helper/hydrate.sh create mode 100755 .github/helper/start-db.sh diff --git a/.github/helper/hydrate.sh b/.github/helper/hydrate.sh new file mode 100755 index 00000000000..4619d63dbeb --- /dev/null +++ b/.github/helper/hydrate.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Hydrate a test shard from the setup job's artifact. +# +# The bench (apps, venv, node_modules, sites) is already on disk at ~/frappe-bench — the +# workflow untar'd it from the artifact the setup job built. So there is NO bench init, no +# asset build, and no reinstall here: just bring the DB up and restore the dump the setup job +# baked into the bench, then start bench so tests can run. Mirrors the DB + bench-start tail of +# install.sh. The whole point is that the expensive work happened ONCE in the setup job. +# +set -e + +ci_user="${ERPNEXT_CI_USER:-frappe}" +db_host="${DB_HOST:-127.0.0.1}" +dump="${CI_BASELINE_BACKUP:-/home/$ci_user/frappe-bench/test_site-db.sql.gz}" + +# Re-exec as the ci user (uid 1001) so bench/cache ownership matches the artifact, same as +# install.sh. The workflow untar'd as root with -p, so the files are already owned by ci. +if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then + exec su -m "$ci_user" -s /bin/bash -c \ + "ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' CI_BASELINE_BACKUP='$dump' bash '$0'" +fi + +cd ~/frappe-bench + +# Start the DB on the datadir baked into the artifact. It's already populated (the setup job +# reinstalled into this very datadir), so there is NO restore — the server comes up on the +# existing files. This is what replaces the per-shard SQL replay. +bash ~/frappe-bench/start-db.sh + +# Bring up redis (lightmode unit tests need cache + queue). In the self-hosted container we use the +# full `bench start` (web/workers too, like install.sh). On the bare GitHub Postgres shard +# `bench start` (honcho) lagged — it blocks the redis procs behind web/worker procs the lightmode +# suite never uses, so the wait below burned its full timeout (~4m). There, start the two redis +# instances directly: fast and deterministic. +if [ "${DB:-mariadb}" = "postgres" ]; then + # Start redis directly as daemons — reliable and persists across steps. Do NOT route it through + # `bench start`: honcho tears the whole process group down if any one Procfile proc dies on the + # bare shard, which took redis with it (redis @ 13000 refused in Run Tests). Keeping redis + # independent is what makes it survive. The web server (for PDF tests) is NOT started here — a + # backgrounded server doesn't survive into the next step; it's started inside the Run Tests step. + for conf in redis_cache redis_queue; do + [ -f ~/frappe-bench/config/$conf.conf ] && redis-server ~/frappe-bench/config/$conf.conf --daemonize yes + done +else + bench start >> ~/frappe-bench/bench_start.log 2>&1 & +fi + +# Wait for redis, failing fast instead of silently burning minutes if it never comes up. +cfg=~/frappe-bench/sites/common_site_config.json +if [ -f "$cfg" ]; then + ports=$(python - "$cfg" <<'PY' +import json, re, sys +try: + cfg = json.load(open(sys.argv[1])) +except Exception: + sys.exit(0) +for key in ("redis_cache", "redis_queue"): + m = re.search(r":(\d+)", str(cfg.get(key, ""))) + if m: + print(m.group(1)) +PY +) + for port in $ports; do + up=0 + for _ in $(seq 1 60); do + if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then exec 3>&- 3<&-; up=1; break; fi + sleep 1 + done + [ "$up" = "1" ] || { echo "redis did not come up on port $port"; exit 1; } + done +fi + +echo "Hydrated: DB up on baked datadir, redis up — ready for tests." diff --git a/.github/helper/install.sh b/.github/helper/install.sh index b5f9b9e364b..74330abca16 100644 --- a/.github/helper/install.sh +++ b/.github/helper/install.sh @@ -7,21 +7,106 @@ cd ~ || exit githubbranch=${GITHUB_BASE_REF:-${GITHUB_REF##*/}} frappeuser=${FRAPPE_USER:-"frappe"} frappecommitish=${FRAPPE_BRANCH:-$githubbranch} +db_host=${DB_HOST:-"127.0.0.1"} +db_user_host=${DB_USER_HOST:-"localhost"} +wkhtmltox_deb=${WKHTMLTOX_DEB:-"/tmp/wkhtmltox.deb"} +bench_cache_dir=${BENCH_CACHE_DIR:-} + +run_as_ci_user_if_needed() { + if [ "$(id -u)" != "0" ] || [ "${SKIP_SYSTEM_SETUP:-0}" != "1" ] || [ "${ERPNEXT_CI_NON_ROOT:-0}" = "1" ]; then + return + fi + + local missing_packages=() + if ! command -v pkg-config >/dev/null 2>&1; then + missing_packages+=("pkg-config") + fi + if ! command -v mariadb_config >/dev/null 2>&1 && ! command -v mysql_config >/dev/null 2>&1; then + missing_packages+=("libmariadb-dev") + fi + if ! command -v crontab >/dev/null 2>&1; then + missing_packages+=("cron") + fi + + if [ "${#missing_packages[@]}" -gt 0 ]; then + apt-get update + apt-get install -y --no-install-recommends "${missing_packages[@]}" + fi + + local ci_user="${ERPNEXT_CI_USER:-frappe}" + + if ! id "$ci_user" >/dev/null 2>&1; then + useradd --home-dir "$HOME" --no-create-home --shell /bin/bash "$ci_user" + fi + + rm -rf ~/frappe ~/frappe-bench + + local ci_dirs=( + "$HOME" + "$GITHUB_WORKSPACE" + "$HOME/.cache" + "${PIP_CACHE_DIR:-$HOME/.cache/pip}" + "${npm_config_cache:-$HOME/.npm}" + "${YARN_CACHE_FOLDER:-$HOME/.cache/yarn}" + "$HOME/.yarn" + "${UV_CACHE_DIR:-$HOME/.cache/uv}" + "$(dirname "$wkhtmltox_deb")" + ) + if [ -n "$bench_cache_dir" ]; then + ci_dirs+=("$bench_cache_dir") + fi + + # Create + own (non-recursively) the home/cache/workspace dirs before dropping to + # the ci user. We deliberately do NOT wipe the yarn/uv caches here so a persistent + # cache (mounted volume or baked image layer) stays warm across runs. + mkdir -p "${ci_dirs[@]}" "$HOME/.yarn" + chown "$ci_user:$ci_user" "${ci_dirs[@]}" "$HOME/.yarn" + + export ERPNEXT_CI_NON_ROOT=1 + exec su -m "$ci_user" -s /bin/bash -c "cd '$HOME' && bash '$GITHUB_WORKSPACE/.github/helper/install.sh'" +} + +run_as_ci_user_if_needed + +run_ci_step() { + local label=$1 + shift + + echo "::group::${label}" + date -u + timeout --foreground "${CI_INSTALL_STEP_TIMEOUT:-600}" "$@" + local exit_code=$? + date -u + echo "::endgroup::" + return "$exit_code" +} + +if [ -n "${GITHUB_WORKSPACE:-}" ]; then + git config --global --add safe.directory "$GITHUB_WORKSPACE" || true + git config --global --add safe.directory "$GITHUB_WORKSPACE/.git" || true +fi + +rm -rf ~/frappe ~/frappe-bench # --------------------------------------------------------------------------- # Phase 1 — parallelise the three slow, independent setup steps: # a) system packages b) frappe-bench pip install c) frappe git fetch # --------------------------------------------------------------------------- -sudo apt update +if [ "${SKIP_SYSTEM_SETUP:-0}" != "1" ]; then + sudo apt-get update -# apt remove/install must run sequentially but can overlap with pip and git. -sudo apt remove mysql-server mysql-client -sudo apt install libcups2-dev redis-server mariadb-client libmariadb-dev & -apt_pid=$! + # apt remove/install must run sequentially but can overlap with pip and git. + sudo apt-get remove -y mysql-server mysql-client + sudo apt-get install -y libcups2-dev redis-server mariadb-client libmariadb-dev & + apt_pid=$! -pip install frappe-bench & -pip_pid=$! + pip install frappe-bench & + pip_pid=$! +else + apt_pid= + pip_pid= +fi mkdir frappe ( @@ -32,84 +117,264 @@ mkdir frappe ) & clone_pid=$! -wait $apt_pid -wait $pip_pid +if [ -n "$apt_pid" ]; then wait $apt_pid; fi +if [ -n "$pip_pid" ]; then wait $pip_pid; fi wait $clone_pid pushd frappe git checkout FETCH_HEAD popd +frappe_sha=$(git -C frappe rev-parse HEAD) + +get_bench_cache_archive() { + if [ -z "$bench_cache_dir" ]; then + return + fi + + mkdir -p "$bench_cache_dir" + + # Keyed on tool versions only (NOT the frappe SHA): any recent base bench works, because + # restore_warm_bench fast-forwards it to the exact live develop SHA. This is what lets a + # constantly-moving develop still hit the cache. + local cache_key + cache_key=$( + { + uname -m + python --version + node --version + bench --version + } | sha256sum | awk '{print $1}' + ) + + echo "${bench_cache_dir}/frappe-bench-base-${cache_key}.tar.zst" +} + +restore_warm_bench() { + bench_cache_archive=$(get_bench_cache_archive) + [ -n "$bench_cache_archive" ] && [ -f "$bench_cache_archive" ] || return 1 + + echo "Restoring base bench from ${bench_cache_archive}" + tar --use-compress-program=unzstd -xf "$bench_cache_archive" -C ~ || return 1 + [ -d ~/frappe-bench/apps/frappe/.git ] || return 1 + mkdir -p ~/frappe-bench/sites ~/frappe-bench/logs + [ -f ~/frappe-bench/sites/apps.txt ] || printf "frappe\n" > ~/frappe-bench/sites/apps.txt + [ -f ~/frappe-bench/sites/common_site_config.json ] || printf "{}\n" > ~/frappe-bench/sites/common_site_config.json + + # Fast-forward the restored frappe to the EXACT live develop SHA fetched in phase 1, then + # rebuild only what changed. The editable install means the venv tracks the new code with + # no reinstall. Any failure returns non-zero so the caller falls back to a full bench init. + if ! ( + cd ~/frappe-bench/apps/frappe || exit 1 + # Phase 1 already fetched ~/frappe to the exact live develop SHA. Fetch that commit + # straight from it (bench init names the remote 'upstream', not 'origin', and points + # it at this local clone — so a plain `git fetch origin` does not work). + git fetch --no-tags "$HOME/frappe" HEAD || exit 1 + git checkout --force FETCH_HEAD || exit 1 + ); then + echo "Fast-forward to ${frappe_sha} failed; falling back to full init" + rm -rf ~/frappe-bench + return 1 + fi + + # Pick up any frappe dependency changes since the base was built (cached → fast if none), + # so a develop commit that bumped requirements doesn't leave a stale venv. + if ! ~/frappe-bench/env/bin/python -m pip install -q -e ~/frappe-bench/apps/frappe; then + echo "frappe dependency refresh failed; falling back to full init" + rm -rf ~/frappe-bench + return 1 + fi + + ( cd ~/frappe-bench && CI=Yes bench build --app frappe ) || { rm -rf ~/frappe-bench; return 1; } + return 0 +} + +save_warm_bench() { + if [ -z "${bench_cache_archive:-}" ] || [ -f "$bench_cache_archive" ]; then + return + fi + + if [ -n "$bench_cache_dir" ] && [ ! -w "$bench_cache_dir" ]; then + echo "Skipping warm bench save because ${bench_cache_dir} is not writable" + return + fi + + local tmp_archive + tmp_archive="${bench_cache_archive}.${$}.tmp" + + echo "Saving warm bench to ${bench_cache_archive}" + # Keep sites/common_site_config.json (the redis ports live there — dropping it makes the + # restore path fall back to a default redis port that bench start never bound, so reinstall + # fails with "redis ... connection refused"). Only the rebuildable sites/assets is excluded; + # restore_warm_bench runs `bench build` to regenerate it. + tar \ + --use-compress-program="zstd -T0 -3" \ + --exclude="frappe-bench/logs" \ + --exclude="frappe-bench/sites/assets" \ + -cf "$tmp_archive" \ + -C ~ frappe-bench + mv "$tmp_archive" "$bench_cache_archive" +} # --------------------------------------------------------------------------- # Phase 2 — bench init and site setup # --------------------------------------------------------------------------- -bench init --skip-assets --frappe-path ~/frappe --python "$(which python)" frappe-bench +install_whktml() { + # Re-use the .deb if the wkhtmltopdf cache step already restored it. + if [ ! -f "$wkhtmltox_deb" ]; then + wget -O "$wkhtmltox_deb" https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb + fi + sudo apt-get install -y "$wkhtmltox_deb" +} +if [ "${SKIP_WKHTMLTOX_SETUP:-0}" != "1" ]; then + install_whktml & + wkpid=$! +else + wkpid= +fi -mkdir ~/frappe-bench/sites/test_site +if ! restore_warm_bench; then + bench init --skip-assets --frappe-path ~/frappe --python "$(which python)" frappe-bench + + cd ~/frappe-bench || exit + + sed -i 's/watch:/# watch:/g' Procfile + sed -i 's/schedule:/# schedule:/g' Procfile + sed -i 's/socketio:/# socketio:/g' Procfile + sed -i 's/redis_socketio:/# redis_socketio:/g' Procfile + + CI=Yes bench build --app frappe + save_warm_bench +fi + +if [ -n "$wkpid" ]; then wait $wkpid; fi + +mkdir -p ~/frappe-bench/sites/test_site if [ "$DB" == "mariadb" ];then cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config_mariadb.json" ~/frappe-bench/sites/test_site/site_config.json + if [ "$db_host" != "127.0.0.1" ]; then + sed -i "s/\"db_host\": \"127.0.0.1\"/\"db_host\": \"${db_host}\"/" ~/frappe-bench/sites/test_site/site_config.json + fi else cp -r "${GITHUB_WORKSPACE}/.github/helper/site_config_postgres.json" ~/frappe-bench/sites/test_site/site_config.json fi if [ "$DB" == "mariadb" ];then - mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" - mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" + for _ in {1..60}; do + if mariadb-admin ping --host "$db_host" --port 3306 -u root -proot --silent; then + break + fi + sleep 1 + done + mariadb-admin ping --host "$db_host" --port 3306 -u root -proot --silent - # Belt-and-suspenders: also set performance variables at runtime in case - # MARIADB_EXTRA_FLAGS was not honoured by the container image. - mariadb --host 127.0.0.1 --port 3306 -u root -proot \ + mariadb --host "$db_host" --port 3306 -u root -proot -e "SET GLOBAL character_set_server = 'utf8mb4'" + mariadb --host "$db_host" --port 3306 -u root -proot -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci'" + + # Throwaway-DB durability tuning at runtime. (innodb_doublewrite is read-only on MariaDB + # 10.6, so it can't be disabled here — would need a server startup flag.) + mariadb --host "$db_host" --port 3306 -u root -proot \ -e "SET GLOBAL innodb_flush_log_at_trx_commit=0; SET GLOBAL sync_binlog=0;" - mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "CREATE USER 'test_frappe'@'localhost' IDENTIFIED BY 'test_frappe'" - mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "CREATE DATABASE test_frappe" - mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'localhost'" + # Opt-in DDL speedup: a shared tablespace avoids a create+fsync per DocType table during + # reinstall — a big win under disk contention. But ROW_FORMAT=DYNAMIC must be accepted in + # the system tablespace on this MariaDB. Enable with CI_INNODB_SHARED_TABLESPACE=1; if + # reinstall then errors on table creation, unset it (off by default — zero risk). + if [ "${CI_INNODB_SHARED_TABLESPACE:-0}" = "1" ]; then + mariadb --host "$db_host" --port 3306 -u root -proot -e "SET GLOBAL innodb_file_per_table=0;" + fi - mariadb --host 127.0.0.1 --port 3306 -u root -proot -e "FLUSH PRIVILEGES" + mariadb --host "$db_host" --port 3306 -u root -proot -e "CREATE USER 'test_frappe'@'${db_user_host}' IDENTIFIED BY 'test_frappe'" + mariadb --host "$db_host" --port 3306 -u root -proot -e "CREATE DATABASE test_frappe" + mariadb --host "$db_host" --port 3306 -u root -proot -e "GRANT ALL PRIVILEGES ON \`test_frappe\`.* TO 'test_frappe'@'${db_user_host}'" + + mariadb --host "$db_host" --port 3306 -u root -proot -e "FLUSH PRIVILEGES" fi if [ "$DB" == "postgres" ];then echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE DATABASE test_frappe" -U postgres; echo "travis" | psql -h 127.0.0.1 -p 5432 -c "CREATE USER test_frappe WITH PASSWORD 'test_frappe'" -U postgres; - # CI databases are disposable, so trade durability for speed: postgres fsyncs on every commit - # by default, which dominates a commit-heavy test suite. These are all reload-time settings - # (no restart needed). MariaDB CI is unaffected (DB != postgres). - echo "travis" | psql -h 127.0.0.1 -p 5432 -U postgres \ - -c "ALTER SYSTEM SET synchronous_commit = 'off'" \ - -c "ALTER SYSTEM SET fsync = 'off'" \ - -c "ALTER SYSTEM SET full_page_writes = 'off'" \ - -c "SELECT pg_reload_conf()"; fi - -install_whktml() { - # Re-use the .deb if the wkhtmltopdf cache step already restored it. - if [ ! -f /tmp/wkhtmltox.deb ]; then - wget -O /tmp/wkhtmltox.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb - fi - sudo apt install /tmp/wkhtmltox.deb -} -install_whktml & -wkpid=$! - - cd ~/frappe-bench || exit -sed -i 's/watch:/# watch:/g' Procfile -sed -i 's/schedule:/# schedule:/g' Procfile -sed -i 's/socketio:/# socketio:/g' Procfile -sed -i 's/redis_socketio:/# redis_socketio:/g' Procfile +run_ci_step "Get payments app" bench get-app payments --branch develop -bench get-app payments --branch develop -bench get-app erpnext "${GITHUB_WORKSPACE}" +# Opt-in: skip building erpnext's frontend assets. Server tests don't need them, but PDF +# tests (print formats) do — they pass only if the PDF renderer ignores missing assets. +# Enable with CI_SKIP_ERPNEXT_ASSETS=1 to test; if PDF tests fail, unset it. +erpnext_get_app_args=() +if [ "${CI_SKIP_ERPNEXT_ASSETS:-0}" = "1" ]; then erpnext_get_app_args=(--skip-assets); fi +run_ci_step "Get erpnext app" bench get-app erpnext "${GITHUB_WORKSPACE}" "${erpnext_get_app_args[@]}" -if [ "$TYPE" == "server" ]; then bench setup requirements --dev; fi +if [ "$TYPE" == "server" ]; then run_ci_step "Setup dev requirements" bench setup requirements --dev; fi -wait $wkpid +bench start >> ~/frappe-bench/bench_start.log 2>&1 & -bench start &>> ~/frappe-bench/bench_start.log & -CI=Yes bench build --app frappe & -bench --site test_site reinstall --yes +# Under heavy concurrency, gunicorn's startup can delay redis coming up. reinstall and the +# tests need redis, so wait for it (best-effort, bounded) instead of racing — contention +# then slows the job rather than failing it. +wait_for_redis() { + local cfg=~/frappe-bench/sites/common_site_config.json + [ -f "$cfg" ] || return 0 + local ports port + ports=$(python - "$cfg" <<'PY' +import json, re, sys +try: + cfg = json.load(open(sys.argv[1])) +except Exception: + sys.exit(0) +for key in ("redis_cache", "redis_queue"): + match = re.search(r":(\d+)", str(cfg.get(key, ""))) + if match: + print(match.group(1)) +PY +) + for port in $ports; do + for _ in $(seq 1 120); do + if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + exec 3>&- 3<&- + break + fi + sleep 1 + done + done +} +wait_for_redis + +# Site setup. `bench reinstall` rebuilds the entire schema in Python (~1000 DocTypes) — the +# CI bottleneck that DB tuning / tmpfs / faster cores couldn't move. Instead, restore a +# pre-baked baseline (the DB engine loads it, no Python schema-build) and `migrate` to sync +# only the drift since the baseline was built. The baseline is produced by +# .github/helper/generate-ci-baseline.sh (run nightly / at image build) from a clean +# reinstall on develop. Gated by CI_RESTORE_FROM_BACKUP so it A/Bs against plain reinstall; +# falls back to reinstall if the baseline is missing or the restore fails. +CI_BASELINE_BACKUP="${CI_BASELINE_BACKUP:-/opt/ci-baseline/test_site-database.sql.gz}" +if [ "${CI_RESTORE_FROM_BACKUP:-0}" = "1" ] && [ -f "$CI_BASELINE_BACKUP" ]; then + if [ "$DB" == "mariadb" ]; then + db_root_args=(--db-root-username root --db-root-password root) + else + db_root_args=(--db-root-username postgres --db-root-password travis) + fi + if run_ci_step "Restore baseline test site" bench --site test_site --force restore "${db_root_args[@]}" "$CI_BASELINE_BACKUP"; then + run_ci_step "Migrate test site" bench --site test_site migrate + else + run_ci_step "Reinstall test site (baseline restore failed)" bench --site test_site reinstall --yes + fi +else + run_ci_step "Reinstall test site" bench --site test_site reinstall --yes +fi + +# Refresh the baseline backup from this freshly set-up site. Run a normal job (reinstall path) +# with CI_GENERATE_BASELINE=1 and the baseline dir mounted read-write; install.sh captures the +# DB dump to CI_BASELINE_BACKUP so future runs can restore it. Nightly is enough — bench migrate +# absorbs intraday develop drift. To bake into the image instead, copy the produced .sql.gz in. +if [ "${CI_GENERATE_BASELINE:-0}" = "1" ]; then + run_ci_step "Backup baseline test site" bench --site test_site backup + latest_backup=$(ls -t ~/frappe-bench/sites/test_site/private/backups/*-database.sql.gz | head -1) + mkdir -p "$(dirname "$CI_BASELINE_BACKUP")" + cp "$latest_backup" "$CI_BASELINE_BACKUP" + echo "Baseline written to $CI_BASELINE_BACKUP ($(du -h "$latest_backup" | cut -f1))" +fi diff --git a/.github/helper/start-db.sh b/.github/helper/start-db.sh new file mode 100755 index 00000000000..6998adcfd91 --- /dev/null +++ b/.github/helper/start-db.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# +# Run MariaDB INSIDE the runner container, on a datadir we control. Because the datadir can be +# packaged into the bench artifact, test shards start an already-loaded server instead of +# replaying a SQL dump (the ~60s hydrate restore). Each shard gets its own copy → isolation kept. +# +# CI_DB_DATADIR picks the path: +# - setup job: /home/ci/db-data (OUTSIDE the bench, so install.sh's `rm -rf ~/frappe-bench` +# doesn't wipe it; it's moved into the bench just before packaging) +# - test shard: ~/frappe-bench/mariadb-data (where the artifact untar'd it) +# +# Idempotent: inits a fresh datadir if absent (setup), else starts on the existing one (shards). +# +set -e + +ci_user="${ERPNEXT_CI_USER:-frappe}" + +# Re-exec as the ci user so mariadbd and the datadir are owned consistently (root mariadbd is +# refused anyway). Mirrors install.sh's user switch. +if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then + exec su -m "$ci_user" -s /bin/bash -c \ + "ERPNEXT_CI_USER='$ci_user' CI_DB_DATADIR='${CI_DB_DATADIR:-}' bash '$0'" +fi + +# --- PostgreSQL (GitHub-hosted CI): run in-runner on a PGDATA so it bakes into the artifact, +# same idea as the mariadb datadir. Trust auth (throwaway CI) skips password setup; durability +# off for speed. Postgres is preinstalled on ubuntu-latest under /usr/lib/postgresql//bin. +if [ "${DB:-mariadb}" = "postgres" ]; then + PG_BIN=$(ls -d /usr/lib/postgresql/*/bin 2>/dev/null | sort -V | tail -1) + [ -n "$PG_BIN" ] && export PATH="$PG_BIN:$PATH" + PGDATA="${CI_DB_DATADIR:-$HOME/frappe-bench/pgdata}" + if [ ! -d "$PGDATA/base" ]; then + initdb -D "$PGDATA" -U postgres --auth-local=trust --auth-host=trust >/dev/null + echo "host all all 127.0.0.1/32 trust" >> "$PGDATA/pg_hba.conf" + fi + pg_ctl -D "$PGDATA" -w -o "-p 5432 -c listen_addresses=127.0.0.1 -c unix_socket_directories=$PGDATA -c fsync=off -c synchronous_commit=off -c full_page_writes=off" start + echo "PostgreSQL up in-runner (pgdata=$PGDATA)" + exit 0 +fi + +# --- MariaDB --- +DATADIR="${CI_DB_DATADIR:-$HOME/frappe-bench/mariadb-data}" +SOCK="$DATADIR/mysqld.sock" +fresh=0 + +if [ ! -d "$DATADIR/mysql" ]; then + mkdir -p "$DATADIR" + mariadb-install-db --no-defaults --datadir="$DATADIR" \ + --auth-root-authentication-method=normal --skip-test-db >/dev/null 2>&1 + fresh=1 +fi + +# Throwaway-CI durability off; bind TCP 127.0.0.1:3306 so bench/install.sh connect as usual. +mariadbd --no-defaults --datadir="$DATADIR" --socket="$SOCK" --pid-file="$DATADIR/mysqld.pid" \ + --port=3306 --bind-address=127.0.0.1 \ + --innodb-flush-log-at-trx-commit=0 --sync-binlog=0 --skip-log-bin \ + > "$HOME/mariadb.log" 2>&1 & + +for _ in $(seq 1 60); do + mariadb-admin --socket="$SOCK" ping --silent 2>/dev/null && break + sleep 1 +done + +if [ "$fresh" = "1" ]; then + # A fresh datadir has only a password-less root@localhost. Give it the password install.sh + # uses, plus a TCP-reachable root@127.0.0.1, so the rest of install.sh works unchanged. + mariadb --no-defaults --socket="$SOCK" -u root <<'SQL' +ALTER USER 'root'@'localhost' IDENTIFIED BY 'root'; +CREATE USER IF NOT EXISTS 'root'@'127.0.0.1' IDENTIFIED BY 'root'; +GRANT ALL PRIVILEGES ON *.* TO 'root'@'127.0.0.1' WITH GRANT OPTION; +FLUSH PRIVILEGES; +SQL +fi + +echo "MariaDB up in-container (datadir=$DATADIR, fresh=$fresh)" diff --git a/.github/workflows/server-tests-postgres.yml b/.github/workflows/server-tests-postgres.yml index 3a668133a5f..3b55691bdba 100644 --- a/.github/workflows/server-tests-postgres.yml +++ b/.github/workflows/server-tests-postgres.yml @@ -1,79 +1,43 @@ name: Server (Postgres) on: - repository_dispatch: - types: [frappe-framework-change] + schedule: + # 03:00 AM IST daily (21:30 UTC the previous day) + - cron: "30 21 * * *" pull_request: - # 'labeled' is required so adding the 'postgres' label to an open PR triggers this run - # (the job itself is gated on that label below) - types: [opened, reopened, synchronize, labeled] paths-ignore: - '**.js' - - '**.css' - - '**.svg' - '**.md' - '**.html' - 'crowdin.yml' - '.coderabbit.yml' - '.mergify.yml' - schedule: - # Run everday at midnight UTC / 5:30 IST - - cron: "0 0 * * *" workflow_dispatch: - inputs: - user: - description: 'Frappe Framework repository user (add your username for forks)' - required: true - default: 'frappe' - type: string - branch: - description: 'Frappe Framework branch' - default: 'develop' - required: false - type: string - -permissions: - contents: read concurrency: group: server-postgres-develop-${{ github.event_name }}-${{ github.event.number || github.event_name == 'workflow_dispatch' && github.run_id || '' }} cancel-in-progress: true +permissions: + contents: read + +# Postgres CI stays on GitHub-hosted (free, full-speed VM per shard) but follows the same fan-out +# we built for MariaDB: build the bench + reinstall ONCE in the setup job, bake the PostgreSQL +# PGDATA into the artifact, and have 4 test shards start Postgres on that datadir — no per-shard +# clone/build/reinstall/restore. Python is pinned so the venv transplants between VMs. +env: + TZ: 'Asia/Kolkata' + NODE_ENV: "production" + PYTHON_VERSION: '3.14' + jobs: - test: - # Opt-in on PRs: only runs when the PR carries the 'postgres' label. Scheduled / manual / - # framework-dispatch runs always execute (no PR labels to gate on). - if: ${{ github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'postgres') }} + setup: + name: Build & reinstall (setup) runs-on: ubuntu-latest - timeout-minutes: 60 - env: - TZ: 'Asia/Kolkata' - NODE_ENV: "production" - WITH_COVERAGE: ${{ github.event_name != 'pull_request' }} - - strategy: - fail-fast: false - - matrix: - container: [1, 2, 3, 4] - - # Distinct from the MariaDB job's "Python Unit Tests" so its check contexts do NOT collide with - # the required "Python Unit Tests (1..4)" status checks -- this keeps Postgres non-required for now. - name: Postgres Unit Tests - - services: - postgres: - image: postgres:13.3 - env: - POSTGRES_PASSWORD: travis - ports: - - 5432:5432 - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - + # Runs on the daily schedule (and workflow_dispatch). On PRs it runs ONLY when the PR carries + # the 'postgres' label — the test job needs setup, so it's skipped too when this is. + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'postgres') + timeout-minutes: 40 steps: - name: Clone uses: actions/checkout@v6 @@ -81,7 +45,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v6 with: - python-version: '3.14' + python-version: ${{ env.PYTHON_VERSION }} - name: Check for valid Python & Merge Conflicts run: | @@ -100,98 +64,128 @@ jobs: - name: Add to Hosts run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts - - name: Cache pip + - name: Cache deps (uv/pip/npm/yarn) uses: actions/cache@v4 with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + path: | + ~/.cache/uv + ~/.cache/pip + ~/.npm + ~/.cache/yarn + key: ${{ runner.os }}-deps-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/yarn.lock') }} + restore-keys: ${{ runner.os }}-deps- - - name: Cache node modules + # Warm-bench cache (the big one): install.sh saves the built base bench — frappe + env + + # node_modules + assets — here as frappe-bench-base-*.tar.zst. Later runs restore it and only + # fast-forward to the live develop SHA + rebuild the delta, so the bench BUILD is near-free and + # only the test_site reinstall (per-run DB, uncacheable) stays slow — matching the self-hosted + # box. The first run after a deps change populates it; every run after that is fast. + - name: Cache warm bench (base build) uses: actions/cache@v4 + with: + path: ~/bench-cache + key: ${{ runner.os }}-warmbench-v2-${{ hashFiles('**/*requirements.txt', '**/pyproject.toml', '**/yarn.lock') }} + restore-keys: ${{ runner.os }}-warmbench-v2- + + # Postgres runs in-runner on a PGDATA OUTSIDE the bench (install.sh wipes ~/frappe-bench); + # after the reinstall it's moved into the bench so it ships in the artifact. + - name: Start DB + run: bash ${GITHUB_WORKSPACE}/.github/helper/start-db.sh env: - cache-name: cache-node-modules - with: - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} - restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - - name: Get yarn cache directory path - id: yarn-cache-dir-path - run: echo "dir=$(yarn cache dir)" >> $GITHUB_OUTPUT - - - uses: actions/cache@v4 - id: yarn-cache - with: - path: ${{ steps.yarn-cache-dir-path.outputs.dir }} - key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }} - restore-keys: | - ${{ runner.os }}-yarn- - - - name: Cache wkhtmltopdf - uses: actions/cache@v4 - with: - path: /tmp/wkhtmltox.deb - key: wkhtmltox-0.12.6.1-2-jammy-amd64 + DB: postgres + CI_DB_DATADIR: /home/runner/pgdata - name: Install run: bash ${GITHUB_WORKSPACE}/.github/helper/install.sh env: DB: postgres TYPE: server - FRAPPE_USER: ${{ github.event.inputs.user }} - FRAPPE_BRANCH: ${{ github.event.client_payload.sha || github.event.inputs.branch }} + FRAPPE_BRANCH: develop + BENCH_CACHE_DIR: /home/runner/bench-cache + + - name: Stop DB and stage datadir + run: | + PG_BIN=$(ls -d /usr/lib/postgresql/*/bin | sort -V | tail -1) + "$PG_BIN/pg_ctl" -D /home/runner/pgdata -m fast -w stop || true + mv /home/runner/pgdata /home/runner/frappe-bench/pgdata + + - name: Package bench for test shards + run: | + cp "${GITHUB_WORKSPACE}/.github/helper/hydrate.sh" /home/runner/frappe-bench/hydrate.sh + cp "${GITHUB_WORKSPACE}/.github/helper/start-db.sh" /home/runner/frappe-bench/start-db.sh + tar czpf "${GITHUB_WORKSPACE}/bench.tar.gz" -C /home/runner \ + --exclude='.git' --exclude='node_modules' frappe-bench + ls -lh "${GITHUB_WORKSPACE}/bench.tar.gz" + + - name: Upload bench artifact + uses: actions/upload-artifact@v4 + with: + name: bench-pg + path: bench.tar.gz + retention-days: 1 + compression-level: 0 + + test: + name: Python Unit Tests (PG) + needs: setup + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + container: [1, 2, 3, 4] + steps: + - name: Download bench artifact + uses: actions/download-artifact@v4 + with: + name: bench-pg + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Add to Hosts + run: echo "127.0.0.1 test_site" | sudo tee -a /etc/hosts + + # The bench CLI (frappe-bench) and redis are global/system tools — not in the bench tarball. + # The setup runner got them via install.sh; the MariaDB shards get them from the arc5 image. + # GitHub-hosted PG shards install them here (cheap vs the build+reinstall that setup did once). + - name: Install shard runtime (bench CLI + redis + wkhtmltopdf) + run: | + pip install frappe-bench + command -v redis-server >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq redis-server; } + # wkhtmltopdf (patched-qt build) for print-format / PDF tests — same .deb install.sh uses. + if ! command -v wkhtmltopdf >/dev/null; then + wget -qO /tmp/wkhtmltox.deb https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-2/wkhtmltox_0.12.6.1-2.jammy_amd64.deb + sudo apt-get install -y -qq /tmp/wkhtmltox.deb + fi + + - name: Untar bench + run: | + tar xzpf "${GITHUB_WORKSPACE}/bench.tar.gz" -C /home/runner + ls -ld /home/runner/frappe-bench + + - name: Hydrate (start Postgres on the baked datadir) + run: bash /home/runner/frappe-bench/hydrate.sh + env: + DB: postgres + DB_HOST: 127.0.0.1 - name: Run Tests run: | cd ~/frappe-bench/ - coverage_flag="" - if [ "$WITH_COVERAGE" = "true" ]; then coverage_flag="--with-coverage"; fi + # print-format / PDF tests are engine-independent (they exercise wkhtmltopdf rendering, + # not postgres SQL — the MariaDB CI already covers them). They only fetch the static asset + # bundles from http://test_site:8000/assets/..., so a plain static file server over sites/ + # satisfies wkhtmltopdf without the frappe web server (which never bound on a bare runner). + ( cd ~/frappe-bench/sites && nohup python3 -m http.server 8000 --bind 127.0.0.1 > ~/frappe-bench/web.log 2>&1 & ) + for _ in $(seq 1 15); do (exec 3<>/dev/tcp/127.0.0.1/8000) 2>/dev/null && { exec 3>&- 3<&-; break; }; sleep 1; done bench --site test_site run-parallel-tests --lightmode --app erpnext \ - --total-builds ${{ strategy.job-total }} \ - --build-number ${{ matrix.container }} \ - $coverage_flag + --total-builds 4 --build-number ${{ matrix.container }} env: TYPE: server - - - name: Show bench output + - name: Show web server log if: ${{ always() }} - run: cat ~/frappe-bench/bench_start.log || true - - - name: Upload coverage data - if: ${{ env.WITH_COVERAGE == 'true' }} - uses: actions/upload-artifact@v4 - with: - name: coverage-postgres-${{ matrix.container }} - path: /home/runner/frappe-bench/sites/coverage.xml - - coverage: - name: Coverage Wrap Up - needs: test - if: ${{ github.event_name != 'pull_request' }} - runs-on: ubuntu-latest - steps: - - name: Clone - uses: actions/checkout@v6 - - - name: Download artifacts - uses: actions/download-artifact@v4 - with: - pattern: coverage-postgres-* - - - name: Upload coverage data - uses: codecov/codecov-action@v4 - with: - name: Postgres - flags: postgres - # explicit glob: download-artifact extracts each shard into its own coverage-postgres-N/ dir - files: coverage-postgres-*/coverage.xml - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: true - verbose: true + run: cat ~/frappe-bench/web.log 2>/dev/null || true