ci(mariadb): revert to 4 shards + cleanup

This commit is contained in:
Mihir Kandoi
2026-06-24 11:54:50 +05:30
parent 3b8674a4a9
commit 62c401badc
3 changed files with 9 additions and 41 deletions

View File

@@ -4,21 +4,19 @@
#
# The bench (apps, venv, node_modules, sites) is already on disk at ~/frappe-bench — the
# workflow untar'd it from the artifact the setup job built. So there is NO bench init, no
# asset build, and no reinstall here: just bring the DB up and restore the dump the setup job
# baked into the bench, then start bench so tests can run. Mirrors the DB + bench-start tail of
# install.sh. The whole point is that the expensive work happened ONCE in the setup job.
# asset build, and no reinstall here: just bring the DB up on the baked datadir and start redis
# so tests can run. The whole point is that the expensive work happened ONCE in the setup job.
#
set -e
ci_user="${ERPNEXT_CI_USER:-frappe}"
db_host="${DB_HOST:-127.0.0.1}"
dump="${CI_BASELINE_BACKUP:-/home/$ci_user/frappe-bench/test_site-db.sql.gz}"
# Re-exec as the ci user (uid 1001) so bench/cache ownership matches the artifact, same as
# install.sh. The workflow untar'd as root with -p, so the files are already owned by ci.
if [ "$(id -u)" = "0" ] && [ "${SKIP_SYSTEM_SETUP:-0}" = "1" ] && [ "$ci_user" != "root" ]; then
exec su -m "$ci_user" -s /bin/bash -c \
"ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' CI_BASELINE_BACKUP='$dump' bash '$0'"
"ERPNEXT_CI_USER='$ci_user' DB_HOST='$db_host' bash '$0'"
fi
cd ~/frappe-bench

View File

@@ -344,37 +344,7 @@ PY
}
wait_for_redis
# Site setup. `bench reinstall` rebuilds the entire schema in Python (~1000 DocTypes) — the
# CI bottleneck that DB tuning / tmpfs / faster cores couldn't move. Instead, restore a
# pre-baked baseline (the DB engine loads it, no Python schema-build) and `migrate` to sync
# only the drift since the baseline was built. The baseline is produced by
# .github/helper/generate-ci-baseline.sh (run nightly / at image build) from a clean
# reinstall on develop. Gated by CI_RESTORE_FROM_BACKUP so it A/Bs against plain reinstall;
# falls back to reinstall if the baseline is missing or the restore fails.
CI_BASELINE_BACKUP="${CI_BASELINE_BACKUP:-/opt/ci-baseline/test_site-database.sql.gz}"
if [ "${CI_RESTORE_FROM_BACKUP:-0}" = "1" ] && [ -f "$CI_BASELINE_BACKUP" ]; then
if [ "$DB" == "mariadb" ]; then
db_root_args=(--db-root-username root --db-root-password root)
else
db_root_args=(--db-root-username postgres --db-root-password travis)
fi
if run_ci_step "Restore baseline test site" bench --site test_site --force restore "${db_root_args[@]}" "$CI_BASELINE_BACKUP"; then
run_ci_step "Migrate test site" bench --site test_site migrate
else
run_ci_step "Reinstall test site (baseline restore failed)" bench --site test_site reinstall --yes
fi
else
run_ci_step "Reinstall test site" bench --site test_site reinstall --yes
fi
# Refresh the baseline backup from this freshly set-up site. Run a normal job (reinstall path)
# with CI_GENERATE_BASELINE=1 and the baseline dir mounted read-write; install.sh captures the
# DB dump to CI_BASELINE_BACKUP so future runs can restore it. Nightly is enough — bench migrate
# absorbs intraday develop drift. To bake into the image instead, copy the produced .sql.gz in.
if [ "${CI_GENERATE_BASELINE:-0}" = "1" ]; then
run_ci_step "Backup baseline test site" bench --site test_site backup
latest_backup=$(ls -t ~/frappe-bench/sites/test_site/private/backups/*-database.sql.gz | head -1)
mkdir -p "$(dirname "$CI_BASELINE_BACKUP")"
cp "$latest_backup" "$CI_BASELINE_BACKUP"
echo "Baseline written to $CI_BASELINE_BACKUP ($(du -h "$latest_backup" | cut -f1))"
fi
# Site setup: build the schema (~1000 DocTypes) into the DB. This is the single-threaded-Python
# bottleneck, but the fan-out amortises it — it runs once here in the setup job, and the test
# shards start the DB on the baked datadir instead of repeating the reinstall.
run_ci_step "Reinstall test site" bench --site test_site reinstall --yes

View File

@@ -151,7 +151,7 @@ jobs:
strategy:
fail-fast: false
matrix:
container: [1, 2, 3, 4, 5, 6, 7, 8]
container: [1, 2, 3, 4]
steps:
- name: Add to Hosts
@@ -178,7 +178,7 @@ jobs:
coverage_flag=""
if [ "$WITH_COVERAGE" = "true" ]; then coverage_flag="--with-coverage"; fi
bench --site test_site run-parallel-tests --lightmode --app erpnext \
--total-builds 8 \
--total-builds 4 \
--build-number ${{ matrix.container }} \
$coverage_flag
EOF