"""Logical backup and restore, in pure Python. The bundled portable PostgreSQL ships only initdb/pg_ctl/postgres — no pg_dump — so this walks the tables with psycopg and writes a self-describing archive instead. That also makes the format readable and diffable, which matters more for a team settings store than raw speed. Archive layout (gzip'd tar): manifest.json # format version, timestamp, per-file sha256, row counts db/users.json # one file per table, rows as objects, IDs preserved db/tokens.json ... plugins//.tar.gz Restoring preserves primary keys because tokens.user_id and plugin_versions.plugin_id reference them; renumbering would break the links. python -m dsh_sync.backup export [--out FILE] python -m dsh_sync.backup verify FILE python -m dsh_sync.backup import FILE [--force] """ from __future__ import annotations import argparse import hashlib import io import json import sys import tarfile from pathlib import Path import psycopg from .config import Config from .db import connect, init_db from .storage import artifact_path from .util import now_iso FORMAT_VERSION = 1 # Restore order matters: parents before children. Reversed for a clean wipe. TABLES_IN_ORDER = [ "users", "tokens", "settings", "plugins", "plugin_versions", "audit_log", ] # Columns that must survive as-is. users.id is a BIGSERIAL: restoring explicit # ids leaves the sequence behind, so it is reset after import. SERIAL_TABLES = {"users": "id", "audit_log": "id"} def _sha256(data: bytes) -> str: return hashlib.sha256(data).hexdigest() def export_backup(cfg: Config, out: Path | None = None) -> Path: """Write a full backup archive and return its path.""" out = out or (cfg.data_dir / "backups" / f"dsh-backup-{now_iso().replace(':', '')}.tar.gz") out.parent.mkdir(parents=True, exist_ok=True) manifest = { "format_version": FORMAT_VERSION, "created_at": now_iso(), "database": _redact(cfg.database_url), "files": {}, "row_counts": {}, } conn = connect(cfg) try: with tarfile.open(out, "w:gz") as tf: for table in TABLES_IN_ORDER: rows = [dict(r) for r in conn.execute(f"SELECT * FROM {table}").fetchall()] blob = json.dumps(rows, ensure_ascii=False, indent=1).encode("utf-8") _add_bytes(tf, f"db/{table}.json", blob) manifest["files"][f"db/{table}.json"] = _sha256(blob) manifest["row_counts"][table] = len(rows) # Plugin artifacts, keyed by the path the DB row points at. version_rows = conn.execute( "SELECT plugin_id, version, artifact_path FROM plugin_versions" ).fetchall() for r in version_rows: try: src = artifact_path(cfg, r["artifact_path"]) except ValueError: continue # malformed row; the DB dump still records it if not src.is_file(): continue name = f"plugins/{r['plugin_id']}/{r['version']}.tar.gz" blob = src.read_bytes() _add_bytes(tf, name, blob) manifest["files"][name] = _sha256(blob) _add_bytes(tf, "manifest.json", json.dumps(manifest, ensure_ascii=False, indent=1).encode("utf-8")) finally: conn.close() return out def verify_backup(path: Path) -> dict: """Check every file against the manifest hashes without touching the DB.""" with tarfile.open(path, "r:gz") as tf: manifest = json.loads(tf.extractfile("manifest.json").read().decode("utf-8")) if manifest.get("format_version") != FORMAT_VERSION: raise ValueError( f"unsupported backup format {manifest.get('format_version')!r} " f"(this build reads {FORMAT_VERSION})" ) problems = [] for name, want in manifest["files"].items(): try: got = _sha256(tf.extractfile(name).read()) except KeyError: problems.append(f"{name}: missing from archive") continue if got != want: problems.append(f"{name}: sha256 {got[:12]}… != {want[:12]}…") if problems: raise ValueError("backup is corrupt:\n " + "\n ".join(problems)) manifest["verified"] = True return manifest def import_backup(cfg: Config, path: Path, force: bool = False) -> dict: """Restore an archive. Refuses to overwrite a non-empty database unless forced.""" manifest = verify_backup(path) conn = connect(cfg) try: existing = conn.execute("SELECT COUNT(*) AS c FROM users").fetchone()["c"] if existing and not force: raise ValueError( f"database already holds {existing} user(s); pass force=True " f"(--force) to overwrite, or restore into an empty database" ) with tarfile.open(path, "r:gz") as tf: # Wipe children first so foreign keys never block the delete. for table in reversed(TABLES_IN_ORDER): conn.execute(f"DELETE FROM {table}") for table in TABLES_IN_ORDER: name = f"db/{table}.json" if name not in manifest["files"]: continue rows = json.loads(tf.extractfile(name).read().decode("utf-8")) for row in rows: cols = list(row) placeholders = ", ".join(["%s"] * len(cols)) conn.execute( f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({placeholders})", [row[c] for c in cols], ) # BIGSERIAL keeps counting from its last value; after inserting # explicit ids that value is stale and the next insert would collide. for table, col in SERIAL_TABLES.items(): conn.execute( f"SELECT setval(pg_get_serial_sequence(%s, %s), " f"COALESCE((SELECT MAX({col}) FROM {table}), 1))", (table, col), ) cfg.plugins_dir.mkdir(parents=True, exist_ok=True) restored = 0 for name in manifest["files"]: if not name.startswith("plugins/"): continue dest = cfg.plugins_dir / name[len("plugins/"):] dest.parent.mkdir(parents=True, exist_ok=True) dest.write_bytes(tf.extractfile(name).read()) restored += 1 conn.commit() finally: conn.close() return {"restored_rows": manifest["row_counts"], "restored_artifacts": restored} def _add_bytes(tf: tarfile.TarFile, name: str, blob: bytes) -> None: info = tarfile.TarInfo(name) info.size = len(blob) info.mtime = 0 # reproducible archives tf.addfile(info, io.BytesIO(blob)) def _redact(dsn: str) -> str: """Keep the backup manifest from leaking a password. Only adds the `***` when the DSN actually carries one — planting it on a passwordless DSN would misreport how the server connects. """ if "@" not in dsn: return dsn scheme, rest = dsn.split("://", 1) if "://" in dsn else ("", dsn) creds, host = rest.rsplit("@", 1) if ":" not in creds: return dsn # user only, nothing to hide user = creds.split(":", 1)[0] redacted = f"{user}:***@{host}" return f"{scheme}://{redacted}" if scheme else redacted def main(argv=None) -> int: ap = argparse.ArgumentParser(prog="dsh_sync.backup", description="dsh-sync backup tool") sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("export", help="write a backup archive") p.add_argument("--out", type=Path) p = sub.add_parser("verify", help="check an archive against its manifest") p.add_argument("archive", type=Path) p = sub.add_parser("import", help="restore an archive") p.add_argument("archive", type=Path) p.add_argument("--force", action="store_true", help="overwrite a non-empty database") args = ap.parse_args(argv) cfg = Config.from_env() cfg.ensure_dirs() init_db(cfg) if args.cmd == "export": path = export_backup(cfg, args.out) size = path.stat().st_size print(json.dumps({"archive": str(path), "size_bytes": size}, indent=2)) return 0 if args.cmd == "verify": m = verify_backup(args.archive) print(json.dumps({"archive": str(args.archive), "ok": True, "created_at": m["created_at"], "rows": m["row_counts"], "files": len(m["files"])}, indent=2, ensure_ascii=False)) return 0 if args.cmd == "import": out = import_backup(cfg, args.archive, force=args.force) print(json.dumps({"archive": str(args.archive), **out}, indent=2, ensure_ascii=False)) return 0 return 2 if __name__ == "__main__": try: sys.exit(main()) except (ValueError, psycopg.Error) as e: print(f"error: {e}", file=sys.stderr) sys.exit(1)