250 lines
9.1 KiB
Python
250 lines
9.1 KiB
Python
"""Logical backup and restore, in pure Python.
|
|
|
|
The bundled portable PostgreSQL ships only initdb/pg_ctl/postgres — no pg_dump —
|
|
so this walks the tables with psycopg and writes a self-describing archive
|
|
instead. That also makes the format readable and diffable, which matters more
|
|
for a team settings store than raw speed.
|
|
|
|
Archive layout (gzip'd tar):
|
|
|
|
manifest.json # format version, timestamp, per-file sha256, row counts
|
|
db/users.json # one file per table, rows as objects, IDs preserved
|
|
db/tokens.json
|
|
...
|
|
plugins/<id>/<ver>.tar.gz
|
|
|
|
Restoring preserves primary keys because tokens.user_id and
|
|
plugin_versions.plugin_id reference them; renumbering would break the links.
|
|
|
|
python -m dsh_sync.backup export [--out FILE]
|
|
python -m dsh_sync.backup verify FILE
|
|
python -m dsh_sync.backup import FILE [--force]
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import sys
|
|
import tarfile
|
|
from pathlib import Path
|
|
|
|
import psycopg
|
|
|
|
from .config import Config
|
|
from .db import connect, init_db
|
|
from .storage import artifact_path
|
|
from .util import now_iso
|
|
|
|
FORMAT_VERSION = 1
|
|
|
|
# Restore order matters: parents before children. Reversed for a clean wipe.
|
|
TABLES_IN_ORDER = [
|
|
"users",
|
|
"tokens",
|
|
"settings",
|
|
"plugins",
|
|
"plugin_versions",
|
|
"audit_log",
|
|
]
|
|
|
|
# Columns that must survive as-is. users.id is a BIGSERIAL: restoring explicit
|
|
# ids leaves the sequence behind, so it is reset after import.
|
|
SERIAL_TABLES = {"users": "id", "audit_log": "id"}
|
|
|
|
|
|
def _sha256(data: bytes) -> str:
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def export_backup(cfg: Config, out: Path | None = None) -> Path:
|
|
"""Write a full backup archive and return its path."""
|
|
out = out or (cfg.data_dir / "backups" / f"dsh-backup-{now_iso().replace(':', '')}.tar.gz")
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
manifest = {
|
|
"format_version": FORMAT_VERSION,
|
|
"created_at": now_iso(),
|
|
"database": _redact(cfg.database_url),
|
|
"files": {},
|
|
"row_counts": {},
|
|
}
|
|
|
|
conn = connect(cfg)
|
|
try:
|
|
with tarfile.open(out, "w:gz") as tf:
|
|
for table in TABLES_IN_ORDER:
|
|
rows = [dict(r) for r in conn.execute(f"SELECT * FROM {table}").fetchall()]
|
|
blob = json.dumps(rows, ensure_ascii=False, indent=1).encode("utf-8")
|
|
_add_bytes(tf, f"db/{table}.json", blob)
|
|
manifest["files"][f"db/{table}.json"] = _sha256(blob)
|
|
manifest["row_counts"][table] = len(rows)
|
|
|
|
# Plugin artifacts, keyed by the path the DB row points at.
|
|
version_rows = conn.execute(
|
|
"SELECT plugin_id, version, artifact_path FROM plugin_versions"
|
|
).fetchall()
|
|
for r in version_rows:
|
|
try:
|
|
src = artifact_path(cfg, r["artifact_path"])
|
|
except ValueError:
|
|
continue # malformed row; the DB dump still records it
|
|
if not src.is_file():
|
|
continue
|
|
name = f"plugins/{r['plugin_id']}/{r['version']}.tar.gz"
|
|
blob = src.read_bytes()
|
|
_add_bytes(tf, name, blob)
|
|
manifest["files"][name] = _sha256(blob)
|
|
|
|
_add_bytes(tf, "manifest.json",
|
|
json.dumps(manifest, ensure_ascii=False, indent=1).encode("utf-8"))
|
|
finally:
|
|
conn.close()
|
|
return out
|
|
|
|
|
|
def verify_backup(path: Path) -> dict:
|
|
"""Check every file against the manifest hashes without touching the DB."""
|
|
with tarfile.open(path, "r:gz") as tf:
|
|
manifest = json.loads(tf.extractfile("manifest.json").read().decode("utf-8"))
|
|
if manifest.get("format_version") != FORMAT_VERSION:
|
|
raise ValueError(
|
|
f"unsupported backup format {manifest.get('format_version')!r} "
|
|
f"(this build reads {FORMAT_VERSION})"
|
|
)
|
|
problems = []
|
|
for name, want in manifest["files"].items():
|
|
try:
|
|
got = _sha256(tf.extractfile(name).read())
|
|
except KeyError:
|
|
problems.append(f"{name}: missing from archive")
|
|
continue
|
|
if got != want:
|
|
problems.append(f"{name}: sha256 {got[:12]}… != {want[:12]}…")
|
|
if problems:
|
|
raise ValueError("backup is corrupt:\n " + "\n ".join(problems))
|
|
manifest["verified"] = True
|
|
return manifest
|
|
|
|
|
|
def import_backup(cfg: Config, path: Path, force: bool = False) -> dict:
|
|
"""Restore an archive. Refuses to overwrite a non-empty database unless forced."""
|
|
manifest = verify_backup(path)
|
|
conn = connect(cfg)
|
|
try:
|
|
existing = conn.execute("SELECT COUNT(*) AS c FROM users").fetchone()["c"]
|
|
if existing and not force:
|
|
raise ValueError(
|
|
f"database already holds {existing} user(s); pass force=True "
|
|
f"(--force) to overwrite, or restore into an empty database"
|
|
)
|
|
with tarfile.open(path, "r:gz") as tf:
|
|
# Wipe children first so foreign keys never block the delete.
|
|
for table in reversed(TABLES_IN_ORDER):
|
|
conn.execute(f"DELETE FROM {table}")
|
|
for table in TABLES_IN_ORDER:
|
|
name = f"db/{table}.json"
|
|
if name not in manifest["files"]:
|
|
continue
|
|
rows = json.loads(tf.extractfile(name).read().decode("utf-8"))
|
|
for row in rows:
|
|
cols = list(row)
|
|
placeholders = ", ".join(["%s"] * len(cols))
|
|
conn.execute(
|
|
f"INSERT INTO {table} ({', '.join(cols)}) VALUES ({placeholders})",
|
|
[row[c] for c in cols],
|
|
)
|
|
# BIGSERIAL keeps counting from its last value; after inserting
|
|
# explicit ids that value is stale and the next insert would collide.
|
|
for table, col in SERIAL_TABLES.items():
|
|
conn.execute(
|
|
f"SELECT setval(pg_get_serial_sequence(%s, %s), "
|
|
f"COALESCE((SELECT MAX({col}) FROM {table}), 1))",
|
|
(table, col),
|
|
)
|
|
|
|
cfg.plugins_dir.mkdir(parents=True, exist_ok=True)
|
|
restored = 0
|
|
for name in manifest["files"]:
|
|
if not name.startswith("plugins/"):
|
|
continue
|
|
dest = cfg.plugins_dir / name[len("plugins/"):]
|
|
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
dest.write_bytes(tf.extractfile(name).read())
|
|
restored += 1
|
|
conn.commit()
|
|
finally:
|
|
conn.close()
|
|
return {"restored_rows": manifest["row_counts"], "restored_artifacts": restored}
|
|
|
|
|
|
def _add_bytes(tf: tarfile.TarFile, name: str, blob: bytes) -> None:
|
|
info = tarfile.TarInfo(name)
|
|
info.size = len(blob)
|
|
info.mtime = 0 # reproducible archives
|
|
tf.addfile(info, io.BytesIO(blob))
|
|
|
|
|
|
def _redact(dsn: str) -> str:
|
|
"""Keep the backup manifest from leaking a password.
|
|
|
|
Only adds the `***` when the DSN actually carries one — planting it on a
|
|
passwordless DSN would misreport how the server connects.
|
|
"""
|
|
if "@" not in dsn:
|
|
return dsn
|
|
scheme, rest = dsn.split("://", 1) if "://" in dsn else ("", dsn)
|
|
creds, host = rest.rsplit("@", 1)
|
|
if ":" not in creds:
|
|
return dsn # user only, nothing to hide
|
|
user = creds.split(":", 1)[0]
|
|
redacted = f"{user}:***@{host}"
|
|
return f"{scheme}://{redacted}" if scheme else redacted
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
ap = argparse.ArgumentParser(prog="dsh_sync.backup", description="dsh-sync backup tool")
|
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
|
|
p = sub.add_parser("export", help="write a backup archive")
|
|
p.add_argument("--out", type=Path)
|
|
|
|
p = sub.add_parser("verify", help="check an archive against its manifest")
|
|
p.add_argument("archive", type=Path)
|
|
|
|
p = sub.add_parser("import", help="restore an archive")
|
|
p.add_argument("archive", type=Path)
|
|
p.add_argument("--force", action="store_true", help="overwrite a non-empty database")
|
|
|
|
args = ap.parse_args(argv)
|
|
cfg = Config.from_env()
|
|
cfg.ensure_dirs()
|
|
init_db(cfg)
|
|
|
|
if args.cmd == "export":
|
|
path = export_backup(cfg, args.out)
|
|
size = path.stat().st_size
|
|
print(json.dumps({"archive": str(path), "size_bytes": size}, indent=2))
|
|
return 0
|
|
if args.cmd == "verify":
|
|
m = verify_backup(args.archive)
|
|
print(json.dumps({"archive": str(args.archive), "ok": True,
|
|
"created_at": m["created_at"],
|
|
"rows": m["row_counts"],
|
|
"files": len(m["files"])}, indent=2, ensure_ascii=False))
|
|
return 0
|
|
if args.cmd == "import":
|
|
out = import_backup(cfg, args.archive, force=args.force)
|
|
print(json.dumps({"archive": str(args.archive), **out}, indent=2, ensure_ascii=False))
|
|
return 0
|
|
return 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
sys.exit(main())
|
|
except (ValueError, psycopg.Error) as e:
|
|
print(f"error: {e}", file=sys.stderr)
|
|
sys.exit(1)
|