diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7036b3fe6..eaa8f7546 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,7 @@ jobs: test/e2e/mail-fail2ban-db.e2e.test.ts test/e2e/mail-dovecot-security.e2e.test.ts test/e2e/mail-state-upgrade.e2e.test.ts + test/e2e/mail-backup-roundtrip.e2e.test.ts # Separate API processes and databases exercise both move directions, # recovery, authentication and Desktop relays on every PR. No live # server, provider account or outbound email is used. diff --git a/apps/api/src/modules/mail/admin/backup-plan.ts b/apps/api/src/modules/mail/admin/backup-plan.ts deleted file mode 100644 index 023e583d2..000000000 --- a/apps/api/src/modules/mail/admin/backup-plan.ts +++ /dev/null @@ -1,228 +0,0 @@ -/** - * Mail backup plan — turns the operator's include-checkboxes into the - * `custom_command` producer's payloadConfig (produce/restore shell). Mail - * path + SQL knowledge lives here in apps/api; the backup adapter stays - * generic (it just runs `sh -c ` on the source via the - * bare SSH executor and streams stdout as one artifact). - * - * produce : stage vmail dump (+ optional DKIM/config, + optional - * maildirs) then `tar | zstd` to stdout. - * restore : `zstd -d | tar -x`, load the vmail data (TRUNCATE the four - * account tables then COPY), restore DKIM/config + maildirs - * when present, recount, reload daemons. - * - * Data-only DB restore: rows load into the TARGET install's existing - * `vmail` schema using the target's own daemon creds — we deliberately do - * NOT carry DB roles/passwords (that would desync dovecot-sql.conf). - * `{SSHA512}` password hashes are host-portable, so accounts work as-is. - */ - -import { HOST_STATE_DIR, MAIL_HOST_STATE_DIR, safePipelineFragment } from "@repo/adapters"; -import { - HOST_AMAVIS_CONF_CANDIDATES, - mailDaemonReloadCommand, - mailEngineCommand, - mailPgDumpToStdout, - mailPsqlFromStdin, - type MailEngineFlavor, -} from "@repo/platform/engine/modules/mail/mail-engine"; - -/** The four tables that hold accounts / domains / aliases / admins. */ -const ACCOUNT_TABLES = ["domain", "mailbox", "forwardings", "domain_admins"] as const; - -/** - * Every place a legacy install may keep amavis's editable config, shared with the - * DKIM writer so a backup can't cover a narrower set of hosts than the thing it is - * backing up. Hardcoding `/etc/amavis/conf.d/50-user` here meant a RHEL-family box - * (monolithic `/etc/amavisd.conf`, no `conf.d`) produced a "keys" archive with no - * amavis config in it at all — and said it succeeded. - */ -const AMAVIS_CONF_PATHS = HOST_AMAVIS_CONF_CANDIDATES.map((c) => c.path); - -/** Debian-family location — the fallback when an archive doesn't name one. */ -const AMAVIS_CONF_DEFAULT = HOST_AMAVIS_CONF_CANDIDATES[0].path; - -/** Archive-relative name for the one amavis config we found, plus where it came from. */ -const AMAVIS_IN_ARCHIVE = "keys/amavis-conf"; - -/** Pre-2026-08 archives stored the Debian path under this fixed name. */ -const AMAVIS_IN_ARCHIVE_LEGACY = "keys/amavis-50-user"; - -export interface MailBackupFlags { - /** Include the maildir message store (/var/vmail). Large. */ - messageData: boolean; - /** Include DKIM keys + amavis config + mail-state.json (secrets). */ - keys: boolean; -} - -export interface MailBackupPayload { - payloadKind: "custom_command"; - payloadConfig: { - produceCommand: string; - restoreCommand: string; - artifactName: string; - /** Recorded for the UI + auditing (not used by the shell). */ - mail: { messageData: boolean; keys: boolean }; - }; -} - -/** - * Build the payloadConfig for a mail-server backup policy. `domain` is - * embedded only in the archive's manifest JSON (never in a shell - * position), so it needs no escaping beyond JSON. - */ -export function buildMailBackupPayload( - domain: string, - flags: MailBackupFlags, - flavor: MailEngineFlavor, -): MailBackupPayload { - const tableArgs = ACCOUNT_TABLES.map((t) => `-t ${t}`).join(" "); - const truncateList = ACCOUNT_TABLES.join(", "); - const info = JSON.stringify({ domain, messageData: flags.messageData, keys: flags.keys }); - const vmailHostDir = flavor === "container" ? `${MAIL_HOST_STATE_DIR}/vmail` : "/var/vmail"; - const produceCommand = [ - "set -e", - 'tmp="$(mktemp -d)"', - "trap 'rm -rf \"$tmp\"' EXIT", - // Accounts + auth — always. Plain-SQL data-only dump (COPY blocks). - `${mailPgDumpToStdout(flavor, `--data-only --no-owner --no-privileges ${tableArgs}`)} > "$tmp/vmail.data.sql"`, - // What's inside — read by the UI / hand-restore. - `printf '%s' '${info.replace(/'/g, "'\\''")}' > "$tmp/mail-backup.json"`, - flags.keys - ? [ - 'mkdir -p "$tmp/keys"', - // Every read here is `sudo -n`, and every copy is allowed to FAIL the backup. - // - // These are root-owned paths — `${HOST_STATE_DIR}` sits under /root (mode 0700) - // and is itself created 0700, and /var/lib/dkim holds private keys. On a host - // Openship logs into as a sudoer rather than as root, the old - // `[ -f … ] && cp … || true` could not even stat them: the test failed for lack - // of privilege, `|| true` absorbed it, and the run exited 0 having recorded - // `keys: true` on an archive containing no keys. mail-state.json is the ONLY - // copy of the DKIM state and the encrypted mailbox/admin credentials, so that - // is a disaster-recovery archive which is silently not one. Same shape as the - // amavis bug this file's own header documents. - // - // The `sudo -n` reads below only distinguish "absent" from "unreadable" if sudo - // is known to WORK — otherwise a failing `test` reads as absence and skips, and - // we are back to an archive stamped `keys: true` with no keys in it. - // - // That used to be guaranteed for free: `sudo -u postgres pg_dump` ran first, so - // under `set -e` a box without sudo never reached here. Routing the dump through - // the DB sidecar for containerized engines (GH-563) removed the guarantee on - // exactly the topology most installs now use, so the probe is explicit. It is - // the whole reason dropping `|| true` on the copies is safe. - "sudo -n true", - 'if sudo -n test -d /var/lib/dkim; then sudo -n cp -a /var/lib/dkim "$tmp/keys/dkim"; fi', - // Whichever of the known locations this box actually uses, with the source - // path recorded alongside so the restore puts it back where amavis reads it. - `for c in ${AMAVIS_CONF_PATHS.join(" ")}; do`, - ` sudo -n test -f "$c" || continue`, - ` sudo -n cp -a "$c" "$tmp/${AMAVIS_IN_ARCHIVE}"`, - ` printf '%s' "$c" > "$tmp/${AMAVIS_IN_ARCHIVE}.path"`, - ` break`, - `done`, - `if sudo -n test -f ${HOST_STATE_DIR}/mail-state.json; then`, - ` sudo -n cp -a ${HOST_STATE_DIR}/mail-state.json "$tmp/keys/mail-state.json"`, - `fi`, - ].join("\n") - : "", - // Stream one tar to stdout: the staged dir + (optionally) the maildirs - // read in place (not re-copied), so the source needs no extra disk. - // - // `sudo -n` for the same reason the staging above uses it, and it is not optional - // here: `cp -a` as root stages root-owned 0600 files, so an unprivileged tar could not - // read back what was just collected. /var/vmail is vmail:vmail 0700 and was already in - // that position. - // - // The pipeline goes through the SHARED fragment so tar's status is no longer masked by - // zstd's. The comment here used to describe that masking — "on a non-root login the - // maildirs were being dropped from the artifact just as quietly" — and leave it in - // place; this was the last pipeline in the module still carrying it. The fragment form - // exists because this is one line of a `set -e` script rather than a standalone - // `sh -c`, and it captures each status via `if` for exactly that reason. - safePipelineFragment( - flags.messageData - ? `sudo -n tar -c -C "$tmp" . -C ${vmailHostDir} vmail1` - : 'sudo -n tar -c -C "$tmp" .', - "zstd -c -3", - { left: "tar (staging the mail archive)", right: "the compressor 'zstd'" }, - ), - ] - .filter(Boolean) - .join("\n"); - - const restoreCommand = [ - "set -e", - 'tmp="$(mktemp -d)"', - "trap 'rm -rf \"$tmp\"' EXIT", - // stdin = the tar.zst artifact. Shared fragment again: a `zstd -d` that fails (a - // truncated artifact, a missing binary) must not hide behind tar's status, because the - // lines after this one TRUNCATE the account tables and then load from "$tmp" — so a - // silently-empty extract would wipe the mail database and restore nothing. - safePipelineFragment("zstd -d", 'tar -x -C "$tmp"', { - left: "the decompressor 'zstd -d'", - right: "tar (extracting the mail archive)", - }), - // Data-only restore: wipe the target's account tables, then load. - `printf '%s' 'TRUNCATE ${truncateList} CASCADE;' | ${mailPsqlFromStdin(flavor)}`, - `${mailPsqlFromStdin(flavor)} < "$tmp/vmail.data.sql"`, - // DKIM keys + amavis config (if the archive carried them). - 'if [ -d "$tmp/keys/dkim" ]; then mkdir -p /var/lib/dkim && cp -a "$tmp/keys/dkim/." /var/lib/dkim/ || true; fi', - // Where to put it is decided by THIS box, not by the box the archive came from. - // - // The archive records its source path, and honouring that verbatim restores a Debian - // archive's /etc/amavis/conf.d/50-user onto a RHEL target, which reads a monolithic - // /etc/amavisd.conf and never looks at the file we just wrote — restored, inert, and - // `|| true` covering it. Cross-family is the NORMAL case here: the reason to restore - // is usually a different machine. The recorded path only breaks the tie when the - // target has no amavis at all, and is still matched against the known list rather - // than trusted, because this shell runs as root on the target. - // - // The probe is each candidate's own `test`, which is the family detector the rest of - // the system already uses — the first is keyed on the include DIRECTORY because - // 50-user is ours to CREATE, so testing for the file would misread a Debian box that - // has no override yet and fall through to the RHEL path. Deliberately NOT the - // produce side's expression: collecting asks "is there a file to copy", placing asks - // "which layout is this box". - `dest=""`, - ...HOST_AMAVIS_CONF_CANDIDATES.map( - (c, i) => `${i === 0 ? "if" : "elif"} [ ${c.test} ]; then dest=${c.path}`, - ), - `fi`, - `if [ -f "$tmp/${AMAVIS_IN_ARCHIVE}" ]; then`, - ` saved="$(cat "$tmp/${AMAVIS_IN_ARCHIVE}.path" 2>/dev/null || true)"`, - ` if [ -z "$dest" ]; then`, - ` dest=${AMAVIS_CONF_DEFAULT}`, - ` for c in ${AMAVIS_CONF_PATHS.join(" ")}; do if [ "$saved" = "$c" ]; then dest="$c"; break; fi; done`, - ` fi`, - ` cp -a "$tmp/${AMAVIS_IN_ARCHIVE}" "$dest" || true`, - // Pre-2026-08 archives carried the Debian path under a fixed name, so they name no - // source at all — which makes the target probe the only thing that can place them. - `elif [ -f "$tmp/${AMAVIS_IN_ARCHIVE_LEGACY}" ]; then`, - ` if [ -z "$dest" ]; then dest=${AMAVIS_CONF_DEFAULT}; fi`, - ` cp -a "$tmp/${AMAVIS_IN_ARCHIVE_LEGACY}" "$dest" || true`, - `fi`, - // Maildirs (if included). Ownership must be vmail:vmail for Dovecot. - // /var/vmail is a bind mount, so the COPY is a host operation either way - but the - // ownership has to be applied where the `vmail` user exists, which on a containerized - // box is the engine, not the host (GH-563). No `|| true` on the chown: maildirs left - // root-owned are maildirs Dovecot cannot read, and swallowing that produced a restore - // that reported success and served nothing. - `if [ -d "$tmp/vmail1" ]; then cp -a "$tmp/vmail1" ${vmailHostDir}/ && ${mailEngineCommand(flavor, "chown -R vmail:vmail /var/vmail/vmail1")}; fi`, - // Recompute per-domain counters (app-managed, not DB triggers). - `printf '%s' "UPDATE domain d SET mailboxes=(SELECT count(*) FROM mailbox m WHERE m.domain=d.domain), aliases=(SELECT count(*) FROM forwardings f WHERE f.domain=d.domain AND f.is_alias) WHERE d.domain IS NOT NULL;" | ${mailPsqlFromStdin(flavor)} || true`, - // Reload daemons so the restored data + keys take effect. - `${mailDaemonReloadCommand(flavor)} || true`, - ].join("\n"); - - return { - payloadKind: "custom_command", - payloadConfig: { - produceCommand, - restoreCommand, - artifactName: "mail.tar.zst", - mail: { messageData: flags.messageData, keys: flags.keys }, - }, - }; -} diff --git a/apps/api/src/modules/mail/mail.controller.ts b/apps/api/src/modules/mail/mail.controller.ts index 49bb84608..8f74d97e7 100644 --- a/apps/api/src/modules/mail/mail.controller.ts +++ b/apps/api/src/modules/mail/mail.controller.ts @@ -30,7 +30,7 @@ import type { Context } from "hono"; import type { SSEStreamingApi } from "hono/streaming"; import crypto from "node:crypto"; import { lookup as dnsLookup } from "node:dns/promises"; -import { buildMailBackupPayload } from "./admin/backup-plan"; +import { mailBackupFlags } from "@repo/platform/engine/modules/mail/backup-plan"; // The mail server is one more backup SOURCE in the general system, so its policy // goes through the same cron validation + schedule registration as a project's. import { syncPolicySchedule, validateCronExpression } from "@repo/platform/engine/modules/backups/triggers/cron"; @@ -67,7 +67,6 @@ import { import { checkMailDelivery } from "./mail-delivery.service"; import { checkMailHealth, mailIsServing, MAIL_COMPONENTS } from "@repo/platform/engine/modules/mail/mail-health.service"; import { checkMailPortReachability } from "@repo/platform/engine/modules/mail/mail-port-reachability.service"; -import { resolveMailFlavor } from "@repo/platform/engine/modules/mail/mail-engine"; import { updatePostmasterPassword } from "./mail-credentials.service"; import { reserveMailSetup } from "./mail-setup-lease"; import { preflightMailSetup } from "./mail-setup-preflight"; @@ -1449,20 +1448,25 @@ export async function saveMailBackupPolicy(c: Context) { if (!body.destinationId) { return c.json({ error: "destinationId is required" }, 400); } + if ((body.messageData !== undefined && typeof body.messageData !== "boolean") || + (body.keys !== undefined && typeof body.keys !== "boolean")) { + return c.json({ error: "messageData and keys must be booleans" }, 400); + } const destination = await repos.backupDestination.findById(body.destinationId); if (!destination || destination.organizationId !== ctx.organizationId) { return c.json({ error: "Destination not found" }, 404); } - const messageData = body.messageData === true; - const keys = body.keys !== false; // default: include keys/secrets - // The produce/restore shell is baked HERE and executed later by the generic - // custom_command producer, which has no mail knowledge — so the topology has to be - // resolved at build time. A containerized engine keeps Postgres in a sidecar and the - // `vmail` user only inside the engine, and the old plan issued `sudo -u postgres` and - // `chown vmail` on the host, where neither exists (GH-563). - const flavor = await resolveMailFlavor(serverId); - const payload = buildMailBackupPayload(mailRow.domain, { messageData, keys }, flavor); + const existing = await repos.backupPolicy.findActiveByMailServer(serverId); + const savedFlags = mailBackupFlags(existing?.payloadConfig); + const messageData = body.messageData ?? savedFlags.messageData; + const keys = body.keys ?? savedFlags.keys; + // A mail policy stores intent; the worker resolves the current topology and + // builds the archive commands once per run through the shared mail lifecycle. + const payload = { + payloadKind: "custom_command" as const, + payloadConfig: { mail: { messageData, keys } }, + }; const cronExpression = typeof body.cronExpression === "string" && body.cronExpression.trim() @@ -1478,8 +1482,6 @@ export async function saveMailBackupPolicy(c: Context) { } } - const existing = await repos.backupPolicy.findActiveByMailServer(serverId); - // Omitted vs explicit-null, the same distinction `createPolicy` draws for // project policies (backup.service.ts) — and it has to be drawn here because // `shared` doubles as the update patch, so a blanket default would overwrite diff --git a/apps/api/test/e2e/fixtures/mail-backup/Dockerfile b/apps/api/test/e2e/fixtures/mail-backup/Dockerfile new file mode 100644 index 000000000..71cda3c5f --- /dev/null +++ b/apps/api/test/e2e/fixtures/mail-backup/Dockerfile @@ -0,0 +1,15 @@ +FROM oven/bun:1.4.4 AS bun +FROM debian:12-slim +ENV DEBIAN_FRONTEND=noninteractive container=docker +RUN printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d \ + && chmod +x /usr/sbin/policy-rc.d \ + && apt-get update && apt-get install -y --no-install-recommends \ + systemd systemd-sysv sudo postgresql dovecot-core dovecot-imapd \ + postfix amavisd-new zstd curl ca-certificates openssl netcat-openbsd \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd -g 2000 vmail && useradd -u 2000 -g vmail -d /var/vmail vmail +COPY --from=bun /usr/local/bin/bun /usr/local/bin/bun +RUN apt-get update && apt-get install -y --no-install-recommends openssh-server dovecot-pgsql \ + && rm -rf /var/lib/apt/lists/* +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/apps/api/test/e2e/fixtures/mail-backup/Dockerfile.engine b/apps/api/test/e2e/fixtures/mail-backup/Dockerfile.engine new file mode 100644 index 000000000..9ffeba4fe --- /dev/null +++ b/apps/api/test/e2e/fixtures/mail-backup/Dockerfile.engine @@ -0,0 +1,8 @@ +FROM openship-mail-backup-test:1602 +RUN apt-get update && apt-get install -y --no-install-recommends supervisor \ + && rm -rf /var/lib/apt/lists/* +# Different IDs prove that restore uses the destination's service users. +RUN groupmod -g 2101 vmail && usermod -u 2101 -g 2101 vmail \ + && groupmod -g 2301 amavis && usermod -u 2301 -g 2301 amavis +COPY apps/api/test/e2e/fixtures/mail-backup/supervisord.conf /etc/supervisor/supervisord.conf +CMD ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisor/supervisord.conf"] diff --git a/apps/api/test/e2e/fixtures/mail-backup/Dockerfile.host b/apps/api/test/e2e/fixtures/mail-backup/Dockerfile.host new file mode 100644 index 000000000..9b99c8a61 --- /dev/null +++ b/apps/api/test/e2e/fixtures/mail-backup/Dockerfile.host @@ -0,0 +1,6 @@ +FROM docker:28-dind +RUN apk add --no-cache openssh-server sudo tar zstd \ + && ssh-keygen -A && mkdir -p /root/.ssh && chmod 0700 /root/.ssh \ + && passwd -d root +# The test starts sshd after installing an ephemeral public key. +ENV DOCKER_TLS_CERTDIR="" diff --git a/apps/api/test/e2e/fixtures/mail-backup/roundtrip.ts b/apps/api/test/e2e/fixtures/mail-backup/roundtrip.ts new file mode 100644 index 000000000..e43714041 --- /dev/null +++ b/apps/api/test/e2e/fixtures/mail-backup/roundtrip.ts @@ -0,0 +1,501 @@ +/** Runs inside a disposable Linux host, so the local transport is exercised on + * the real PostgreSQL/Postfix/Dovecot filesystem, never the developer's host. */ +import assert from "node:assert/strict"; +import { Readable } from "node:stream"; +import { readFile, readdir, stat } from "node:fs/promises"; +import { join } from "node:path"; +import { createHash, randomBytes } from "node:crypto"; +import { LocalExecutor, initPlatform, resetPlatform, type CommandExecutor } from "@repo/adapters"; +import { shellQuote as sq } from "@repo/core"; +import { repos, closeDb } from "@repo/db"; +import { encrypt, decrypt } from "@repo/platform/engine/lib/encryption"; +import { encryptSecretField } from "@repo/platform/engine/lib/credential-encryption"; +import { resolvePlatformConfig } from "@repo/platform/engine/lib/platform-config"; +import { buildBackgroundContext } from "@repo/platform/engine/lib/background-context"; +import { sshManager } from "@repo/platform/engine/lib/ssh-manager"; +import { + makeFreshState, + readState, + writeState, +} from "@repo/platform/engine/modules/mail/mail-state"; +import { backupOrchestrator } from "@repo/platform/engine/modules/backups/backup.orchestrator"; +import { restoreOrchestrator } from "@repo/platform/engine/modules/backups/restore.orchestrator"; +import { + mailEngineCommand, + mailHostPath, + mailPsqlFromStdin, + pauseMailDaemons, + type MailEngineFlavor, +} from "@repo/platform/engine/modules/mail/mail-engine"; +import { + seedOrg, + seedBackupDestination, + seedBackupPolicy, + seedBackupRun, +} from "../../../helpers/seed"; + +const local = new LocalExecutor(); +const domain = "example.test"; +const user = `alice@${domain}`; +const password = "mail-restore-fixture-password"; +const flavors = new WeakMap(); +const flavorFor = (exec: CommandExecutor) => flavors.get(exec) ?? "host"; +const mail = (exec: CommandExecutor, command: string) => + exec.exec(mailEngineCommand(flavorFor(exec), `sh -ec ${sq(command)}`)); +const mailFile = (exec: CommandExecutor, path: string) => + exec.readFile(mailHostPath(flavorFor(exec), path)); +const sql = (exec: CommandExecutor, statement: string, db = "vmail") => + exec.exec( + `${flavorFor(exec) === "container" ? "docker exec openship-mail-db psql -U postgres" : "sudo -u postgres psql"} -v ON_ERROR_STOP=1 -d ${sq(db)} -Atc ${sq(statement)}`, + ); +const sha = (text: string) => createHash("sha256").update(text).digest("hex"); + +async function setup( + exec: CommandExecutor, + serverId: string, + dbPassword: string, + source: boolean, + flavor: MailEngineFlavor = "host", +) { + flavors.set(exec, flavor); + if (flavor === "host") await exec.exec("systemctl start postgresql"); + await sql( + exec, + `CREATE ROLE vmail LOGIN PASSWORD '${dbPassword}'; CREATE ROLE vmailadmin LOGIN; CREATE ROLE amavisd LOGIN; CREATE ROLE iredapd LOGIN; CREATE ROLE fail2ban LOGIN`, + "postgres", + ); + for (const name of ["vmail", "amavisd", "iredapd", "fail2ban"]) { + if (flavor === "container" && name === "vmail") continue; + await exec.exec( + `${flavor === "container" ? "docker exec openship-mail-db createdb -U postgres" : "sudo -u postgres createdb"} ${name}`, + ); + } + const schema = await readFile("apps/email/engine/samples/iredmail/iredmail.pgsql", "utf8"); + assert.equal( + (await exec.execWithInput!(mailPsqlFromStdin(flavor), Readable.from([schema]))).code, + 0, + ); + await sql( + exec, + `GRANT SELECT ON ALL TABLES IN SCHEMA public TO vmail; + CREATE SCHEMA mail_app; + CREATE TABLE mail_app.notes(id serial PRIMARY KEY, body text NOT NULL); + INSERT INTO mail_app.notes(body) VALUES ('${source ? "source-note" : "target-note"}'); + INSERT INTO domain(domain) VALUES ('example.test'),('second.test'); + INSERT INTO mailbox(username,password,domain,maildir,storagebasedirectory,storagenode) VALUES ('${user}','{PLAIN}${password}','example.test','example.test/alice/','/var/vmail','vmail1'); + INSERT INTO alias(address,domain) VALUES ('team@example.test','example.test'); + INSERT INTO forwardings(address,forwarding,domain) VALUES ('team@example.test','${user}','example.test'); + INSERT INTO alias_domain(alias_domain,target_domain) VALUES ('alias.test','example.test'); + INSERT INTO sender_bcc_user(username,bcc_address,domain) VALUES ('${user}','audit@example.test','example.test');`, + ); + for (const name of ["amavisd", "iredapd", "fail2ban"]) { + await sql( + exec, + `CREATE TABLE restored_rows(id serial PRIMARY KEY, body text); INSERT INTO restored_rows(body) VALUES ('${source ? "source" : "target"}-${name}');`, + name, + ); + } + await mail( + exec, + `sh -ec ${sq(`install -d -o vmail -g vmail /var/vmail/vmail1/example.test/alice/Maildir/{cur,new,tmp} + chown -R vmail:vmail /var/vmail + install -d -m 0700 /var/lib/dkim + openssl genrsa -out /var/lib/dkim/example.test.pem 2048 2>/dev/null + openssl genrsa -out /var/lib/dkim/second.test.pem 2048 2>/dev/null + chown -R amavis:amavis /var/lib/dkim + chmod 0700 /var/lib/dkim + chmod 0600 /var/lib/dkim/*.pem`)}`, + ); + const config = `protocols = imap +listen = 127.0.0.1 +ssl = no +disable_plaintext_auth = no +mail_location = maildir:/var/vmail/vmail1/%d/%n/Maildir +auth_mechanisms = plain login +passdb { + driver = sql + args = /etc/dovecot/backup-test-sql.conf +} +userdb { + driver = static + args = uid=vmail gid=vmail home=/var/vmail/vmail1/%d/%n +} +service auth-worker { + user = root +} +`; + await exec.writeFile(mailHostPath(flavor, "/etc/dovecot/dovecot.conf"), config); + await exec.writeFile( + mailHostPath(flavor, "/etc/dovecot/backup-test-sql.conf"), + `driver = pgsql\nconnect = host=127.0.0.1 dbname=vmail user=vmail password=${dbPassword}\npassword_query = SELECT password FROM mailbox WHERE username = '%u' AND active = 1\n`, + ); + await mail(exec, "chmod 0600 /etc/dovecot/backup-test-sql.conf"); + const amavis = `use strict; +$mydomain = 'example.test'; +$myhostname = 'mail.example.test'; +$daemon_user = 'amavis'; $daemon_group = 'amavis'; +$MYHOME = '/var/lib/amavis'; +$TEMPBASE = "$MYHOME/tmp"; +$inet_socket_bind = '127.0.0.1'; $inet_socket_port = 10024; +@local_domains_maps = (1); +@bypass_virus_checks_maps = (1); @bypass_spam_checks_maps = (1); +$forward_method = undef; $notify_method = undef; +$do_syslog = 0; $max_servers = 1; +dkim_key('example.test', 'dkim', '/var/lib/dkim/example.test.pem'); +dkim_key('second.test', 'dkim', '/var/lib/dkim/second.test.pem'); +@dkim_signature_options_bysender_maps = ( { + '.example.test' => { d => 'example.test' }, + '.second.test' => { d => 'second.test' }, +} ); +1; +`; + await exec.writeFile(mailHostPath(flavor, "/etc/amavis/conf.d/50-user"), amavis); + await mail( + exec, + `sh -ec ${sq(`install -d -o amavis -g amavis /var/lib/amavis/tmp /run/amavis; postfix check; ${flavor === "host" ? "systemctl restart amavis postfix dovecot" : "supervisorctl start amavis postfix dovecot"}`)}`, + ); + const state = makeFreshState(serverId, domain); + state.secrets = { + VMAIL_DB_BIND_PASSWD: dbPassword, + DOMAIN_ADMIN_PASSWD_PLAIN: "postmaster-fixture", + }; + state.platformMailbox = { + email: user, + password: encrypt(password), + smtpHost: "mail.example.test", + smtpPort: 587, + secure: false, + updatedAt: new Date().toISOString(), + }; + state.dnsAcknowledged = true; + state.ptrAcknowledged = true; + state.finishedAt = new Date().toISOString(); + await writeState(exec, state); + await mail( + exec, + `sh -ec ${sq(`install -d -o amavis -g amavis /var/lib/amavis/quarantine /var/lib/amavis/db + printf 'Subject: Retained quarantine\\n\\nQuarantined message\\n' > /var/lib/amavis/quarantine/${source ? "source" : "target"}-marker + printf filter-state > /var/lib/amavis/db/${source ? "source" : "target"}-marker + chown -R amavis:amavis /var/lib/amavis/quarantine /var/lib/amavis/db`)}`, + ); + if (source) + await exec.exec( + "install -d /srv; mv /var/vmail /srv/mail-data; ln -s /srv/mail-data /var/vmail", + ); + if (source) + await exec.exec( + "mkdir -p /var/vmail/link-test; printf preserved-link-data > /var/vmail/link-test/a; ln /var/vmail/link-test/a /var/vmail/link-test/b; ln -s a /var/vmail/link-test/relative", + ); + // Deliver via Dovecot, rather than merely dropping a file into an assumed path. + if (source) { + const body = + "Subject: Preserved attachment\nMIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=fixture\n\n--fixture\nContent-Type: text/plain\n\nPreserve this mail.\n--fixture\nContent-Type: application/octet-stream\nContent-Transfer-Encoding: base64\n\n" + + Buffer.from(Array.from({ length: 256 }, (_, i) => i)).toString("base64") + + "\n--fixture--\n"; + const result = await exec.execWithInput!( + `doveadm save -u ${sq(user)} -m INBOX`, + Readable.from([body]), + ); + assert.equal(result.code, 0, result.stderr); + await exec.exec(`doveadm mailbox create -u ${sq(user)} Archive`); + assert.equal( + ( + await exec.execWithInput!( + `doveadm save -u ${sq(user)} -m Archive`, + Readable.from(["Subject: Saved folder\n\nArchive message\n"]), + ) + ).code, + 0, + ); + await exec.exec( + "printf 'Subject: Queued mail\n\nHold this message\n' | sendmail -f alice@example.test nobody@example.invalid; postsuper -h ALL", + ); + } +} + +async function waitForRestore(id: string, wanted: string) { + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + const row = await repos.backupRestore.findById(id); + if (row?.status === wanted) return row; + assert( + !["failed", "cancelled", "succeeded"].includes(row?.status ?? ""), + row?.errorMessage ?? `unexpected ${row?.status}`, + ); + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error(`Restore ${id} did not become ${wanted}`); +} + +try { + await initPlatform(resolvePlatformConfig()); + const org = await seedOrg({ ownsHost: true }); + const ctx = buildBackgroundContext(org); + const source = await repos.server.create({ + organizationId: org.organizationId, + name: "This server", + isLocal: true, + sshHost: "127.0.0.1", + }); + const target = await repos.server.create({ + organizationId: org.organizationId, + name: "Mail destination", + sshHost: "mail-target", + sshUser: "root", + sshAuthMethod: "key", + sshPrivateKey: encryptSecretField(await readFile("/tmp/mail-test-key", "utf8")), + }); + await repos.mailServer.markInstalled(source.id, domain); + await repos.mailServer.markInstalled(target.id, domain); + await setup(local, source.id, "source-db-password", true); + await sshManager.withExecutor(target.id, (exec) => + setup( + exec, + target.id, + "target-db-password", + false, + process.env.MAIL_TEST_TARGET_FLAVOR as MailEngineFlavor, + ), + ); + if (process.env.MAIL_TEST_TARGET_FLAVOR === "container") { + await sshManager.withExecutor(target.id, async (exec) => { + const paused = await pauseMailDaemons(exec, "container"); + // Supervisor's STOPPED verdict alone is insufficient: the old image's + // start-fg wrapper can leave Postfix's master and queue writers alive. + await mail(exec, "if postfix status >/dev/null 2>&1; then exit 1; fi"); + await paused.resume(); + await mail(exec, "postfix status"); + }); + } + + const original = await local.exec(`doveadm fetch -u ${sq(user)} 'mailbox text' all`); + const keys = await local.exec("sha256sum /var/lib/dkim/*.pem"); + const mailboxHash = sha(original); + const destination = await seedBackupDestination(org.organizationId, { + kind: "local", + endpoint: "/var/lib/openship/backups/mail-test", + }); + const payload = { + payloadKind: "custom_command" as const, + payloadConfig: { mail: { messageData: true, keys: true } }, + }; + const policy = await seedBackupPolicy(destination.id, { + sourceKind: "mail_server", + mailServerId: source.id, + payloadKind: payload.payloadKind, + payloadConfig: payload.payloadConfig, + }); + const run = await seedBackupRun(org.organizationId, { + policyId: policy.id, + destinationId: destination.id, + mailServerId: source.id, + sourceKind: "mail_server", + status: "queued", + }); + await backupOrchestrator.execute(run.id); + const captured = await repos.backupRun.findById(run.id); + assert.equal(captured?.status, "succeeded", captured?.errorMessage ?? undefined); + assert((captured?.bytesTransferred ?? 0) > 0); + const artifacts = captured!.artifacts as Array<{ + key: string; + metadata: Record; + }>; + assert.deepEqual(artifacts[0]!.metadata.mail, { + version: 2, + domain, + messageData: true, + keys: true, + }); + assert( + !JSON.stringify(captured).includes(password), + "portable passwords leaked into run metadata", + ); + assert.equal( + (await readdir("/root/.openship")).filter((name) => name.startsWith("mail-backup-")).length, + 0, + ); + await local.exec("systemctl is-active --quiet postfix dovecot amavis"); + + async function restore( + mode: "in_place" | "to_fork", + runId = run.id, + wanted = "succeeded", + forkId = target.id, + ) { + const token = randomBytes(24).toString("base64url"); + const prepared = await restoreOrchestrator.beginPrepare({ + runId, + mode, + ...(mode === "to_fork" ? { forkMailServerId: forkId } : {}), + confirmationToken: token, + trigger: { source: "manual", userId: org.userId }, + }); + await waitForRestore(prepared.restoreId, "prepared"); + await restoreOrchestrator.apply(ctx, prepared.restoreId, prepared.confirmationToken); + return waitForRestore(prepared.restoreId, wanted); + } + await local.exec(`doveadm expunge -u ${sq(user)} mailbox '*' all`); + await sql( + local, + "DELETE FROM alias; DELETE FROM sender_bcc_user; UPDATE mail_app.notes SET body='changed'", + ); + await restore("in_place"); + await local.exec( + 'test -L /var/vmail; test /var/vmail/link-test/a -ef /var/vmail/link-test/b; test "$(readlink /var/vmail/link-test/relative)" = a', + ); + assert.equal( + sha(await local.exec(`doveadm fetch -u ${sq(user)} 'mailbox text' all`)), + mailboxHash, + ); + assert.equal((await sql(local, "SELECT count(*) FROM alias")).trim(), "1"); + assert.equal((await sql(local, "SELECT body FROM mail_app.notes")).trim(), "source-note"); + await restore("to_fork"); + await sshManager.withExecutor(target.id, async (exec) => { + assert.equal( + sha(await mail(exec, `doveadm fetch -u ${sq(user)} 'mailbox text' all`)), + mailboxHash, + ); + assert.equal((await mail(exec, "sha256sum /var/lib/dkim/*.pem")).trim(), keys.trim()); + assert.equal((await sql(exec, "SELECT body FROM mail_app.notes")).trim(), "source-note"); + assert.equal((await sql(exec, "SELECT count(*) FROM sender_bcc_user")).trim(), "1"); + for (const db of ["amavisd", "iredapd", "fail2ban"]) + assert.equal((await sql(exec, "SELECT body FROM restored_rows", db)).trim(), `source-${db}`); + const state = await readState(exec, { strict: true }); + assert.equal(state?.serverId, target.id); + assert.equal(state?.secrets.VMAIL_DB_BIND_PASSWD, "target-db-password"); + assert.equal(decrypt(state!.platformMailbox!.password), password); + assert.equal(state?.dnsAcknowledged, false); + assert(state?.additionalDomains?.["second.test"]?.records.dkim); + assert( + (await mailFile(exec, "/etc/dovecot/backup-test-sql.conf")).includes("target-db-password"), + ); + assert( + (await mail(exec, `doveadm auth test ${sq(user)} ${sq(password)}`)).includes( + "auth succeeded", + ), + ); + assert((await mail(exec, "postqueue -p")).includes("nobody@example.invalid")); + await mail( + exec, + 'test /var/vmail/link-test/a -ef /var/vmail/link-test/b; test "$(readlink /var/vmail/link-test/relative)" = a', + ); + assert( + (await mail(exec, "cat /var/lib/amavis/quarantine/source-marker")).includes( + "Quarantined message", + ), + ); + assert.equal((await mail(exec, "cat /var/lib/amavis/db/source-marker")).trim(), "filter-state"); + await mail(exec, "test ! -e /var/lib/amavis/quarantine/target-marker"); + if (flavorFor(exec) === "host") + await exec.exec("systemctl is-active --quiet postfix dovecot amavis"); + else + assert( + (await mail(exec, "supervisorctl status postfix dovecot amavis")) + .split("\n") + .every((line) => line.includes("RUNNING")), + ); + assert((await repos.mailServer.get(target.id))?.installedAt); + }); + if (process.env.MAIL_TEST_TARGET_FLAVOR === "container") { + const reversePolicy = await seedBackupPolicy(destination.id, { + sourceKind: "mail_server", + mailServerId: target.id, + payloadKind: payload.payloadKind, + payloadConfig: payload.payloadConfig, + }); + const reverseRun = await seedBackupRun(org.organizationId, { + policyId: reversePolicy.id, + destinationId: destination.id, + mailServerId: target.id, + sourceKind: "mail_server", + status: "queued", + }); + await backupOrchestrator.execute(reverseRun.id); + const reverse = await repos.backupRun.findById(reverseRun.id); + assert.equal(reverse?.status, "succeeded", reverse?.errorMessage ?? undefined); + await local.exec(`doveadm expunge -u ${sq(user)} mailbox '*' all`); + await restore("to_fork", reverseRun.id, "succeeded", source.id); + assert.equal( + sha(await local.exec(`doveadm fetch -u ${sq(user)} 'mailbox text' all`)), + mailboxHash, + ); + assert.equal( + (await readState(local, { strict: true }))?.secrets.VMAIL_DB_BIND_PASSWD, + "source-db-password", + ); + } + // A requested secret cannot silently disappear, and failure must undo the + // writer pause even though the archive stream failed before upload completed. + await local.exec("mv /usr/bin/pg_dump /usr/bin/pg_dump-held"); + const failedCapture = await seedBackupRun(org.organizationId, { + policyId: policy.id, + destinationId: destination.id, + mailServerId: source.id, + sourceKind: "mail_server", + status: "queued", + }); + await backupOrchestrator.execute(failedCapture.id); + assert.equal((await repos.backupRun.findById(failedCapture.id))?.status, "failed"); + await local.exec( + "systemctl is-active --quiet postfix dovecot amavis; mv /usr/bin/pg_dump-held /usr/bin/pg_dump", + ); + assert.equal( + (await readdir("/root/.openship")).filter((name) => name.startsWith("mail-backup-")).length, + 0, + ); + + // A syntactically valid archive whose database replay fails must roll back + // that database, report the failed restore, and keep mail stopped for review. + const archive = join(destination.endpoint!, artifacts[0]!.key); + const broken = join(destination.endpoint!, "broken.sql.tar.zst"); + await local.exec(`mkdir -m 0700 /tmp/mail-broken; zstd -dc ${sq(archive)} | tar -x -C /tmp/mail-broken + printf '\\nSELECT nonexistent_restore_column;\\n' >> /tmp/mail-broken/vmail.sql + tar -c -C /tmp/mail-broken . | zstd -c > ${sq(broken)}`); + await sql(local, "UPDATE mail_app.notes SET body='must-survive-failed-restore'"); + const brokenRun = await seedBackupRun(org.organizationId, { + policyId: policy.id, + destinationId: destination.id, + mailServerId: source.id, + sourceKind: "mail_server", + status: "succeeded", + artifacts: [ + { + ...artifacts[0]!, + key: "broken.sql.tar.zst", + sizeBytes: (await stat(broken)).size, + sha256: createHash("sha256") + .update(await readFile(broken)) + .digest("hex"), + }, + ], + }); + const failedRestore = await restore("in_place", brokenRun.id, "failed"); + assert.match(failedRestore.errorMessage ?? "", /nonexistent_restore_column/); + assert.equal( + (await sql(local, "SELECT body FROM mail_app.notes")).trim(), + "must-survive-failed-restore", + ); + assert.equal((failedRestore.meta as Record).serviceLeftStopped, true); + assert.equal((failedRestore.meta as Record).partialWrite, true); + assert.equal((await local.exec("systemctl is-active dovecot || true")).trim(), "inactive"); + assert.equal( + (await readdir("/root/.openship")).filter((name) => name.startsWith("mail-backup-")).length, + 0, + ); + // Explicit retry repairs the stopped partial restore and resumes only services + // that were running when it was requested. + await local.exec("systemctl start postfix dovecot amavis"); + await restore("in_place"); + assert.equal( + sha(await local.exec(`doveadm fetch -u ${sq(user)} 'mailbox text' all`)), + mailboxHash, + ); + console.log( + "MAIL_BACKUP_ROUNDTRIP_OK: local capture, in-place restore, SSH migration, messages/attachments/folders, queue, all mail tables, keys, destination credentials, registration, failed capture recovery and failed SQL rollback", + ); +} finally { + await local.dispose(); + await sshManager.destroy(); + resetPlatform(); + await closeDb(); +} diff --git a/apps/api/test/e2e/fixtures/mail-backup/supervisord.conf b/apps/api/test/e2e/fixtures/mail-backup/supervisord.conf new file mode 100644 index 000000000..adb6f31da --- /dev/null +++ b/apps/api/test/e2e/fixtures/mail-backup/supervisord.conf @@ -0,0 +1,22 @@ +[supervisord] +user=root +logfile=/tmp/supervisord.log +pidfile=/run/supervisord.pid +[unix_http_server] +file=/run/supervisor.sock +[rpcinterface:supervisor] +supervisor.rpcinterface_factory=supervisor.rpcinterface:make_main_rpcinterface +[supervisorctl] +serverurl=unix:///run/supervisor.sock +[program:postfix] +command=/usr/sbin/postfix start-fg +autostart=false +autorestart=true +[program:dovecot] +command=/usr/sbin/dovecot -F +autostart=false +autorestart=true +[program:amavis] +command=/usr/sbin/amavisd foreground +autostart=false +autorestart=true diff --git a/apps/api/test/e2e/mail-backup-roundtrip.e2e.test.ts b/apps/api/test/e2e/mail-backup-roundtrip.e2e.test.ts new file mode 100644 index 000000000..a7db18b93 --- /dev/null +++ b/apps/api/test/e2e/mail-backup-roundtrip.e2e.test.ts @@ -0,0 +1,243 @@ +import { execFile, spawn } from "node:child_process"; +import { promisify } from "node:util"; +import { pipeline } from "node:stream/promises"; +import { MAIL_CONTAINER_MOUNTS } from "@repo/adapters"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { beforeAll, afterAll, expect, it } from "vitest"; +import { shellQuote as sq } from "@repo/core"; +import { describeDockerE2E, requireDocker } from "../helpers/docker-e2e"; + +const exec = promisify(execFile); +const root = join(dirname(fileURLToPath(import.meta.url)), "../../../.."); +for (const targetFlavor of ["host", "container"] as const) { + const prefix = `openship-e2e-mail-backup-${targetFlavor}-${process.pid.toString(36)}`; + const source = `${prefix}-source`; + const target = `${prefix}-target`; + const network = `${prefix}-net`; + const image = "openship-mail-backup-test:1602"; + let keys = ""; + async function docker(args: string[]) { + return exec("docker", args, { timeout: 600_000, maxBuffer: 8 * 1024 * 1024 }); + } + + describeDockerE2E(`mail backup and migration to a ${targetFlavor} mail engine`, () => { + beforeAll(async () => { + await requireDocker(); + await docker([ + "build", + "-f", + join(root, "apps/api/test/e2e/fixtures/mail-backup/Dockerfile"), + "-t", + image, + root, + ]); + if (targetFlavor === "container") { + await docker([ + "build", + "-f", + join(root, "apps/api/test/e2e/fixtures/mail-backup/Dockerfile.host"), + "-t", + "openship-mail-backup-host:1602", + root, + ]); + await docker([ + "build", + "-f", + join(root, "apps/api/test/e2e/fixtures/mail-backup/Dockerfile.engine"), + "-t", + "openship-mail-backup-engine:1602", + root, + ]); + await docker(["pull", "postgres:16-alpine"]); + } + keys = await mkdtemp(join(tmpdir(), "openship-mail-key-")); + await exec( + "bun", + [ + "build", + join(root, "apps/api/test/e2e/fixtures/mail-backup/roundtrip.ts"), + "--target=bun", + "--external=cpu-features", + "--external=@electric-sql/pglite", + `--outdir=${keys}`, + ], + { cwd: root }, + ); + await exec("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", join(keys, "key")]); + // Both hosts are disposable; no outside SMTP/DNS or host Docker socket. + await docker(["network", "create", "--internal", network]); + for (const name of [source, target]) { + await docker([ + "run", + "-d", + "--name", + name, + "--hostname", + "mail.example.test", + "--privileged", + "--cgroupns=private", + "--tmpfs", + "/run", + "--tmpfs", + "/run/lock", + "--network", + network, + ...(name === target ? ["--network-alias", "mail-target"] : []), + ...(name === target && targetFlavor === "container" + ? [ + "-v", + `${prefix}-docker:/var/lib/docker`, + "openship-mail-backup-host:1602", + "--host=unix:///var/run/docker.sock", + "--bridge=none", + "--iptables=false", + "--storage-driver=vfs", + ] + : [image]), + ]); + const deadline = Date.now() + 60_000; + for (;;) { + try { + await docker( + name === target && targetFlavor === "container" + ? ["exec", name, "docker", "info"] + : ["exec", name, "systemctl", "is-active", "--quiet", "multi-user.target"], + ); + break; + } catch (error) { + if (Date.now() >= deadline) throw error; + await new Promise((r) => setTimeout(r, 200)); + } + } + } + if (targetFlavor === "container") { + const save = spawn("docker", [ + "save", + "openship-mail-backup-engine:1602", + "postgres:16-alpine", + ]); + const load = spawn("docker", ["exec", "-i", target, "docker", "load"]); + const errors: Buffer[] = []; + save.stderr.on("data", (chunk) => errors.push(chunk)); + load.stderr.on("data", (chunk) => errors.push(chunk)); + load.stdout.resume(); + const closed = [save, load].map( + (child) => + new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code) => + code === 0 ? resolve() : reject(new Error(Buffer.concat(errors).toString())), + ); + }), + ); + await Promise.all([pipeline(save.stdout, load.stdin), ...closed]); + const command = [ + ...MAIL_CONTAINER_MOUNTS.map((mount) => `mkdir -p ${sq(mount.host)}`), + "docker create --name mail-seed openship-mail-backup-engine:1602", + ...MAIL_CONTAINER_MOUNTS.filter( + (mount) => mount.seed && mount.container.startsWith("/etc/"), + ).map((mount) => `docker cp mail-seed:${mount.container}/. ${sq(mount.host)}`), + "docker rm mail-seed", + "docker run -d --network host --name openship-mail-db -e POSTGRES_DB=vmail -e POSTGRES_PASSWORD=mail-fixture-db-root postgres:16-alpine", + `docker run -d --network host --hostname mail.example.test --name openship-mail ${MAIL_CONTAINER_MOUNTS.map((mount) => `-v ${sq(`${mount.host}:${mount.container}${mount.readonly ? ":ro" : ""}`)}`).join(" ")} openship-mail-backup-engine:1602`, + "for attempt in $(seq 1 100); do if docker exec openship-mail-db pg_isready -U postgres -d vmail >/dev/null 2>&1 && docker exec openship-mail test -S /run/supervisor.sock; then exit 0; fi; sleep 0.1; done; docker logs openship-mail >&2; docker logs openship-mail-db >&2; df -h /var/lib/docker >&2; exit 1", + ].join("\n"); + await docker(["exec", target, "sh", "-ec", command]); + } + // Copy the small bundle and its portable database assets. Bind mounts would + // require the daemon to share the checkout's filesystem (remote/Colima do not). + await docker([ + "exec", + source, + "mkdir", + "-p", + "/workspace/node_modules/@electric-sql/pglite", + "/workspace/apps/email/engine/samples/iredmail", + ]); + await docker(["cp", join(keys, "roundtrip.js"), `${source}:/workspace/roundtrip.js`]); + await docker([ + "cp", + `${join(root, "node_modules/@electric-sql/pglite")}/.`, + `${source}:/workspace/node_modules/@electric-sql/pglite`, + ]); + await docker(["cp", join(root, "packages/db/drizzle"), `${source}:/workspace/drizzle`]); + await docker([ + "cp", + join(root, "apps/email/engine/samples/iredmail/iredmail.pgsql"), + `${source}:/workspace/apps/email/engine/samples/iredmail/iredmail.pgsql`, + ]); + await docker(["cp", join(keys, "key"), `${source}:/tmp/mail-test-key`]); + const publicKey = await readFile(join(keys, "key.pub"), "utf8"); + await docker([ + "exec", + target, + "sh", + "-ec", + `install -d -m 0700 /root/.ssh; printf %s ${sq(publicKey)} > /root/.ssh/authorized_keys; chmod 0600 /root/.ssh/authorized_keys; ${targetFlavor === "container" ? "/usr/sbin/sshd" : "systemctl start ssh"}`, + ]); + // The fixture models a bare host: PID 1 is real systemd in a private + // cgroup namespace. Remove only this fixture's Docker marker so the normal + // host resolver selects LocalExecutor, exactly as on the reported install. + await docker(["exec", source, "rm", "-f", "/.dockerenv"]); + }, 900_000); + + afterAll(async () => { + for (const name of [source, target]) await docker(["rm", "-f", name]).catch(() => {}); + await docker(["network", "rm", network]).catch(() => {}); + if (targetFlavor === "container") + await docker(["volume", "rm", `${prefix}-docker`]).catch(() => {}); + if (keys) await rm(keys, { recursive: true, force: true }); + }); + + it("restores attachments, folders, mail databases, DKIM and target registration through the shared backup workers", async () => { + let result; + try { + result = await docker([ + "exec", + "-w", + "/workspace", + "-e", + "NODE_ENV=test", + "-e", + "DEPLOY_MODE=bare", + "-e", + "CLOUD_MODE=false", + "-e", + "INTERNAL_TOKEN=mail-fixture-internal-token-00000000000000000000", + "-e", + "BETTER_AUTH_SECRET=mail-fixture-controller-secret-00000000000000000000", + "-e", + "BACKUP_ALLOW_LOCAL_DESTINATION=true", + "-e", + "BACKUP_LOCAL_ROOT=/var/lib/openship/backups", + "-e", + "OPENSHIP_MIGRATIONS_DIR=/workspace/drizzle", + "-e", + `MAIL_TEST_TARGET_FLAVOR=${targetFlavor}`, + source, + "bun", + "roundtrip.js", + ]); + } catch (error) { + const failure = error as Error & { stdout?: string; stderr?: string }; + const inspect = targetFlavor === "container" + ? "docker exec openship-mail sh -c 'supervisorctl status; cat /tmp/supervisord.log /tmp/postfix*.log; postfix status; postfix check; ps aux'" + : "systemctl status postfix dovecot amavis; journalctl --no-pager -n 60 -u postfix -u dovecot -u amavis"; + const diagnostics = await docker(["exec", target, "sh", "-c", inspect]).catch( + (diagnosticError: Error & { stdout?: string; stderr?: string }) => diagnosticError, + ); + throw new Error([ + failure.message, + failure.stdout, + failure.stderr, + diagnostics.stdout, + diagnostics.stderr, + ].filter(Boolean).join("\n"), { cause: error }); + } + expect(result.stdout, result.stderr).toContain("MAIL_BACKUP_ROUNDTRIP_OK"); + }, 300_000); + }); +} diff --git a/apps/api/test/modules/backups/backup-run-durability.test.ts b/apps/api/test/modules/backups/backup-run-durability.test.ts index 34f6aeb22..f087a5b35 100644 --- a/apps/api/test/modules/backups/backup-run-durability.test.ts +++ b/apps/api/test/modules/backups/backup-run-durability.test.ts @@ -155,6 +155,7 @@ describe("backupRun.transition — status must not ride a payload that can fail" // ─── Layer 2: the orchestrator ─────────────────────────────────────────────── const h = vi.hoisted(() => ({ + recoveryFailure: false, run: null as Record | null, policy: null as Record | null, services: [] as Array>, @@ -225,6 +226,14 @@ vi.mock("@repo/db", () => ({ // `node:stream` is imported INSIDE the factory: vi.mock factories are hoisted // above the file's own imports, so closing over a top-level binding throws. +vi.mock("@repo/platform/engine/modules/mail/backup-lifecycle", () => ({ + prepareMailBackup: async () => ({ + payload: { payloadConfig: {} }, + metadata: { version: 2, domain: "example.test", messageData: false, keys: false }, + cleanup: async () => { if (h.recoveryFailure) throw new Error("mail restart refused"); }, + }), +})); + vi.mock("@repo/adapters", async () => { const { Readable, PassThrough } = await import("node:stream"); // Pulled from the real (side-effect-free) leaf module rather than restated, so a @@ -245,6 +254,7 @@ vi.mock("@repo/adapters", async () => { runPrefix: () => "pfx/", buildManifest: () => ({ version: 1 }), resolveDestination: () => ({ + deleteMany: async (keys: string[]) => ({ deleted: keys, failed: [] }), preflight: async () => ({ ok: true }), put: async (_key: string, body: AsyncIterable) => { let bytesWritten = 0; @@ -270,7 +280,11 @@ vi.mock("@repo/adapters", async () => { }; }, }), - resolveProducer: () => ({ async *produce() {} }), + resolveProducer: () => ({ + async *produce() { + yield { name: "vmail.tar.zst", stream: Readable.from([Buffer.from("bytes")]), sizeHint: 5, payloadKind: "custom_command", metadata: h.artifactMetadata }; + }, + }), }; }); @@ -312,6 +326,7 @@ import { boundedStorableText } from "@repo/platform/engine/modules/deployments/b /** Wire the orchestrator to the REAL run repo over the Postgres-shaped fake, so * a rejected write fails exactly where it fails in production. */ function wireRun(): PgFake { + h.policy = { ...h.policy, payloadKind: "custom_command" }; const pg = makePgFake({ id: "bkr_live", status: "queued" }); const repo = createBackupRunRepo(pg.db as never); h.run = { @@ -330,6 +345,7 @@ function wireRun(): PgFake { } beforeEach(() => { + h.recoveryFailure = false; h.executionRow = null; h.policy = { id: "pol_1", @@ -509,6 +525,15 @@ describe("a terminal status is final — one owner per verdict", () => { }); describe("BackupOrchestrator.execute — hook output cannot fail the backup", () => { + it("records source recovery failures before the terminal verdict and still acknowledges execution", async () => { + const pg = wireRun(); + h.recoveryFailure = true; + await new BackupOrchestrator().execute("bkr_live"); + expect(pg.row.status).toBe("failed"); + expect(pg.row.errorMessage).toContain("Mail recovery needs attention: mail restart refused"); + expect(h.acknowledgements).toEqual([{ runId: "bkr_live", status: "failed" }]); + }); + it("delivers failure references when policy and destination lookup are unavailable", async () => { const pg = wireRun(); h.policy = null; diff --git a/apps/api/test/modules/backups/custom-command-config.test.ts b/apps/api/test/modules/backups/custom-command-config.test.ts index c3978c506..868e91003 100644 --- a/apps/api/test/modules/backups/custom-command-config.test.ts +++ b/apps/api/test/modules/backups/custom-command-config.test.ts @@ -21,7 +21,7 @@ import { describe, expect, it, beforeEach, vi } from "vitest"; import { PassThrough, Readable } from "node:stream"; import { createHash } from "node:crypto"; -import { buildMailBackupPayload } from "../../../src/modules/mail/admin/backup-plan"; +import { buildMailBackupPayload } from "@repo/platform/engine/modules/mail/backup-plan"; const h = vi.hoisted(() => ({ /** payloadConfig the policy under test carries. */ @@ -142,6 +142,15 @@ vi.mock("@repo/platform/engine/lib/deployment-runtime", () => ({ resolveDeploymentPlatform: async () => ({ platform: { runtime: { name: "bare" } } }), })); vi.mock("@repo/platform/engine/lib/encryption", () => ({ decryptEnvMap: (v: unknown) => v })); +// This suite verifies the generic producer/manifest boundary. The mail lifecycle +// has its own real-host round trip; keep its private staging out of this fixture. +vi.mock("@repo/platform/engine/modules/mail/backup-lifecycle", () => ({ + prepareMailBackup: async () => ({ + payload: { payloadConfig: h.payloadConfig }, + metadata: { version: 2, domain: "mail.example.com", messageData: true, keys: true }, + cleanup: async () => {}, + }), +})); vi.mock("@repo/platform/engine/lib/notification-dispatcher", () => ({ notification: { emit: () => {} }, })); diff --git a/apps/api/test/modules/mail/backup-script.ts b/apps/api/test/modules/mail/backup-script.ts new file mode 100644 index 000000000..4a2132172 --- /dev/null +++ b/apps/api/test/modules/mail/backup-script.ts @@ -0,0 +1,11 @@ +import { execFileSync } from "node:child_process"; +import { expect } from "vitest"; + +/** Decode the one root shell argument without executing its contents. Tests + * inspect the actual script, independent of its outer shell-quoting layer. */ +export function backupScript(command: string): string { + expect(command).toMatch(/^sudo -n sh -c /); + return execFileSync("sh", ["-c", command.replace(/^sudo -n sh -c /, "printf '%s' ")], { + encoding: "utf8", + }); +} diff --git a/apps/api/test/modules/mail/mail-backup-flavor.test.ts b/apps/api/test/modules/mail/mail-backup-flavor.test.ts index 8635a38f2..961210557 100644 --- a/apps/api/test/modules/mail/mail-backup-flavor.test.ts +++ b/apps/api/test/modules/mail/mail-backup-flavor.test.ts @@ -1,81 +1,40 @@ -/** - * The backup shell has to be written for the topology it will run on (GH-563). - * - * The produce/restore commands are BAKED here and executed later by the generic - * custom_command producer over a bare SSH executor, which knows nothing about mail. So a - * containerized engine cannot be discovered at run time - the plan has to already say - * `docker exec`. It did not: it issued `sudo -u postgres pg_dump` and - * `chown -R vmail:vmail` on the host, where on a container box there is no `postgres` - * user, no `pg_dump`, and no `vmail`. The dump aborted the whole backup under `set -e`, - * and on restore the chown was masked by `|| true`, leaving maildirs root-owned and - * unreadable by Dovecot while the run reported success. - */ +import "./_setup-env"; import { describe, expect, it } from "vitest"; -import { buildMailBackupPayload } from "../../../src/modules/mail/admin/backup-plan"; +import { buildMailBackupPayload } from "@repo/platform/engine/modules/mail/backup-plan"; +import { MAIL_CONTAINER_MOUNTS } from "@repo/adapters"; +import { backupScript } from "./backup-script"; -const FLAGS = { messageData: true, keys: true }; - -describe("mail backup plan targets the right topology", () => { - it("[container] runs Postgres in the sidecar and chown in the engine", () => { - const { payloadConfig } = buildMailBackupPayload("example.com", FLAGS, "container"); - const { produceCommand: produce, restoreCommand: restore } = payloadConfig; - - // Nothing may assume a host-side postgres or vmail user. - expect(produce).not.toContain("sudo -u postgres"); - expect(restore).not.toContain("sudo -u postgres"); - expect(produce).toContain("docker exec openship-mail-db pg_dump -U postgres"); - - // The dump file is staged in the producer's $tmp ON THE HOST, which the sidecar - // cannot see - so the replay must arrive over stdin, and docker exec needs -i. +const flags = { messageData: true, keys: true }; +describe("mail archive topology", () => { + it("runs database tools in the sidecar and repairs mail ownership in the engine", () => { + const plan = buildMailBackupPayload("example.test", flags, "container").payloadConfig; + const capture = backupScript(plan.produceCommand); + const restore = backupScript(plan.restoreCommand); + expect(capture).toContain("docker exec openship-mail-db pg_dump -U postgres"); expect(restore).toContain("docker exec -i openship-mail-db psql -U postgres"); - expect(restore).not.toMatch(/psql[^\n]*-f "\$tmp/); - - // Ownership must be applied where the vmail user exists: the ENGINE, not the sidecar, - // and not the host. - expect(restore).toContain("docker exec openship-mail chown -R vmail:vmail /var/vmail/vmail1"); - // ...and a failure there must not be swallowed - root-owned maildirs are unreadable. - expect(restore).not.toMatch(/chown -R vmail:vmail [^\n]*\|\| true/); - - // Daemons re-read via supervisord in the container; systemctl does not exist there. - expect(restore).toContain("supervisorctl restart postfix dovecot amavis"); - expect(restore).not.toContain("systemctl reload"); + expect(restore).toContain("--single-transaction"); + expect(restore).toContain("docker exec openship-mail chown -R -H vmail:vmail /var/vmail"); + expect(restore).not.toContain("sudo -u postgres"); + expect(restore).not.toMatch(/chown[^\n]*\|\| true/); }); - - it("[host] keeps the legacy sudo/systemctl form", () => { - const { payloadConfig } = buildMailBackupPayload("example.com", FLAGS, "host"); - const { produceCommand: produce, restoreCommand: restore } = payloadConfig; - - expect(produce).toContain("sudo -u postgres pg_dump -d vmail"); - expect(restore).toContain("sudo -u postgres psql"); - expect(restore).toContain("chown -R vmail:vmail /var/vmail/vmail1"); - expect(restore).toContain("systemctl reload postfix dovecot"); - expect(produce).not.toContain("docker exec"); - expect(restore).not.toContain("docker exec"); + it("uses native database tools on bare hosts", () => { + const plan = buildMailBackupPayload("example.test", flags, "host").payloadConfig; + expect(backupScript(plan.produceCommand)).toContain("sudo -u postgres pg_dump"); + expect(backupScript(plan.restoreCommand)).toContain("sudo -u postgres psql"); + expect(backupScript(plan.restoreCommand)).not.toContain("docker exec"); }); - - it("proves sudo works before relying on it to tell absent from unreadable", () => { - // The `sudo -n test`/`sudo -n cp` reads below only distinguish those two cases if sudo - // is known to work. That used to be guaranteed by `sudo -u postgres pg_dump` running - // first under `set -e`; routing the dump through the sidecar removed the guarantee on - // container boxes, so the probe has to be explicit - otherwise a failing `test` reads - // as absence and the archive is stamped `keys: true` with no keys in it. - for (const flavor of ["container", "host"] as const) { - const { payloadConfig } = buildMailBackupPayload("example.com", FLAGS, flavor); - const produce = payloadConfig.produceCommand; - expect(produce).toContain("sudo -n true"); - expect(produce.indexOf("sudo -n true")).toBeLessThan(produce.indexOf("sudo -n test -d /var/lib/dkim")); + it("captures complete mail roots through the installation's canonical mount map", () => { + const plan = buildMailBackupPayload("example.test", flags, "container").payloadConfig; + const capture = backupScript(plan.produceCommand); + const restore = backupScript(plan.restoreCommand); + for (const path of ["/var/vmail", "/var/lib/dkim"]) { + const mount = MAIL_CONTAINER_MOUNTS.find((entry) => entry.container === path)!; + expect(mount).toBeDefined(); + // tar addresses a root through -C parent + basename; restore uses its full path. + expect(restore).toContain(mount.host); } - }); - - it("keeps the maildir tar on the host, where the bind mount is", () => { - // /var/vmail is a bind mount on container setups (${MAIL_HOST_STATE_DIR}/vmail) and - // a direct path on host setups. - const { payloadConfig: container } = buildMailBackupPayload("example.com", FLAGS, "container"); - expect(container.produceCommand).toContain('sudo -n tar -c -C "$tmp" . -C /var/lib/openship/mail/vmail vmail1'); - expect(container.restoreCommand).toContain('if [ -d "$tmp/vmail1" ]; then cp -a "$tmp/vmail1" /var/lib/openship/mail/vmail/'); - - const { payloadConfig: host } = buildMailBackupPayload("example.com", FLAGS, "host"); - expect(host.produceCommand).toContain('sudo -n tar -c -C "$tmp" . -C /var/vmail vmail1'); - expect(host.restoreCommand).toContain('if [ -d "$tmp/vmail1" ]; then cp -a "$tmp/vmail1" /var/vmail/'); + expect(capture).not.toContain("vmail1"); // Other stores/sieve/quarantine must survive too. + expect(capture).toContain("--clean --if-exists --no-owner"); + expect(capture).not.toContain("--table="); }); }); diff --git a/apps/api/test/modules/mail/mail-backup-lifecycle.test.ts b/apps/api/test/modules/mail/mail-backup-lifecycle.test.ts new file mode 100644 index 000000000..e538ca33d --- /dev/null +++ b/apps/api/test/modules/mail/mail-backup-lifecycle.test.ts @@ -0,0 +1,151 @@ +import "./_setup-env"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +const mocks = vi.hoisted(() => ({ + writeFile: vi.fn(), + readState: vi.fn(), + writeState: vi.fn(), + readFile: vi.fn(), + removeFile: vi.fn(), + target: vi.fn(), + installed: vi.fn(), + pause: vi.fn(), + resume: vi.fn(), + dkim: vi.fn(), + dns: vi.fn(), + rows: vi.fn(), + relay: vi.fn(), + disableRelay: vi.fn(), +})); +vi.mock("@repo/db", async (original) => ({ + ...(await original()), + repos: { mailServer: { get: mocks.target, markInstalled: mocks.installed } }, +})); +vi.mock("@repo/platform/engine/lib/openship-server-store", () => ({ + OPENSHIP_DIR: "/root/.openship", + HOST_STATE_DIR: "/root/.openship", + readOpenshipFile: mocks.readFile, + removeOpenshipFile: mocks.removeFile, + writeOpenshipFile: mocks.writeFile, +})); +vi.mock("@repo/platform/engine/modules/mail/mail-state", async (original) => ({ + ...(await original()), + readState: mocks.readState, + writeState: mocks.writeState, +})); +vi.mock("@repo/platform/engine/modules/mail/mail-engine", async (original) => ({ + ...(await original()), + resolveMailFlavor: async () => "host", + pauseMailDaemons: mocks.pause, +})); +vi.mock("@repo/platform/engine/modules/mail/mail.service", () => ({ + provisionDomainDkim: mocks.dkim, + stepDkimKeys: mocks.dns, +})); +vi.mock("@repo/platform/engine/modules/mail/admin/psql-runner", () => ({ queryRows: mocks.rows })); +vi.mock("@repo/platform/engine/modules/mail/admin/domain-dns.service", () => ({ + buildDomainDnsRecords: (_primary: string, domain: string, dkim: string) => ({ + mx: { value: domain }, + dkim: { value: dkim }, + }), +})); +vi.mock("@repo/platform/engine/modules/mail/admin/outbound-relay.service", () => ({ + configureOutboundRelay: mocks.relay, + disableOutboundRelay: mocks.disableRelay, +})); +import { BareRuntime, LocalExecutor } from "@repo/adapters"; +import { makeFreshState } from "@repo/platform/engine/modules/mail/mail-state"; +import { exportMailState } from "@repo/platform/engine/modules/mail/backup-state"; +import { + prepareMailRestore, + prepareMailBackup, + readMailBackupMetadata, +} from "@repo/platform/engine/modules/mail/backup-lifecycle"; +import { backupScript } from "./backup-script"; + +const metadata = { version: 2 as const, domain: "example.test", messageData: true, keys: true }; +const runtime = () => new BareRuntime({ executor: new LocalExecutor() }); +beforeEach(() => { + vi.clearAllMocks(); + mocks.target.mockResolvedValue({ domain: metadata.domain }); + mocks.readState.mockResolvedValue(makeFreshState("target", metadata.domain)); + mocks.readFile.mockResolvedValue( + JSON.stringify(exportMailState(makeFreshState("source", metadata.domain))), + ); + mocks.pause.mockResolvedValue({ resume: mocks.resume, stopped: true }); + mocks.rows.mockResolvedValue([{ domain: metadata.domain }, { domain: "second.test" }]); + mocks.dkim.mockImplementation(async (_exec, domain) => `public-key:${domain}`); + mocks.dns.mockResolvedValue({ + stepId: 6, + success: true, + message: "DNS", + data: { dnsRecords: { a: { value: "192.0.2.2" } } }, + }); +}); +describe("mail restore uses the destination's normal configuration paths", () => { + it("reconciles DKIM through normal setup, retains target identity and registers completion", async () => { + const operation = await prepareMailRestore(runtime(), "target", "restore-1", metadata); + const script = backupScript(operation.restoreCommand); + // Source amavis DB passwords/paths must not replace the destination config. + expect(script).not.toContain('cp -a "$tmp/keys/amavis'); + await operation.finalize(); + expect(mocks.dkim).toHaveBeenCalledWith(expect.anything(), "second.test", { reload: false }); + const state = mocks.writeState.mock.calls.at(-1)![1]; + expect(state.serverId).toBe("target"); + expect(state.dnsAcknowledged).toBe(false); + expect(state.additionalDomains["second.test"].records.dkim.value).toBe( + "public-key:second.test", + ); + expect(mocks.resume).toHaveBeenCalledOnce(); + expect(mocks.installed).toHaveBeenCalledWith("target", metadata.domain); + expect(mocks.disableRelay).toHaveBeenCalledWith(expect.anything(), { reload: false }); + }); + it("rejects a different target domain before pausing or modifying it", async () => { + mocks.target.mockResolvedValue({ domain: "unrelated.test" }); + await expect(prepareMailRestore(runtime(), "target", "restore-2", metadata)).rejects.toThrow( + "primary domain", + ); + expect(mocks.pause).not.toHaveBeenCalled(); + expect(mocks.writeState).not.toHaveBeenCalled(); + }); + + it("binds restored on-server state to the destination's current registration", async () => { + mocks.readState.mockResolvedValue(makeFreshState("previous-controller-id", metadata.domain)); + const operation = await prepareMailRestore(runtime(), "target", "restore-adopted", metadata); + await operation.finalize(); + expect(mocks.writeState.mock.calls.at(-1)![1].serverId).toBe("target"); + }); + it("leaves mail stopped and does not report completion when configuration recovery fails", async () => { + const operation = await prepareMailRestore(runtime(), "target", "restore-3", metadata); + mocks.dkim.mockRejectedValue(new Error("DKIM key unreadable")); + await expect(operation.finalize()).rejects.toThrow("DKIM key unreadable"); + expect(mocks.resume).not.toHaveBeenCalled(); + expect(mocks.installed).not.toHaveBeenCalled(); + await operation.cleanup(); + expect(mocks.removeFile).toHaveBeenCalledWith(expect.anything(), "mail-backup-restore-3.json"); + }); + it("uses recorded commands for legacy archives and refuses an unknown format", () => { + expect(readMailBackupMetadata({ restoreCommand: "recorded legacy command" })).toBeNull(); + expect(() => readMailBackupMetadata({ mail: { ...metadata, version: 99 } })).toThrow( + "unsupported", + ); + }); +}); + +describe("mail capture preflight", () => { + it("does not pause delivery when a required key/configuration check fails", async () => { + mocks.rows.mockResolvedValue([{ storagebasedirectory: "/var/vmail" }]); + const exec = vi.fn().mockRejectedValue(new Error("primary key missing")); + const host = new BareRuntime({ executor: { exec } as never }); + await expect(prepareMailBackup(host, "capture-1", {})).rejects.toThrow("primary key missing"); + expect(mocks.pause).not.toHaveBeenCalled(); + expect(mocks.removeFile).toHaveBeenCalledWith(expect.anything(), "mail-backup-capture-1.json"); + }); + it("refuses to claim a full backup of a custom mailbox root it would not capture", async () => { + mocks.rows.mockResolvedValue([{ storagebasedirectory: "/srv/external-mail" }]); + await expect(prepareMailBackup(runtime(), "capture-2", {})).rejects.toThrow( + "outside /var/vmail", + ); + expect(mocks.pause).not.toHaveBeenCalled(); + expect(mocks.writeFile).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/test/modules/mail/mail-backup-policy.test.ts b/apps/api/test/modules/mail/mail-backup-policy.test.ts new file mode 100644 index 000000000..0fbcb7b19 --- /dev/null +++ b/apps/api/test/modules/mail/mail-backup-policy.test.ts @@ -0,0 +1,89 @@ +import "./_setup-env"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +const mocks = vi.hoisted(() => ({ + existing: null as unknown, + destinationOrg: "org", + saved: vi.fn(), + flavor: vi.fn(), +})); +vi.mock("@repo/db", async (original) => ({ + ...(await original()), + repos: { + mailServer: { get: async () => ({ domain: "example.test" }) }, + backupDestination: { findById: async () => ({ organizationId: mocks.destinationOrg }) }, + backupPolicy: { + findActiveByMailServer: async () => mocks.existing, + create: async (data: unknown) => { + mocks.saved(data); + return data; + }, + update: async (_id: string, data: unknown) => { + mocks.saved(data); + return data; + }, + }, + }, +})); +vi.mock("../../../src/lib/permission", () => ({ permission: { assert: async () => {} } })); +vi.mock("@repo/platform/engine/lib/resource-access", () => ({ isServerInOrg: async () => true })); +vi.mock("../../../src/lib/request-context", () => ({ + getRequestContext: () => ({ userId: "owner", organizationId: "org" }), +})); +vi.mock("@repo/platform/engine/modules/backups/triggers/cron", () => ({ + syncPolicySchedule: async () => {}, + validateCronExpression: () => ({ valid: true }), +})); +vi.mock("@repo/platform/engine/modules/mail/mail-engine", async (original) => ({ + ...(await original()), + resolveMailFlavor: mocks.flavor, +})); +import { saveMailBackupPolicy } from "../../../src/modules/mail/mail.controller"; +async function save(body: Record) { + const json = vi.fn((body: unknown, status = 200) => ({ body, status })); + const ctx = { + req: { + param: () => "mail-server", + json: async () => ({ destinationId: "destination", ...body }), + }, + json, + }; + return (await saveMailBackupPolicy(ctx as never)) as unknown as { body: unknown; status: number }; +} +beforeEach(() => { + vi.clearAllMocks(); + mocks.existing = null; + mocks.destinationOrg = "org"; + mocks.flavor.mockResolvedValue("host"); +}); +describe("mail backup policy defaults and scope", () => { + it("includes messages and keys in a new policy", async () => { + expect((await save({})).status).toBe(200); + expect(mocks.saved.mock.calls[0]![0].payloadConfig.mail).toEqual({ + messageData: true, + keys: true, + }); + }); + it("retains explicit exclusions on an existing policy", async () => { + mocks.existing = { id: "policy", payloadConfig: { mail: { messageData: false, keys: false } } }; + expect((await save({})).status).toBe(200); + expect(mocks.saved.mock.calls[0]![0].payloadConfig.mail).toEqual({ + messageData: false, + keys: false, + }); + }); + it("can explicitly upgrade an old policy to a full backup", async () => { + mocks.existing = { id: "policy", payloadConfig: { mail: { messageData: false, keys: false } } }; + expect((await save({ messageData: true, keys: true })).status).toBe(200); + expect(mocks.saved.mock.calls[0]![0].payloadConfig.mail.messageData).toBe(true); + }); + it("rejects a foreign destination before touching the mail host", async () => { + mocks.destinationOrg = "foreign"; + expect((await save({})).status).toBe(404); + expect(mocks.flavor).not.toHaveBeenCalled(); + expect(mocks.saved).not.toHaveBeenCalled(); + }); + it("rejects ambiguous non-boolean capture flags", async () => { + expect((await save({ messageData: "false" })).status).toBe(400); + expect(mocks.saved).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/api/test/modules/mail/mail-backup-secrets.test.ts b/apps/api/test/modules/mail/mail-backup-secrets.test.ts index a5914863b..1aa35f35b 100644 --- a/apps/api/test/modules/mail/mail-backup-secrets.test.ts +++ b/apps/api/test/modules/mail/mail-backup-secrets.test.ts @@ -1,99 +1,46 @@ +import "./_setup-env"; import { describe, expect, it } from "vitest"; +import { buildMailBackupPayload } from "@repo/platform/engine/modules/mail/backup-plan"; +import { backupScript } from "./backup-script"; -import { buildMailBackupPayload } from "../../../src/modules/mail/admin/backup-plan"; - -/** - * A `keys: true` backup must not be able to succeed without the keys. - * - * Every secret this archive exists to carry lives on a root-owned path — `/root/.openship` - * is mode 0700, `/var/lib/dkim` holds private keys, and `/var/vmail` is vmail:vmail 0700 — - * while `produceCommand` runs through the BARE ssh executor, unelevated. The reads were - * written as `[ -f … ] && cp … || true`, which on a host Openship logs into as a sudoer - * rather than as root cannot even stat the path: the test fails for lack of privilege, - * `|| true` absorbs it, and the run exits 0 recording `keys: true` on an archive with no - * keys in it. mail-state.json is the only copy of the DKIM state and the encrypted - * mailbox/admin credentials, so the artifact is a disaster-recovery archive that isn't one. - * - * These assert the two halves of the fix — read as root, and don't swallow the failure — - * against the emitted script, which is the only observable: the shell runs on a mail host. - */ - -const SECRET_READS = [ - { what: "DKIM private keys", path: "/var/lib/dkim" }, - { what: "mail-state.json (DKIM state + encrypted credentials)", path: "/root/.openship/mail-state.json" }, -]; - -describe("mail backup cannot silently omit the secrets it was asked for", () => { - const produce = (keys: boolean) => - buildMailBackupPayload("example.com", { messageData: false, keys }).payloadConfig - .produceCommand; - - it.each(SECRET_READS)("reads $what with sudo, not as the login user", ({ path }) => { - const script = produce(true); - const line = script.split("\n").find((l) => l.includes(path)); - - expect(line, `no line reads ${path}`).toBeDefined(); - expect(line).toContain("sudo -n"); - }); - - it.each(SECRET_READS)("does not absorb a failed read of $what", ({ path }) => { - // `|| true` on these is the whole defect: it converts "could not read the secret" into - // "the host does not have one", and the two are not the same answer. - const line = produce(true) - .split("\n") - .find((l) => l.includes(path) && l.includes("cp ")); - - expect(line, `no copy of ${path}`).toBeDefined(); - expect(line).not.toContain("|| true"); - }); - - it("distinguishes an absent secret from an unreadable one", () => { - const script = produce(true); - // Absence must still skip — a box with no amavis config is a gap in the archive, not a - // failed backup. The `test` is what expresses that, so it has to be there AND be - // privileged; a `[ -f ]` under the login user answers "absent" for both cases. - expect(script).toContain("sudo -n test -f /root/.openship/mail-state.json"); - expect(script).toContain("sudo -n test -d /var/lib/dkim"); - expect(script).not.toMatch(/^\s*\[ -[fd] \/(root|var\/lib\/dkim)/m); - }); - - it("tars as root, so it can read back what it staged as root", () => { - // Not cosmetic: `sudo -n cp -a` stages root-owned 0600 files, and an unprivileged tar - // would fail to read them. Same reason /var/vmail needs it. - // - // Asserted as "EVERY `tar -c` is privileged" rather than "the tar line starts with - // sudo": the pipeline is now built by the shared `safePipelineFragment`, so the - // invocation sits inside an `if` and no longer begins the line. The property is the - // privilege, not the column it appears in — and the old prefix match would also have - // gone green if a second, unprivileged tar were added below it. - for (const messageData of [false, true]) { - const script = buildMailBackupPayload("example.com", { messageData, keys: true }) - .payloadConfig.produceCommand; - const invocations = [...script.matchAll(/(\S*\s*\S*\s*)?tar -c/g)].map((m) => m[0]); - expect(invocations.length, `no tar -c (messageData: ${messageData})`).toBeGreaterThan(0); - for (const found of invocations) { - expect(found, `unprivileged tar -c (messageData: ${messageData})`).toContain("sudo -n"); - } - } - }); - - it("no longer lets zstd mask tar's exit status", () => { - // The masking this file's comments described for as long as they existed: a shell - // pipeline reports its LAST command's status, so an unprivileged tar that dropped every - // root-owned secret still exited 0 behind zstd. This was the last pipeline in the - // backup module still carrying it. - const script = buildMailBackupPayload("example.com", { messageData: true, keys: true }) +describe("mail backup secret handling", () => { + it("runs the entire capture as root, with private staging and mandatory requested secrets", () => { + const command = buildMailBackupPayload("example.test", { messageData: true, keys: true }) .payloadConfig.produceCommand; + const script = backupScript(command); + expect(script).toContain("umask 077"); + expect(script).toContain("test -d '/var/lib/dkim'"); + expect(script).toContain("test -s '/root/.openship/mail-state.json'"); + expect(script).toContain("cp -a '/var/lib/dkim'"); + expect(script).toContain("cp -a '/root/.openship/mail-state.json'"); + expect(script).not.toContain("|| true"); + expect(script).toContain("trap 'rm -rf \"$tmp\"' EXIT"); + }); + it("uses the per-run portable file, without embedding its contents in commands", () => { + const command = buildMailBackupPayload( + "example.test", + { messageData: false, keys: true }, + "host", + { statePath: "/root/.openship/mail-backup-run.json" }, + ).payloadConfig.produceCommand; + expect(backupScript(command)).toContain("cp -a '/root/.openship/mail-backup-run.json'"); + expect(backupScript(command)).not.toContain("cp -a '/root/.openship/mail-state.json'"); + }); + it("propagates both tar and compressor failures", () => { + const script = backupScript( + buildMailBackupPayload("example.test", { messageData: true, keys: true }).payloadConfig + .produceCommand, + ); expect(script).toContain("openship: tar (staging the mail archive) failed with status"); expect(script).toContain("openship: the compressor 'zstd' failed with status"); - // And the raw masking shape is gone. - expect(script).not.toMatch(/tar -c[^\n]*\| zstd -c -3\s*$/m); }); - - it("stages nothing privileged when keys were not requested", () => { - // The contrast case: without it, a fix that sudo'd unconditionally would pass the - // assertions above while changing what a keys:false backup touches. - const script = produce(false); - for (const { path } of SECRET_READS) expect(script).not.toContain(path); + it("does not read private configuration when explicitly excluded", () => { + const script = backupScript( + buildMailBackupPayload("example.test", { messageData: false, keys: false }).payloadConfig + .produceCommand, + ); + expect(script).not.toContain("/var/lib/dkim"); + expect(script).not.toContain("/root/.openship/mail-state.json"); + expect(script).not.toContain("-C '/var' vmail"); }); }); diff --git a/apps/api/test/modules/mail/mail-backup-state.test.ts b/apps/api/test/modules/mail/mail-backup-state.test.ts new file mode 100644 index 000000000..ffdc66488 --- /dev/null +++ b/apps/api/test/modules/mail/mail-backup-state.test.ts @@ -0,0 +1,97 @@ +import "./_setup-env"; +import { describe, expect, it } from "vitest"; +import { createEncryption } from "@repo/db/encryption"; +import { makeFreshState } from "@repo/platform/engine/modules/mail/mail-state"; +import { exportMailState, importMailState } from "@repo/platform/engine/modules/mail/backup-state"; + +describe("mail configuration transferred between instances", () => { + it("re-encrypts mailbox and relay credentials while retaining the destination's database credentials", () => { + const sourceCipher = createEncryption("source-instance-key"); + const targetCipher = createEncryption("different-target-instance-key"); + const source = makeFreshState("source", "example.test"); + source.secrets = { + VMAIL_DB_BIND_PASSWD: "source-db", + DOMAIN_ADMIN_PASSWD_PLAIN: "source-postmaster", + }; + source.dnsAcknowledged = true; + source.ptrAcknowledged = true; + source.platformMailbox = { + email: "openship@example.test", + password: sourceCipher.encrypt("mail-password"), + smtpHost: "mail.example.test", + smtpPort: 587, + secure: false, + updatedAt: "2026-01-01", + }; + source.testMailboxes = { "example.test": { ...source.platformMailbox } }; + source.outboundRelay = { + enabled: true, + provider: "custom", + host: "relay.example.test", + port: 587, + username: "sender", + passwordEncrypted: sourceCipher.encrypt("relay-password"), + updatedAt: "2026-01-01", + }; + const target = makeFreshState("destination", "example.test"); + target.secrets = { VMAIL_DB_BIND_PASSWD: "target-db", PGSQL_ROOT_PASSWD: "target-root" }; + target.engineImage = "target-engine"; + const portable = exportMailState(source, sourceCipher); + const restored = importMailState(portable, target, targetCipher); + + expect(portable.state.platformMailbox?.password).toBe("mail-password"); + expect(JSON.stringify(portable)).not.toContain("source-db"); + expect(targetCipher.decrypt(restored.platformMailbox!.password)).toBe("mail-password"); + expect(targetCipher.decrypt(restored.testMailboxes!["example.test"]!.password)).toBe( + "mail-password", + ); + expect(targetCipher.decrypt(restored.outboundRelay!.passwordEncrypted)).toBe("relay-password"); + expect(() => sourceCipher.decrypt(restored.platformMailbox!.password)).toThrow(); + expect(restored.secrets).toEqual({ + VMAIL_DB_BIND_PASSWD: "target-db", + PGSQL_ROOT_PASSWD: "target-root", + DOMAIN_ADMIN_PASSWD_PLAIN: "source-postmaster", + }); + expect(restored.serverId).toBe("destination"); + expect(restored.engineImage).toBe("target-engine"); + expect(restored.dnsAcknowledged).toBe(false); + expect(restored.ptrAcknowledged).toBe(false); + expect(sourceCipher.decrypt(source.platformMailbox!.password)).toBe("mail-password"); + sourceCipher.close(); + targetCipher.close(); + }); + + it("refuses an undecryptable credential instead of making an unrestorable archive", () => { + const source = makeFreshState("source", "example.test"); + const wrong = createEncryption("wrong-key"); + const original = createEncryption("original-key"); + source.platformMailbox = { + email: "openship@example.test", + password: original.encrypt("secret"), + smtpHost: "mail.example.test", + smtpPort: 587, + secure: false, + updatedAt: "2026-01-01", + }; + expect(() => exportMailState(source, wrong)).toThrow(); + original.close(); + wrong.close(); + }); + + it("refuses restoring a different primary domain", () => { + const source = makeFreshState("source", "example.test"); + const target = makeFreshState("target", "unrelated.test"); + expect(() => importMailState(exportMailState(source), target)).toThrow("primary domain"); + }); + + it("does not retain a destination password for a postmaster account replaced by the backup", () => { + const source = makeFreshState("source", "example.test"); + const target = makeFreshState("target", "example.test"); + target.secrets = { + VMAIL_DB_BIND_PASSWD: "target-db", + DOMAIN_ADMIN_PASSWD_PLAIN: "old-account-password", + }; + const restored = importMailState(exportMailState(source), target); + expect(restored.secrets).toEqual({ VMAIL_DB_BIND_PASSWD: "target-db" }); + }); +}); diff --git a/apps/api/test/modules/mail/mail-dkim-domain-validation.test.ts b/apps/api/test/modules/mail/mail-dkim-domain-validation.test.ts new file mode 100644 index 000000000..9a358f06f --- /dev/null +++ b/apps/api/test/modules/mail/mail-dkim-domain-validation.test.ts @@ -0,0 +1,12 @@ +import "./_setup-env"; +import { expect, it, vi } from "vitest"; +import { provisionDomainDkim } from "@repo/platform/engine/modules/mail/mail.service"; + +it.each(["x;touch /tmp/executed.test", "../example.test", "x'example.test", "x$(id).test"])( + "refuses unsafe restored domain %s before filesystem or shell access", + async (domain) => { + const exec = vi.fn(); + await expect(provisionDomainDkim({ exec } as never, domain)).rejects.toThrow("Invalid domain"); + expect(exec).not.toHaveBeenCalled(); + }, +); diff --git a/apps/api/test/modules/mail/mail-restore-amavis-path.test.ts b/apps/api/test/modules/mail/mail-restore-amavis-path.test.ts deleted file mode 100644 index e7fb719e1..000000000 --- a/apps/api/test/modules/mail/mail-restore-amavis-path.test.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { execFileSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; - -import { describe, expect, it } from "vitest"; - -import { buildMailBackupPayload } from "../../../src/modules/mail/admin/backup-plan"; - -/** - * A restored amavis config has to land where THIS box's amavis reads it. - * - * Restoring is normally cross-machine — that is the point of a backup — so source and - * target routinely disagree about where amavis keeps its editable config: Debian has an - * include dir (`/etc/amavis/conf.d/50-user`), the RHEL family a monolithic - * `/etc/amavisd.conf`. The archive records the path it came from, and honouring that - * verbatim writes a file the target's amavis never opens — `cp -a … || true` succeeds, the - * restore reports success, and the config silently does nothing. - * - * These run the REAL emitted shell rather than asserting on its text, with the candidate - * paths rebased under a temp root: the rule is a chain of `test`s, and a string assertion - * cannot tell a correct chain from a plausible one. It also pins the direction that is easy - * to get wrong — on Debian the destination FILE legitimately does not exist yet (50-user is - * ours to create), so a probe keyed on the file instead of the include DIRECTORY reads a - * Debian box as "no amavis here" and falls through to the RHEL path. - */ - -const DEBIAN = "/etc/amavis/conf.d/50-user"; -const DEBIAN_DIR = "/etc/amavis/conf.d"; -const RHEL = "/etc/amavisd.conf"; -const CONTENT = "$final_spam_destiny = D_DISCARD;\n"; - -/** - * The amavis placement block of the real restore script, rebased under `root`. - * - * Sliced between two stable markers rather than by line number, so an edit elsewhere in the - * restore cannot silently shrink what is under test — a missing marker fails loudly here - * instead of passing vacuously. - */ -function placementScript(root: string, tmp: string): string { - const script = buildMailBackupPayload("example.com", { - messageData: false, - keys: true, - }).payloadConfig.restoreCommand; - - const start = script.indexOf('dest=""'); - const end = script.indexOf('if [ -d "$tmp/vmail1" ]'); - expect(start, "restore script no longer resolves an amavis `dest`").toBeGreaterThan(-1); - expect(end, "restore script has no maildir step to slice against").toBeGreaterThan(start); - - const block = script.slice(start, end); - expect(block).toContain("cp -a"); - // Every absolute path in this block is an /etc one, and nothing else in it is. - return `set -e\ntmp=${JSON.stringify(tmp)}\n${block.replaceAll("/etc/", `${root}/etc/`)}`; -} - -/** Stage an archive + a target layout, run the placement, report where the file went. */ -function place(opts: { - /** Pre-existing amavis config on the TARGET, or none at all. */ - target: "debian" | "rhel" | "none"; - /** Path the ARCHIVE says it came from; null for a legacy archive that records none. */ - archiveFrom: string | null; - legacyName?: boolean; -}): { at: (p: string) => boolean; read: (p: string) => string } { - const root = mkdtempSync(join(tmpdir(), "amavis-target-")); - const tmp = mkdtempSync(join(tmpdir(), "amavis-archive-")); - - mkdirSync(join(root, "etc"), { recursive: true }); - if (opts.target === "debian") mkdirSync(join(root, DEBIAN_DIR), { recursive: true }); - if (opts.target === "rhel") writeFileSync(join(root, RHEL), "# the target's own\n"); - - mkdirSync(join(tmp, "keys"), { recursive: true }); - writeFileSync(join(tmp, opts.legacyName ? "keys/amavis-50-user" : "keys/amavis-conf"), CONTENT); - if (opts.archiveFrom) { - // A real archive names a path in the SOURCE box's namespace, and the candidate list the - // script matches it against is rebased — so a known path has to be rebased too, or the - // fallback could never match its own list and every case would test the same branch. - // An unknown path is left alone: not matching is the whole point of it. - const recorded = opts.archiveFrom.startsWith("/etc/") - ? join(root, opts.archiveFrom) - : opts.archiveFrom; - writeFileSync(join(tmp, "keys/amavis-conf.path"), recorded); - } - - execFileSync("sh", ["-c", placementScript(root, tmp)], { stdio: "pipe" }); - - return { - at: (p) => existsSync(join(root, p)), - read: (p) => readFileSync(join(root, p), "utf8"), - }; -} - -describe("a restored amavis config lands where the TARGET reads it", () => { - it("writes the RHEL path on a RHEL target, even from a Debian archive", () => { - const r = place({ target: "rhel", archiveFrom: DEBIAN }); - - expect(r.read(RHEL)).toBe(CONTENT); - // The source's path must not be recreated: a file there is one this box's amavis will - // never open, and its presence is what made the failure look like a success. - expect(r.at(DEBIAN)).toBe(false); - }); - - it("writes the include file on a Debian target, even from a RHEL archive", () => { - // The direction a file-existence probe gets wrong: 50-user is ours to CREATE, so the - // only evidence a Debian box offers is the include DIRECTORY. - const r = place({ target: "debian", archiveFrom: RHEL }); - - expect(r.read(DEBIAN)).toBe(CONTENT); - expect(r.at(RHEL)).toBe(false); - }); - - it("falls back to the archive's recorded path when the target has no amavis", () => { - // Nothing to probe, so the source is the best evidence left. - const r = place({ target: "none", archiveFrom: RHEL }); - - expect(r.read(RHEL)).toBe(CONTENT); - expect(r.at(DEBIAN)).toBe(false); - }); - - it("never writes a path the archive invented", () => { - // This shell runs as root on the target, so the recorded path is matched against the - // known candidates rather than used — otherwise `cp -a` is an arbitrary root write. - // With no amavis to probe and no usable recorded path, nothing is placed at all: the - // default's include dir doesn't exist on such a box and `|| true` absorbs the miss. - const r = place({ target: "none", archiveFrom: "/tmp/openship-amavis-escape/authorized_keys" }); - - expect(existsSync("/tmp/openship-amavis-escape/authorized_keys")).toBe(false); - expect(r.at(RHEL)).toBe(false); - expect(r.at(DEBIAN)).toBe(false); - }); - - it("places a legacy archive by the target's layout, since it records no source", () => { - // Pre-2026-08 archives carried the Debian path under a fixed name with no `.path` - // sidecar, so the target probe is the only thing that can place them at all. - const r = place({ target: "rhel", archiveFrom: null, legacyName: true }); - - expect(r.read(RHEL)).toBe(CONTENT); - expect(r.at(DEBIAN)).toBe(false); - }); -}); diff --git a/apps/dashboard/src/app/(dashboard)/emails/_components/admin/backup-tab.tsx b/apps/dashboard/src/app/(dashboard)/emails/_components/admin/backup-tab.tsx index 85e95c71a..a5587e9f8 100644 --- a/apps/dashboard/src/app/(dashboard)/emails/_components/admin/backup-tab.tsx +++ b/apps/dashboard/src/app/(dashboard)/emails/_components/admin/backup-tab.tsx @@ -89,7 +89,7 @@ export function BackupTab({ serverId, domain }: { serverId: string; domain: stri // Form state const [destinationId, setDestinationId] = useState(""); - const [messageData, setMessageData] = useState(false); + const [messageData, setMessageData] = useState(true); const [keys, setKeys] = useState(true); const [schedule, setSchedule] = useState(DEFAULT_SCHEDULE_PARTS); // Kept as strings: an empty box is "no limit", which a number state can't hold diff --git a/apps/dashboard/src/app/(dashboard)/emails/_components/admin/mail-restore-modal.tsx b/apps/dashboard/src/app/(dashboard)/emails/_components/admin/mail-restore-modal.tsx index 78b4e7be8..f1c55e36d 100644 --- a/apps/dashboard/src/app/(dashboard)/emails/_components/admin/mail-restore-modal.tsx +++ b/apps/dashboard/src/app/(dashboard)/emails/_components/admin/mail-restore-modal.tsx @@ -12,7 +12,7 @@ import { Icon as UiIcon } from "@repo/ui/icons"; * mode "in_place" → restore the backup onto THIS mail server. * mode "to_fork" → migrate: restore onto a DIFFERENT mail server. * - * Both are destructive on the target: its accounts are TRUNCATED before the + * Both are destructive on the target: the database and mail stores are replaced as the * backup's data loads. */ @@ -92,14 +92,14 @@ function MailRestoreContent({ .then(({ servers }) => setTargets( servers - .filter((s) => s.id !== sourceServerId && s.completed) + .filter((s) => s.id !== sourceServerId && s.completed && s.domain === domain) .map((s) => ({ id: s.id, name: s.name, host: s.host, domain: s.domain, completed: s.completed })), ), ) .catch((diagnosticFailure) => { observeCaughtError(diagnosticFailure, "dashboard/app/(dashboard)/emails/_components/admin/mail-restore-modal"); }); - }, [mode, sourceServerId]); + }, [mode, sourceServerId, domain]); const canStart = confirmText.trim() === domain && (mode === "in_place" || !!targetId); diff --git a/apps/dashboard/src/i18n/locales/ar/emailsAdmin.json b/apps/dashboard/src/i18n/locales/ar/emailsAdmin.json index e1ffca6dd..69029926f 100644 --- a/apps/dashboard/src/i18n/locales/ar/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/ar/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "الحسابات والنطاقات والأسماء المستعارة", "accountsDesc": "صناديق البريد (مع تجزئات كلمات المرور) والنطاقات والأسماء المستعارة — قاعدة بيانات vmail.", "messageTitle": "بيانات رسائل صندوق البريد", - "messageDesc": "كل بريد مخزّن (مجلدات maildir في /var/vmail). قد تكون كبيرة.", + "messageDesc": "الرسائل والمجلدات وقائمة الانتظار والحجر. يتوقف التسليم وIMAP مؤقتًا أثناء النسخ.", "keysTitle": "مفاتيح وأسرار DKIM", "keysDesc": "مفاتيح DKIM + الإعدادات + mail-state.json. يبقي DKIM صالحًا عند الاستعادة — لكن الأرشيف عندئذٍ يحتوي على مفاتيح خاصة وأسرار.", "keysWarning": "ستحتوي النسخة الاحتياطية على تجزئات كلمات المرور ومفاتيح DKIM الخاصة وأسرار بنص صريح. أبقِ الوجهة خاصة.", diff --git a/apps/dashboard/src/i18n/locales/de/emailsAdmin.json b/apps/dashboard/src/i18n/locales/de/emailsAdmin.json index 560653a0f..51832d480 100644 --- a/apps/dashboard/src/i18n/locales/de/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/de/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "Konten, Domains & Aliase", "accountsDesc": "Postfächer (mit Passwort-Hashes), Domains und Aliase – die vmail-Datenbank.", "messageTitle": "Postfach-Nachrichtendaten", - "messageDesc": "Jede gespeicherte E-Mail (die /var/vmail-Maildirs). Kann groß sein.", + "messageDesc": "Nachrichten, Ordner, Warteschlange und Quarantäne. Zustellung und IMAP pausieren während des Kopierens.", "keysTitle": "DKIM-Schlüssel & Geheimnisse", "keysDesc": "DKIM-Schlüssel + Konfiguration + mail-state.json. Hält DKIM bei der Wiederherstellung gültig – aber das Archiv enthält dann private Schlüssel und Geheimnisse.", "keysWarning": "Die Sicherung enthält Passwort-Hashes, private DKIM-Schlüssel und Geheimnisse im Klartext. Halte das Ziel privat.", diff --git a/apps/dashboard/src/i18n/locales/en/emailsAdmin.json b/apps/dashboard/src/i18n/locales/en/emailsAdmin.json index 759da566d..787532eb7 100644 --- a/apps/dashboard/src/i18n/locales/en/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/en/emailsAdmin.json @@ -566,7 +566,7 @@ "accountsTitle": "Accounts, domains & aliases", "accountsDesc": "Mailboxes (with password hashes), domains, and aliases — the vmail database.", "messageTitle": "Mailbox message data", - "messageDesc": "Every stored email (the /var/vmail maildirs). Can be large.", + "messageDesc": "Messages, folders, queue and quarantine. Delivery and IMAP pause while copying.", "keysTitle": "DKIM keys & secrets", "keysDesc": "DKIM keys + config + mail-state.json. Keeps DKIM valid on restore — but the archive then holds private keys and secrets.", "keysWarning": "The backup will contain password hashes, DKIM private keys, and plaintext secrets. Keep the destination private.", diff --git a/apps/dashboard/src/i18n/locales/es/emailsAdmin.json b/apps/dashboard/src/i18n/locales/es/emailsAdmin.json index a23a250ce..5b377181a 100644 --- a/apps/dashboard/src/i18n/locales/es/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/es/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "Cuentas, dominios y alias", "accountsDesc": "Buzones (con hashes de contraseñas), dominios y alias: la base de datos vmail.", "messageTitle": "Datos de los mensajes del buzón", - "messageDesc": "Cada correo almacenado (los maildirs de /var/vmail). Puede ser grande.", + "messageDesc": "Mensajes, carpetas, cola y cuarentena. La entrega e IMAP se pausan durante la copia.", "keysTitle": "Claves DKIM y secretos", "keysDesc": "Claves DKIM + configuración + mail-state.json. Mantiene DKIM válido al restaurar, pero el archivo contendrá entonces claves privadas y secretos.", "keysWarning": "La copia de seguridad contendrá hashes de contraseñas, claves privadas DKIM y secretos en texto plano. Mantén el destino privado.", diff --git a/apps/dashboard/src/i18n/locales/fr/emailsAdmin.json b/apps/dashboard/src/i18n/locales/fr/emailsAdmin.json index f6a906094..39a78b344 100644 --- a/apps/dashboard/src/i18n/locales/fr/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/fr/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "Comptes, domaines et alias", "accountsDesc": "Boîtes aux lettres (avec les hachages de mots de passe), domaines et alias — la base de données vmail.", "messageTitle": "Données des messages des boîtes aux lettres", - "messageDesc": "Chaque e-mail stocké (les maildirs /var/vmail). Peut être volumineux.", + "messageDesc": "Messages, dossiers, file d’attente et quarantaine. La livraison et IMAP sont suspendus pendant la copie.", "keysTitle": "Clés DKIM et secrets", "keysDesc": "Clés DKIM + configuration + mail-state.json. Maintient le DKIM valide à la restauration — mais l'archive contient alors des clés privées et des secrets.", "keysWarning": "La sauvegarde contiendra des hachages de mots de passe, des clés privées DKIM et des secrets en clair. Gardez la destination privée.", diff --git a/apps/dashboard/src/i18n/locales/ja/emailsAdmin.json b/apps/dashboard/src/i18n/locales/ja/emailsAdmin.json index 0ff723482..72707f483 100644 --- a/apps/dashboard/src/i18n/locales/ja/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/ja/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "アカウント、ドメイン、エイリアス", "accountsDesc": "メールボックス (パスワードハッシュを含む)、ドメイン、エイリアス — vmail データベースです。", "messageTitle": "メールボックスのメッセージデータ", - "messageDesc": "保存されているすべてのメール (/var/vmail の maildir)。サイズが大きくなることがあります。", + "messageDesc": "メール、フォルダー、キュー、隔離データ。コピー中は配信とIMAPが一時停止します。", "keysTitle": "DKIM 鍵とシークレット", "keysDesc": "DKIM 鍵 + 設定 + mail-state.json。リストア時に DKIM を有効なまま保ちます - ただしアーカイブには秘密鍵とシークレットが含まれることになります。", "keysWarning": "バックアップにはパスワードハッシュ、DKIM 秘密鍵、平文のシークレットが含まれます。保存先は非公開に保ってください。", diff --git a/apps/dashboard/src/i18n/locales/pt/emailsAdmin.json b/apps/dashboard/src/i18n/locales/pt/emailsAdmin.json index 046ff98c2..33fc1a349 100644 --- a/apps/dashboard/src/i18n/locales/pt/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/pt/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "Contas, domínios e aliases", "accountsDesc": "Caixas de correio (com hashes de senha), domínios e aliases — o banco de dados vmail.", "messageTitle": "Dados de mensagens das caixas de correio", - "messageDesc": "Cada e-mail armazenado (os maildirs em /var/vmail). Pode ser grande.", + "messageDesc": "Mensagens, pastas, fila e quarentena. A entrega e o IMAP ficam pausados durante a cópia.", "keysTitle": "Chaves DKIM e segredos", "keysDesc": "Chaves DKIM + config + mail-state.json. Mantém o DKIM válido na restauração — mas o arquivo então contém chaves privadas e segredos.", "keysWarning": "O backup conterá hashes de senha, chaves privadas DKIM e segredos em texto puro. Mantenha o destino privado.", diff --git a/apps/dashboard/src/i18n/locales/ru/emailsAdmin.json b/apps/dashboard/src/i18n/locales/ru/emailsAdmin.json index ae5a20e7c..391270a43 100644 --- a/apps/dashboard/src/i18n/locales/ru/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/ru/emailsAdmin.json @@ -566,7 +566,7 @@ "accountsTitle": "Аккаунты, домены и алиасы", "accountsDesc": "Почтовые ящики (с хешами паролей), домены и алиасы — база данных vmail.", "messageTitle": "Сообщения почтовых ящиков", - "messageDesc": "Все сохранённые письма (директории /var/vmail). Может занимать много места.", + "messageDesc": "Письма, папки, очередь и карантин. Доставка и IMAP приостанавливаются на время копирования.", "keysTitle": "Ключи DKIM и секреты", "keysDesc": "Ключи DKIM + конфигурация + mail-state.json. Сохраняет валидность DKIM при восстановлении, но архив будет содержать приватные ключи и секреты.", "keysWarning": "Резервная копия будет содержать хеши паролей, закрытые ключи DKIM и секреты в открытом виде. Хранилище должно быть защищено.", diff --git a/apps/dashboard/src/i18n/locales/tr/emailsAdmin.json b/apps/dashboard/src/i18n/locales/tr/emailsAdmin.json index 62437da6f..92a74e097 100644 --- a/apps/dashboard/src/i18n/locales/tr/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/tr/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "Hesaplar, alan adları ve takma adlar", "accountsDesc": "Posta kutuları (parola özetleriyle), alan adları ve takma adlar — vmail veritabanı.", "messageTitle": "Posta kutusu mesaj verileri", - "messageDesc": "Saklanan tüm e-postalar (/var/vmail maildir klasörleri). Büyük boyutlu olabilir.", + "messageDesc": "İletiler, klasörler, kuyruk ve karantina. Kopyalama sırasında teslimat ve IMAP duraklatılır.", "keysTitle": "DKIM anahtarları ve gizli veriler", "keysDesc": "DKIM anahtarları + yapılandırma + mail-state.json. Geri yüklemede DKIM'i geçerli tutar — ancak arşiv özel anahtarları ve gizli verileri barındırır.", "keysWarning": "Yedek; şifre özetlerini, DKIM özel anahtarlarını ve düz metin hâlindeki gizli verileri içerir. Yedekleme hedefine erişimi kısıtlı tutun.", diff --git a/apps/dashboard/src/i18n/locales/zh/emailsAdmin.json b/apps/dashboard/src/i18n/locales/zh/emailsAdmin.json index 4235653a4..d39c72d9b 100644 --- a/apps/dashboard/src/i18n/locales/zh/emailsAdmin.json +++ b/apps/dashboard/src/i18n/locales/zh/emailsAdmin.json @@ -519,7 +519,7 @@ "accountsTitle": "账户、域名与别名", "accountsDesc": "邮箱(含密码哈希)、域名和别名——即 vmail 数据库。", "messageTitle": "邮箱邮件数据", - "messageDesc": "所有已存储的电子邮件(/var/vmail maildir 目录)。可能会很大。", + "messageDesc": "邮件、文件夹、队列和隔离数据。复制期间暂停投递和 IMAP。", "keysTitle": "DKIM 密钥与机密信息", "keysDesc": "DKIM 密钥 + 配置 + mail-state.json。还原后可保持 DKIM 有效——但此后归档文件中将包含私钥和机密信息。", "keysWarning": "备份将包含密码哈希、DKIM 私钥和明文机密信息。请对目标位置保密。", diff --git a/apps/web/content/docs/dashboard/emails.mdx b/apps/web/content/docs/dashboard/emails.mdx index 6a2895f68..f91902a99 100644 --- a/apps/web/content/docs/dashboard/emails.mdx +++ b/apps/web/content/docs/dashboard/emails.mdx @@ -285,7 +285,9 @@ usually means this outer layer is dropping the connection. Backups run through Openship's general backup system, so the mail server is just another backup source. Choose **what to back up**: accounts, domains & aliases (always included), **Mailbox message data** (the -maildirs — can be large), and **DKIM keys & secrets**. +maildirs, folders, queue and quarantine — can be large), and **DKIM keys & secrets**. +New policies include messages and keys by default; existing exclusions are retained. +Full backups pause delivery and IMAP while copying, then resume the daemons that were running. Including keys means the archive holds password hashes, DKIM private keys, and plaintext secrets. Keep the @@ -295,6 +297,11 @@ destination private. Pick a **Destination** (managed on the [Backups](/docs/dashboard/backups) page) and an **Automatic backups** schedule (Manual only / Daily / Weekly), then **Save settings** or **Back up now**. **Recent backups** lists each run; successful runs offer **Restore** (in place) and **Migrate** (onto another server). +Prepare the target mail server with the same primary domain and a compatible mail/PostgreSQL version +first. Restore includes all mail tables, additional domains, managed mailbox/relay credentials and +DKIM keys; the destination retains its database-role credentials and TLS configuration. Review the +new DNS/PTR records before cutover. Failed partial restores stay stopped for review and retry. +Settings → Instance export does not include mailbox messages by default. ### Advanced diff --git a/docs/backups.md b/docs/backups.md index c34d0e5b2..7f56b80d0 100644 --- a/docs/backups.md +++ b/docs/backups.md @@ -46,6 +46,54 @@ completion or disconnect removes the subscription and timer. The dashboard ticks elapsed time independently, updates history in place, and shows the current phase and transferred bytes without inventing a percentage for an unknown capture size. +## Mail-server backups and migration + +Use **Emails → server → Backup** for mail data. It uses the same destinations, +workers, integrity checks, retention and restore confirmation as other backups. +New policies include message data and keys by default. Existing policies retain +explicit exclusions; review them before relying on a backup for disaster recovery. + +The archive contains the complete `vmail` database schema and rows, including +aliases, forwards, mailing lists, BCC/relay rules, quotas, and `mail_app` settings. +The mail stack's `amavisd`, `iredapd` and `fail2ban` PostgreSQL databases are included +when present. It never dumps unrelated databases or cluster-wide role passwords. +With message data enabled it also captures the whole `/var/vmail` tree, Postfix's +queued messages, and Amavis quarantine/filter data. With keys enabled it includes +DKIM private keys and portable mail-server state, including managed mailbox and +outbound-relay credentials. These are secrets: protect access to the destination. +Mailbox stores must use the managed `/var/vmail` root, including bind mounts or a +symlink at that root. A different configured root is refused before pausing mail, +rather than reporting a full backup that omitted it. + +Full captures pause Postfix, Dovecot, Amavis and iRedAPD while copying the snapshot. +Allow a maintenance window proportional to message volume and destination speed. +Previously running daemons resume when capture succeeds or fails; daemons already +stopped are not started. Bare installs use the existing local command transport; +remote servers use the same flow over SSH. Local workload execution remains +forbidden on a SaaS controller. + +**Restore** replaces the current server's data. **Migrate** restores to another +registered, installed mail server with the **same primary domain**. Prepare that +server first with a compatible PostgreSQL/mail version and the mail database roles; +restoring does not install an operating system or daemons. The destination's engine, +TLS installation and database-role credentials stay authoritative. Restored mailbox +and relay credentials are encrypted for the destination controller. DKIM/DNS/relay +setup uses the same operations as normal mail administration. The target registration +is updated, while DNS and PTR acknowledgement is cleared for review before cutover. +Moving mail does not automatically repoint public DNS or the instance SMTP settings. + +Each PostgreSQL database restores in a transaction. The complete operation spans +multiple databases and filesystem stores, so it is not one atomic transaction. +If a write or configuration recovery fails, the restore reports failure and leaves +mail stopped for review/retry. Cancellation also stops the active restore command. +Private staging files are removed after normal success/failure cleanup. + +Legacy archives keep their recorded restore command and their original contents; +an old archive cannot recover messages, tables or state it never captured. Take a +new full backup after updating. Settings → Instance export remains a control-plane +export and does **not** include mailbox payloads by default; use the mail archive +for mail recovery rather than treating an instance JSON export as a message backup. + ## Global backup pages `/backups` shows recent runs across destinations, with service/project names, diff --git a/docs/ship-sdk-relocations.json b/docs/ship-sdk-relocations.json index a8f3a3db5..cb718a670 100644 --- a/docs/ship-sdk-relocations.json +++ b/docs/ship-sdk-relocations.json @@ -1501,6 +1501,12 @@ "to": "packages/platform/src/engine/modules/jobs/job.types.ts", "hadBaseline": true }, + { + "from": "apps/api/src/modules/mail/admin/backup-plan.ts", + "to": "packages/platform/src/engine/modules/mail/backup-plan.ts", + "hadBaseline": true, + "baselineCommit": "367afdec910ee096bc3427ff52a881aa44025bde" + }, { "from": "apps/api/src/modules/mail/admin/domain-dns.service.ts", "to": "packages/platform/src/engine/modules/mail/admin/domain-dns.service.ts", diff --git a/packages/adapters/src/backup/producers/custom-command.ts b/packages/adapters/src/backup/producers/custom-command.ts index 54fb4c1e0..afe508b4a 100644 --- a/packages/adapters/src/backup/producers/custom-command.ts +++ b/packages/adapters/src/backup/producers/custom-command.ts @@ -73,7 +73,7 @@ class CustomCommandProducerImpl implements BackupProducer { service: ServiceHandle, executor: BackupExecutor, artifact: ArtifactRef, - _opts: RestoreOpts, + opts: RestoreOpts, ): Promise { const restoreCommand = (artifact.metadata.restoreCommand as string | null) ?? null; @@ -104,7 +104,7 @@ class CustomCommandProducerImpl implements BackupProducer { ["sh", "-c", restoreCommand], body, // Ceiling from the catalog, so the number is not a per-producer literal. - { timeoutMs: payloadSpec("custom_command").restoreTimeoutMs }, + { timeoutMs: payloadSpec("custom_command").restoreTimeoutMs, signal: opts.signal }, ); if (exit.code !== 0) { throw new Error( diff --git a/packages/adapters/src/backup/types.ts b/packages/adapters/src/backup/types.ts index fd2a62443..eac679419 100644 --- a/packages/adapters/src/backup/types.ts +++ b/packages/adapters/src/backup/types.ts @@ -401,9 +401,8 @@ export interface ProducerOpts { export interface RestoreOpts { /** Pass clearTarget through. */ clearTarget?: boolean; - /** Forwarded to the executor so a cancel doesn't have to wait out the - * whole extract. Producers that restore through `pipeIntoCommand` ignore - * it — those writes are transactional at the engine, not the volume. */ + /** Abort in-flight restore work. The payload's failure semantics determine + * whether an interrupted write rolls back or leaves partial data. */ signal?: AbortSignal; } diff --git a/packages/adapters/src/infra/nginx-dns-hooks.test.ts b/packages/adapters/src/infra/nginx-dns-hooks.test.ts index 744b4be8c..527e37a75 100644 --- a/packages/adapters/src/infra/nginx-dns-hooks.test.ts +++ b/packages/adapters/src/infra/nginx-dns-hooks.test.ts @@ -5,7 +5,7 @@ import { basename, dirname, join } from "node:path"; import { promisify } from "node:util"; import { expect, test } from "vitest"; import { LocalExecutor } from "../system/local-executor"; -import type { RootChecked } from "../system/privilege"; +import { rootChecked } from "../system/privilege"; import { NginxProvider } from "./nginx"; import { OPENRESTY_DEFAULT_PATHS } from "./openresty-lua"; @@ -86,7 +86,7 @@ test.each([ const nginx = new NginxProvider({ paths: { ...OPENRESTY_DEFAULT_PATHS, sitesDir: join(scratch, "sites-enabled") }, certDir: join(root, "live"), - executor: executor as RootChecked, + executor: rootChecked(executor, "Test executor intercepts privileged commands"), }); try { await expect( diff --git a/packages/adapters/src/system/local-execution-policy.test.ts b/packages/adapters/src/system/local-execution-policy.test.ts index 09056eae0..877d4372a 100644 --- a/packages/adapters/src/system/local-execution-policy.test.ts +++ b/packages/adapters/src/system/local-execution-policy.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { access, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; +import { Readable } from "node:stream"; import { LocalExecutor } from "./local-executor"; import { hasLocalCommand, sq } from "./local-shell"; import { createExecutor } from "./executor"; @@ -36,6 +37,8 @@ describe("Cloud cannot execute on the API host", () => { for (const operation of [ () => executor.exec(command), () => direct.exec(command), + () => direct.rawExec(command), + () => direct.execWithInput(command, Readable.from(["private backup data"])), () => executor.streamExec(command, onLog), () => executor.streamExec(command, onLog, { signal: AbortSignal.abort() }), () => executor.transferIn!("/does-not-exist", path), diff --git a/packages/adapters/src/system/local-executor-stream.test.ts b/packages/adapters/src/system/local-executor-stream.test.ts new file mode 100644 index 000000000..14c36ca1a --- /dev/null +++ b/packages/adapters/src/system/local-executor-stream.test.ts @@ -0,0 +1,154 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Readable, Writable } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { LocalExecutor } from "./local-executor"; +import { sq } from "./local-shell"; +import { BareRuntime } from "../runtime/bare"; +import { BareBackupExecutor } from "../backup/executors/bare"; +import { CustomCommandProducer } from "../backup/producers/custom-command"; + +const executors: LocalExecutor[] = []; +const dirs: string[] = []; +const node = (script: string) => `${sq(process.execPath)} -e ${sq(script)}`; +function executor() { + vi.stubEnv("CLOUD_MODE", "false"); + const result = new LocalExecutor(); + executors.push(result); + return result; +} +afterEach(async () => { + await Promise.all(executors.splice(0).map((exec) => exec.dispose())); + await Promise.all(dirs.splice(0).map((path) => rm(path, { recursive: true, force: true }))); + vi.unstubAllEnvs(); +}); + +describe("local bare backup command streams", () => { + it("captures and restores a binary artifact through the real bare executor", async () => { + const dir = await mkdtemp(join(tmpdir(), "openship-local-backup-")); + dirs.push(dir); + const file = join(dir, "restored.bin"); + const transport = executor(); + const bare = new BareBackupExecutor(new BareRuntime({ executor: transport })); + const service = { id: "mail", name: "mail", projectId: "", volumes: [], env: {} } as never; + const payload = Buffer.from(Array.from({ length: 256 }, (_, i) => i)); + const expected = createHash("sha256"); + for (let i = 0; i < 8192; i++) expected.update(payload); + const capture = await bare.execStream(service, [ + "sh", + "-c", + node( + "const b=Buffer.from(Array.from({length:256},(_,i)=>i));for(let i=0;i<8192;i++)process.stdout.write(b)", + ), + ]); + const hash = createHash("sha256"); + const restore = bare.pipeIntoCommand( + service, + ["sh", "-c", `cat > ${sq(file)}`], + capture.stdout, + ); + expect(await restore).toMatchObject({ code: 0 }); + expect(await capture.awaitExit).toMatchObject({ code: 0 }); + const restored = await readFile(file); + hash.update(restored); + expect(restored.length).toBe(256 * 8192); + expect(hash.digest("hex")).toBe(expected.digest("hex")); + }); + + it("preserves a failing loader's exit status and bounded diagnostic tail", async () => { + const exec = executor(); + const result = await exec.execWithInput( + node( + "process.stdin.resume();process.stdin.on('end',()=>{process.stderr.write('x'.repeat(32768)+'loader failed');process.exitCode=42})", + ), + Readable.from([Buffer.from([0, 255, 128])]), + ); + expect(result.code).toBe(42); + expect(result.stderr).toHaveLength(16 * 1024); + expect(result.stderr).toMatch(/loader failed$/); + }); + + it("propagates an input failure instead of accepting a partial restore", async () => { + const exec = executor(); + const source = Readable.from( + (async function* () { + yield Buffer.from("partial"); + throw new Error("object download interrupted"); + })(), + ); + await expect(exec.execWithInput("cat >/dev/null", source)).rejects.toThrow( + "object download interrupted", + ); + }); + + it("cancels a command that has consumed stdin but is still running", async () => { + const exec = executor(); + const cancel = new AbortController(); + const result = exec.runWithAbortSignal(cancel.signal, () => + exec.execWithInput( + node("process.stdin.resume();process.stdin.on('end',()=>setInterval(()=>{},1000))"), + Readable.from(["data"]), + ), + ); + const assertion = expect(result).rejects.toThrow("stop restore"); + setTimeout(() => cancel.abort(new Error("stop restore")), 150); + await assertion; + }); + + it("forwards restore cancellation through the shared custom-command producer", async () => { + const transport = executor(); + const bare = new BareBackupExecutor(new BareRuntime({ executor: transport })); + const cancel = new AbortController(); + const service = { id: "mail", name: "mail", projectId: "", volumes: [], env: {} } as never; + const restore = CustomCommandProducer.restore( + service, + bare, + { + key: "mail.tar.zst", + payloadKind: "custom_command", + sha256: "", + sizeBytes: 4, + metadata: { + restoreCommand: node( + "process.stdin.resume();process.stdin.on('end',()=>setInterval(()=>{},1000))", + ), + }, + open: async () => Readable.from(["mail"]), + }, + { signal: cancel.signal }, + ); + const assertion = expect(restore).rejects.toThrow("stop mail restore"); + setTimeout(() => cancel.abort(new Error("stop mail restore")), 150); + await assertion; + }); + + it("streams with backpressure and reports a capture failure", async () => { + const exec = executor(); + const capture = await exec.rawExec( + node( + "process.stdout.write(Buffer.alloc(100000));process.stderr.write('capture failed');process.exitCode=7", + ), + ); + let bytes = 0; + let stderr = ""; + capture.stderr.on("data", (chunk) => { + stderr += chunk; + }); + await pipeline( + capture.stdout, + new Writable({ + highWaterMark: 1024, + write(chunk, _encoding, done) { + bytes += chunk.length; + setTimeout(done, 1); + }, + }), + ); + expect(await capture.onClose).toBe(7); + expect(bytes).toBe(100000); + expect(stderr).toBe("capture failed"); + }); +}); diff --git a/packages/adapters/src/system/local-executor.ts b/packages/adapters/src/system/local-executor.ts index 06daaa8fd..ca85ef655 100644 --- a/packages/adapters/src/system/local-executor.ts +++ b/packages/adapters/src/system/local-executor.ts @@ -1,5 +1,8 @@ import { reportCaughtError as observeCaughtError } from "@repo/core/diagnostics"; +import { AsyncLocalStorage } from "node:async_hooks"; import { exec, spawn } from "node:child_process"; +import { pipeline, type Readable } from "node:stream"; +import { promisify } from "node:util"; import { access, mkdir as fsMkdir, @@ -27,6 +30,103 @@ import { * File operations use node:fs directly. */ export class LocalExecutor implements CommandExecutor { + private readonly abortScope = new AsyncLocalStorage(); + private readonly activeRawCommands = new Set<{ kill: () => void; onClose: Promise }>(); + + runWithAbortSignal(signal: AbortSignal, work: () => Promise): Promise { + return this.abortScope.run(signal, work); + } + + /** Binary transport for self-hosted/desktop workloads. Never available to a + * Cloud controller, including when this executor predates mode initialization. */ + private async spawnRawCommand(command: string) { + assertLocalHostExecutionAllowed(); + const signal = this.abortScope.getStore(); + signal?.throwIfAborted(); + const grouped = process.platform !== "win32"; + const child = spawn(getLocalShellPath(), getLocalShellArgs(command), { + stdio: ["pipe", "pipe", "pipe"], + env: getLocalExecEnv(), + detached: grouped, + }); + let terminating = false; + let escalation: ReturnType | undefined; + const send = (kind: NodeJS.Signals) => { + try { + if (grouped && child.pid) process.kill(-child.pid, kind); + else child.kill(kind); + } catch (error) { + // A child can exit between the close event and cancellation. + if ((error as NodeJS.ErrnoException).code !== "ESRCH") + observeCaughtError(error, "adapters/system/local-executor"); + } + }; + const kill = () => { + if (terminating) return; + terminating = true; + send("SIGTERM"); + escalation = setTimeout(() => send("SIGKILL"), 1_000); + escalation.unref?.(); + }; + const onClose = new Promise((resolve, reject) => { + let failure: Error | undefined; + child.once("error", (error) => { + // diagnostics-ignore: onClose rejects with this failure after stdio closes. + failure = error; + }); + child.once("close", (code) => { + if (terminating && grouped) send("SIGKILL"); + clearTimeout(escalation); + signal?.removeEventListener("abort", kill); + this.activeRawCommands.delete(active); + if (signal?.aborted) reject(signal.reason ?? new Error("Local command cancelled")); + else if (failure) reject(failure); + else resolve(code ?? 1); + }); + }); + // The process can fail before the caller attaches its stream lifecycle. + void onClose.catch((error) => observeCaughtError(error, "adapters/system/local-executor")); + const active = { kill, onClose }; + this.activeRawCommands.add(active); + signal?.addEventListener("abort", kill, { once: true }); + if (signal?.aborted) kill(); + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + return { child, ...active }; + } + + async rawExec(command: string) { + const { child, onClose, kill } = await this.spawnRawCommand(command); + child.stdin.end(); + return { stdout: child.stdout, stderr: child.stderr, onClose, kill }; + } + + async execWithInput(command: string, body: Readable) { + const { child, onClose, kill } = await this.spawnRawCommand(command); + let stderr = ""; + let stdout = ""; + child.stdout.on("data", (data: Buffer) => { stdout = (stdout + data.toString()).slice(-16 * 1024); }); + child.stderr.on("data", (data: Buffer) => { stderr = (stderr + data.toString()).slice(-16 * 1024); }); + let inputError: NodeJS.ErrnoException | undefined; + const input = promisify(pipeline)(body, child.stdin).catch((error) => { + // diagnostics-ignore: propagated below after the subprocess and input settle. + inputError = error; + kill(); + }); + try { + const code = await onClose; + await input; + if (inputError && (code === 0 || !["EPIPE", "ERR_STREAM_PREMATURE_CLOSE"].includes(inputError.code ?? ""))) + throw inputError; + return { code, stderr: stderr.trim(), stdout: stdout.trim() }; + } finally { + // Await upstream teardown before a restore releases its execution lock. + await input; + } + } + async exec(command: string, opts?: { timeout?: number }): Promise { assertLocalHostExecutionAllowed(); return new Promise((resolve, reject) => { @@ -221,6 +321,8 @@ export class LocalExecutor implements CommandExecutor { } async dispose(): Promise { - // Nothing to clean up for local execution + const commands = [...this.activeRawCommands]; + for (const command of commands) command.kill(); + await Promise.allSettled(commands.map((command) => command.onClose)); } } diff --git a/packages/adapters/src/types.ts b/packages/adapters/src/types.ts index 03ac2b605..a0fe5b399 100644 --- a/packages/adapters/src/types.ts +++ b/packages/adapters/src/types.ts @@ -983,7 +983,7 @@ export interface CommandExecutor extends ExecOnly { * Run a command and return the raw stdout/stderr streams without * line splitting. Enables byte-for-byte piping of command output. * - * Only available on SshExecutor - local executors do not implement this. + * Local execution enforces the shared Cloud-host refusal at invocation time. */ rawExec?(command: string): Promise<{ stdout: Readable; @@ -999,7 +999,7 @@ export interface CommandExecutor extends ExecOnly { * rawExec (stdout) — lets a Readable move INTO a remote command byte-for-byte * without staging to a temp file first. * - * Only available on SshExecutor — local executors do not implement this. + * Local execution enforces the shared Cloud-host refusal at invocation time. */ execWithInput?( command: string, diff --git a/packages/platform/src/engine/modules/backups/backup-lock.ts b/packages/platform/src/engine/modules/backups/backup-lock.ts index 3e3bc8e5a..4a13b01c6 100644 --- a/packages/platform/src/engine/modules/backups/backup-lock.ts +++ b/packages/platform/src/engine/modules/backups/backup-lock.ts @@ -14,3 +14,8 @@ export function withBackupRunLock(runId: string, work: () => Promise): Pro export function withBackupDestinationLock(destinationId: string, work: () => Promise): Promise { return createProvisionLock(`backup-destination:${destinationId}`).run(work); } + +/** Capture and restore must never independently stop/resume the same mail writers. */ +export function withMailBackupLock(serverId: string, work: () => Promise): Promise { + return createProvisionLock(`mail-backup:${serverId}`).run(work); +} diff --git a/packages/platform/src/engine/modules/backups/backup.orchestrator.ts b/packages/platform/src/engine/modules/backups/backup.orchestrator.ts index 51de33e5d..398e61bcb 100644 --- a/packages/platform/src/engine/modules/backups/backup.orchestrator.ts +++ b/packages/platform/src/engine/modules/backups/backup.orchestrator.ts @@ -63,11 +63,12 @@ import { import { notification } from "../../lib/notification-dispatcher"; import { prunePolicy } from "./retention-prune"; import { assertPlausibleBackupSizes } from "./backup-size-check"; -import { withBackupDestinationLock, withBackupPolicyLock } from "./backup-lock"; +import { withBackupDestinationLock, withBackupPolicyLock, withMailBackupLock } from "./backup-lock"; import { withProjectRuntimeLock } from "../../lib/project-runtime-lock"; import { serviceHandleFor, withContainerEnv } from "./service-handle"; import { resolveSourceExecutor } from "./source-platform"; import { projectBackupVolumes, resolveProjectBackupSource } from "./project-source"; +import type { prepareMailBackup } from "../mail/backup-lifecycle"; import crypto from "node:crypto"; import { detectDbImage, incrementalBackupStorage, payloadSpec, safeErrorMessage, ValidationError } from "@repo/core"; import { @@ -347,7 +348,9 @@ export class BackupOrchestrator { const destinationId = pending.destinationId ?? policy?.destinationId; const executeUnderProject = () => pending.projectId ? withProjectRuntimeLock(pending.projectId, () => this.executeRun(runId, policy ?? null)) - : this.executeRun(runId, policy ?? null); + : pending.mailServerId + ? withMailBackupLock(pending.mailServerId, () => this.executeRun(runId, policy ?? null)) + : this.executeRun(runId, policy ?? null); const executeUnderDestination = () => destinationId ? withBackupDestinationLock(destinationId, executeUnderProject) : executeUnderProject(); @@ -402,6 +405,7 @@ export class BackupOrchestrator { // remote server it carries a Docker-over-SSH loopback bridge that only // `dispose()` closes, so a scheduled policy would otherwise strand one per run. let sourceRuntime: RuntimeAdapter | null = null; + let mailCapture: Awaited> | null = null; /** Did `destination.preflight()` succeed this run? Gates the failure path's * verdict on the destination — see the catch block. */ let destinationProven = false; @@ -590,6 +594,13 @@ export class BackupOrchestrator { // value outside the codec vocabulary reached the shell layer and produced an // artifact whose recorded codec no restore can read. See that function. const producerOpts = sanitizeProducerOpts(policy.payloadConfig); + if (policy.sourceKind === "mail_server") { + if (policy.payloadKind !== "custom_command") + throw new Error("The mail policy does not target a supported mail host"); + const { prepareMailBackup } = await import("../mail/backup-lifecycle"); + mailCapture = await prepareMailBackup(sourceRuntime, runId, policy.payloadConfig); + Object.assign(producerOpts, mailCapture.payload.payloadConfig); + } const incremental = producerOpts.incremental === true; // Compress blocks independently so editing one file does not change a // compressor's state for the remainder of a filesystem archive. @@ -633,6 +644,7 @@ export class BackupOrchestrator { }; for await (const artifact of producer.produce(serviceHandle, executor, producerOpts)) { + if (mailCapture) artifact.metadata = { ...artifact.metadata, mail: mailCapture.metadata }; const key = artifactKey(baseKey, artifact.name); if (artifactNames.has(key) || key === manifestKey(baseKey)) { artifact.stream.destroy(); @@ -658,6 +670,11 @@ export class BackupOrchestrator { }); } + if (mailCapture) { + await mailCapture.cleanup(); + mailCapture = null; + } + // 5b. A run that captured nothing is a FAILED run, whichever producer ran. // // This is the backstop behind #611: the volume producer used to return @@ -794,7 +811,19 @@ export class BackupOrchestrator { await finishHook(); // Both forms are scrubbed independently: a second `.slice` over an // already-scrubbed string can split a surrogate pair back open. - const raw = safeErrorMessage(err); + let raw = safeErrorMessage(err); + // Recover the source before recording the terminal verdict. A failed + // restart must be included in that verdict, not lost to the terminal guard. + if (mailCapture) { + const capture = mailCapture; + mailCapture = null; + try { + await capture.cleanup(); + } catch (recoveryError) { + raw += `\nMail recovery needs attention: ${safeErrorMessage(recoveryError)}`; + observeCaughtError(recoveryError, "platform/engine/modules/backups/backup.orchestrator"); + } + } const message = boundedStorableText(raw, TRUNCATE_ERROR); const summary = boundedStorableText(raw, TRUNCATE_ERROR_SUMMARY); const finalizedElsewhere = err instanceof BackupRunFinalizedElsewhereError; @@ -919,7 +948,13 @@ export class BackupOrchestrator { }); } } finally { - disposeRuntime(sourceRuntime); + try { + if (mailCapture) await mailCapture.cleanup(); + } catch (error) { + errorDiagnostics.error("platform/engine/modules/backups/backup.orchestrator", "Mail backup cleanup/restart needs attention", error); + } finally { + disposeRuntime(sourceRuntime); + } // Status says what happened; this acknowledgement says the worker can no // longer mutate project resources. It must be the last durable write in // the outermost worker boundary so project teardown never treats an early diff --git a/packages/platform/src/engine/modules/backups/restore.orchestrator.ts b/packages/platform/src/engine/modules/backups/restore.orchestrator.ts index aad1b3ae1..453b706db 100644 --- a/packages/platform/src/engine/modules/backups/restore.orchestrator.ts +++ b/packages/platform/src/engine/modules/backups/restore.orchestrator.ts @@ -78,8 +78,9 @@ import { notification } from "../../lib/notification-dispatcher"; import { boundedStorableText } from "../deployments/build-log-sanitize"; import { deferBackgroundWork } from "../../lib/background-work"; import { assertNativeJobs } from "../../native/execution-policy"; -import { withBackupRunLock } from "./backup-lock"; +import { withBackupRunLock, withMailBackupLock } from "./backup-lock"; import { withProjectRuntimeLock } from "../../lib/project-runtime-lock"; +import type { prepareMailRestore } from "../mail/backup-lifecycle"; const TRUNCATE_ERROR = 4096; @@ -976,7 +977,12 @@ export class RestoreOrchestrator { const restore = await repos.backupRestore.findById(restoreId); const target = restore?.forkServiceId ? await repos.service.findById(restore.forkServiceId) : null; const projectId = target?.projectId ?? restore?.projectId; - return projectId ? withProjectRuntimeLock(projectId, () => this.runApplyLocked(restoreId)) : this.runApplyLocked(restoreId); + if (projectId) return withProjectRuntimeLock(projectId, () => this.runApplyLocked(restoreId)); + const run = restore ? await repos.backupRun.findById(restore.runId) : null; + const mailServerId = restore?.mode === "to_fork" ? restore.forkMailServerId : run?.mailServerId; + return mailServerId + ? withMailBackupLock(mailServerId, () => this.runApplyLocked(restoreId)) + : this.runApplyLocked(restoreId); } private async runApplyLocked(restoreId: string): Promise { @@ -988,6 +994,7 @@ export class RestoreOrchestrator { this.inFlight.set(restoreId, controller); // Released alongside the in-flight handle at the bottom — see resolveTarget. let targetRuntime: RuntimeAdapter | null = null; + let mailRestore: Awaited> | null = null; // The two facts the terminal row reports, hoisted to the function because the outer // catch is what writes them and they are decided deep inside the apply. // @@ -1067,9 +1074,23 @@ export class RestoreOrchestrator { await executor.stopService(serviceHandle); serviceDown = true; } + if (sourceRun.sourceKind === "mail_server") { + const { prepareMailRestore, readMailBackupMetadata } = await import("../mail/backup-lifecycle"); + const metadata = readMailBackupMetadata(recordedArtifacts(sourceRun)[0]?.metadata ?? {}); + if (metadata) { + const serverId = restore.mode === "to_fork" ? restore.forkMailServerId : sourceRun.mailServerId; + if (!serverId) throw new Error("Mail restore has no target server"); + mailRestore = await prepareMailRestore(runtime, serverId, restoreId, metadata); + serviceDown = mailRestore.stopped; + } + } // Checkpoint 2 — stopped, still nothing written. await this.throwIfCancelRequested(restoreId, null, async () => { + if (mailRestore) { + await mailRestore.resume(); + serviceDown = false; + } if (stoppedByUs) { await executor .startService(serviceHandle) @@ -1135,7 +1156,9 @@ export class RestoreOrchestrator { executor, { key: recorded.key, - metadata: recorded.metadata, + metadata: mailRestore + ? { ...recorded.metadata, restoreCommand: mailRestore.restoreCommand } + : recorded.metadata, payloadKind: recorded.payloadKind, sha256: recorded.sha256 ?? "", sizeBytes: recorded.sizeBytes, @@ -1207,6 +1230,10 @@ export class RestoreOrchestrator { // telling the operator their data is partial and to re-run the restore. The one // exception raises its own `PartialWriteError` with its own precise wording: a // failed bounce means the bytes are down but the server has not loaded them. + if (mailRestore) { + await mailRestore.finalize(); + serviceDown = false; + } unfinishedTarget = null; // Checkpoint 4 DECLINES: every byte is written and the only thing left is @@ -1306,6 +1333,17 @@ export class RestoreOrchestrator { throw innerErr; } } catch (err) { + // Mail's writer pause is independent of the synthetic host service. A + // download/probe failure before the first write must undo that pause too. + if (mailRestore) serviceDown = mailRestore.stopped; + if (mailRestore && unfinishedTarget === null && !(err instanceof PartialWriteError)) { + try { + await mailRestore.resume(); + serviceDown = false; + } catch (recoveryError) { + observeCaughtError(recoveryError, "platform/engine/modules/backups/restore.orchestrator"); + } + } // Two INDEPENDENT facts, stamped independently — see the declarations at the top // of this method. `partialWrite` is about the data, `serviceLeftStopped` about the // container, and a restore can be any combination of the two: a cancelled volume @@ -1352,7 +1390,13 @@ export class RestoreOrchestrator { }); } finally { this.inFlight.delete(restoreId); - disposeRuntime(targetRuntime); + try { + await mailRestore?.cleanup(); + } catch (cleanupError) { + observeCaughtError(cleanupError, "platform/engine/modules/backups/restore.orchestrator"); + } finally { + disposeRuntime(targetRuntime); + } } } @@ -1531,6 +1575,9 @@ export class RestoreOrchestrator { if (!targetMailServerId) { throw new Error("Mail restore has no target mail server"); } + const { assertMailRestoreTarget, readMailBackupMetadata } = await import("../mail/backup-lifecycle"); + const metadata = readMailBackupMetadata(recordedArtifacts(sourceRun)[0]?.metadata ?? {}); + if (metadata) await assertMailRestoreTarget(targetMailServerId, metadata); const built = await this.buildMailTarget(targetMailServerId, destinationRow.organizationId); return { executor: built.executor, serviceHandle: built.handle, runtime: built.runtime }; } diff --git a/packages/platform/src/engine/modules/mail/admin/domains.service.ts b/packages/platform/src/engine/modules/mail/admin/domains.service.ts index ef1796985..9aa1eb276 100644 --- a/packages/platform/src/engine/modules/mail/admin/domains.service.ts +++ b/packages/platform/src/engine/modules/mail/admin/domains.service.ts @@ -12,6 +12,7 @@ */ import { reportCaughtError as observeCaughtError, diagnostics as errorDiagnostics } from "@repo/core/diagnostics"; +import { validateMailDomain as validateDomain } from "../mail-domain"; import { sshManager } from "../../../lib/ssh-manager"; import { readState } from "../mail-state"; import { provisionDomainDkim, genSecret } from "../mail.service"; @@ -24,7 +25,6 @@ import { } from "./domain-dns.service"; import { relaySpfInclude } from "./outbound-relay.service"; -const DOMAIN_RE = /^[a-z0-9][a-z0-9-]*(\.[a-z0-9][a-z0-9-]*)+$/i; export interface DomainRow { domain: string; @@ -82,12 +82,7 @@ const SELECT_COLUMNS = ` * `openship_meta` table on the mail VPS (NOT on openship's DB). */ -export function validateDomain(domain: string): void { - const d = domain.trim().toLowerCase(); - if (!DOMAIN_RE.test(d) || d.length > 255) { - throw new Error(`Invalid domain: ${domain}`); - } -} +export { validateMailDomain as validateDomain } from "../mail-domain"; export async function listDomains(serverId: string): Promise { return queryRows( diff --git a/packages/platform/src/engine/modules/mail/admin/outbound-relay.service.ts b/packages/platform/src/engine/modules/mail/admin/outbound-relay.service.ts index e2506708a..15685ff78 100644 --- a/packages/platform/src/engine/modules/mail/admin/outbound-relay.service.ts +++ b/packages/platform/src/engine/modules/mail/admin/outbound-relay.service.ts @@ -508,6 +508,7 @@ async function dropRelayTlsPolicy( export async function configureOutboundRelay( exec: CommandExecutor, input: ConfigureRelayInput, + options: { reload?: boolean } = {}, ): Promise { validate(input); const { saslMap, tlsPolicy, hostFiles, engineExec, engine } = await relayTransport(exec); @@ -613,7 +614,7 @@ export async function configureOutboundRelay( } // 4) Reload Postfix (inside the engine). - await engineExec.exec(engine("postfix reload")); + if (options.reload !== false) await engineExec.exec(engine("postfix reload")); // 5) Persist the relay block (encrypted password) + fan DNS across every // relayed domain (primary + additional). @@ -650,7 +651,7 @@ export async function configureOutboundRelay( } /** Disable the outbound relay — revert Postfix to direct-to-MX + clear state. */ -export async function disableOutboundRelay(exec: CommandExecutor): Promise { +export async function disableOutboundRelay(exec: CommandExecutor, options: { reload?: boolean } = {}): Promise { const { saslMap, tlsPolicy, hostFiles, engineExec, engine } = await relayTransport(exec); // Remove the relay-specific directives (leave smtp_tls_security_level — it's @@ -678,7 +679,7 @@ export async function disableOutboundRelay(exec: CommandExecutor): Promise, +): MailBackupMetadata | null { + const mail = metadata.mail as Partial | undefined; + if (!Object.hasOwn(metadata, "mail")) return null; // Legacy recorded command. + if ( + !mail || + typeof mail !== "object" || + mail.version !== 2 || + typeof mail.domain !== "string" || + typeof mail.keys !== "boolean" || + typeof mail.messageData !== "boolean" + ) + throw new Error("This mail backup has unsupported or invalid archive metadata"); + return mail as MailBackupMetadata; +} + +function stagingName(id: string) { + if (!/^[a-zA-Z0-9_-]+$/.test(id)) throw new Error("Invalid mail backup operation id"); + return `mail-backup-${id}.json`; +} + +function commandExecutor(runtime: RuntimeAdapter | null): CommandExecutor { + if (!(runtime instanceof BareRuntime)) throw new Error("Mail backups require a host runtime"); + return runtime.commandExecutor; +} + +/** Per-run preparation is shared by scheduled/manual mail captures. Never put + * portable credentials in the policy, artifact metadata or API progress response. */ +export async function prepareMailBackup( + runtime: RuntimeAdapter | null, + runId: string, + config: unknown, +) { + const exec = commandExecutor(runtime); + const state = await readState(exec, { strict: true }); + if (!state) throw new Error("The mail server has no saved configuration to back up"); + const flags = mailBackupFlags(config); + const flavor = await resolveMailFlavor(exec); + if (flags.messageData) { + const stores = await queryRows<{ storagebasedirectory: string }>( + exec, + "SELECT DISTINCT storagebasedirectory FROM mailbox", + ); + if (stores.some((store) => store.storagebasedirectory?.replace(/\/+$/, "") !== "/var/vmail")) { + throw new Error( + "This mail server uses a mailbox storage root outside /var/vmail. Back up that custom store before migrating; a full managed-mail backup cannot omit it.", + ); + } + } + const name = stagingName(runId); + let paused: Awaited> | undefined; + const cleanup = async () => { + try { + await paused?.resume(); + } finally { + await removeOpenshipFile(exec, name); + } + }; + try { + if (flags.keys) await writeOpenshipFile(exec, name, JSON.stringify(exportMailState(state))); + const payload = buildMailBackupPayload(state.domain, flags, flavor, { + ...(flags.keys ? { statePath: `${OPENSHIP_DIR}/${name}` } : {}), + }); + await exec.exec(payload.preflightCommand); + if (flags.messageData) paused = await pauseMailDaemons(exec, flavor); + return { + payload, + metadata: { version: 2, domain: state.domain, ...flags } satisfies MailBackupMetadata, + cleanup, + }; + } catch (error) { + await cleanup(); + throw error; + } +} + +export async function assertMailRestoreTarget(serverId: string, metadata: MailBackupMetadata) { + const target = await repos.mailServer.get(serverId); + if (!target || target.domain !== metadata.domain) + throw new Error( + `Prepare the target mail server with the backup's primary domain (${metadata.domain}) before restoring. No data was changed.`, + ); +} + +/** Data is streamed by the normal backup producer. This finalizer restores its + * portable server state and uses the same DKIM, DNS and relay paths as setup. */ +export async function prepareMailRestore( + runtime: RuntimeAdapter | null, + serverId: string, + restoreId: string, + metadata: MailBackupMetadata, +) { + await assertMailRestoreTarget(serverId, metadata); + const exec = commandExecutor(runtime); + const savedTarget = await readState(exec, { strict: true }); + if (!savedTarget) throw new Error("Run mail setup on the target before restoring its data"); + if (savedTarget.domain !== metadata.domain) + throw new Error("The target mail configuration has a different primary domain"); + // An adopted server can still carry its former controller's id on disk. + const target = { ...savedTarget, serverId }; + const flavor = await resolveMailFlavor(exec); + const name = stagingName(restoreId); + // Remove a leftover handoff before a retry; missing configuration must never + // accidentally reuse data left by a different attempt. + await removeOpenshipFile(exec, name); + const payload = buildMailBackupPayload(metadata.domain, metadata, flavor, { + restoreStatePath: `${OPENSHIP_DIR}/${name}`, + }); + const paused = await pauseMailDaemons(exec, flavor); + return { + resume: paused.resume, + get stopped() { + return paused.stopped; + }, + restoreCommand: payload.payloadConfig.restoreCommand, + cleanup: () => removeOpenshipFile(exec, name), + finalize: async () => { + // Without saved credentials, the old target's cached mailbox passwords no + // longer correspond to the restored account hashes. Normal mailbox setup + // can recreate these managed credentials when next needed. + const { DOMAIN_ADMIN_PASSWD_PLAIN: _oldPostmaster, ...targetSecrets } = target.secrets; + let state: MailServerState = { + ...target, + secrets: targetSecrets, + platformMailbox: undefined, + testMailboxes: undefined, + additionalDomains: undefined, + }; + if (metadata.keys) { + const raw = await readOpenshipFile(exec, name, { strict: true }); + if (!raw) + throw new Error("The restored archive did not contain its portable mail configuration"); + state = importMailState(JSON.parse(raw) as PortableMailState, target); + await writeState(exec, state); + } + const domains = await queryRows<{ domain: string }>( + exec, + "SELECT domain FROM domain ORDER BY domain", + ); + if (!domains.some(({ domain }) => domain === state.domain)) + throw new Error("The restored mail database has no primary domain"); + const additionalDomains: NonNullable = {}; + for (const { domain } of domains) { + const dkim = await provisionDomainDkim(exec, domain, { reload: false }); + if (domain !== state.domain) + additionalDomains[domain] = { + ...state.additionalDomains?.[domain], + records: buildDomainDnsRecords(state.domain, domain, dkim), + acknowledgedAt: null, + createdAt: state.additionalDomains?.[domain]?.createdAt ?? new Date().toISOString(), + }; + } + const dns = await stepDkimKeys(exec, state.domain, () => {}); + if (!dns.success || !dns.data?.dnsRecords) throw new Error(dns.message); + state = { + ...recordStep(state, dns), + additionalDomains, + dnsRecords: dns.data.dnsRecords as Record, + dnsAcknowledged: false, + ptrAcknowledged: false, + finishedAt: new Date().toISOString(), + }; + await writeState(exec, state); + if (state.outboundRelay?.enabled) { + const relay = state.outboundRelay; + await configureOutboundRelay( + exec, + { ...relay, password: decrypt(relay.passwordEncrypted) }, + { reload: false }, + ); + } else if (metadata.keys) { + await disableOutboundRelay(exec, { reload: false }); + } + await removeOpenshipFile(exec, name); + await paused.resume(); + await repos.mailServer.markInstalled(serverId, state.domain); + }, + }; +} diff --git a/packages/platform/src/engine/modules/mail/backup-plan.ts b/packages/platform/src/engine/modules/mail/backup-plan.ts new file mode 100644 index 000000000..2d62c0c93 --- /dev/null +++ b/packages/platform/src/engine/modules/mail/backup-plan.ts @@ -0,0 +1,190 @@ +/** Mail's archive layout; capture, restore and migration share this one plan. */ +import { HOST_STATE_DIR, safePipelineFragment } from "@repo/adapters"; +import { shellQuote } from "@repo/core"; +import { + HOST_AMAVIS_CONF_CANDIDATES, + mailDaemonReloadCommand, + mailEngineCommand, + mailHostPath, + mailPgDumpToStdout, + mailPsqlFromStdin, + mailPsqlCommand, + type MailEngineFlavor, +} from "./mail-engine"; + +export interface MailBackupFlags { + messageData: boolean; + keys: boolean; +} +export function mailBackupFlags(config: unknown): MailBackupFlags { + const mail = (config as { mail?: Partial } | null)?.mail; + return { messageData: mail?.messageData !== false, keys: mail?.keys !== false }; +} + +export interface MailBackupPayload { + /** Required-file checks, repeated at capture to guard against later changes. */ + preflightCommand: string; + payloadKind: "custom_command"; + payloadConfig: { + produceCommand: string; + restoreCommand: string; + artifactName: string; + mail: MailBackupFlags & { version: 2 }; + }; +} +export interface MailBackupPlanOptions { + /** A root-only portable state file prepared by the shared mail lifecycle. */ + statePath?: string; + /** A root-only handoff consumed and removed by restore finalization. */ + restoreStatePath?: string; +} + +// These databases belong to the mail stack (db-bootstrap.sh). Never dump +// unrelated application/control-plane databases from a shared host cluster. +const AUXILIARY_DATABASES = ["amavisd", "iredapd", "fail2ban"] as const; +const dumpFlags = "--clean --if-exists --no-owner"; +// Postfix queue data, excluding target-specific chroot libraries, sockets and PID files. +const QUEUE_DIRECTORIES = + "active bounce corrupt defer deferred flush hold incoming maildrop saved trace"; +// iRedMail uses these Amavis roots on supported Debian/Ubuntu and RHEL hosts. +const amavisRoot = + 'if [ -d /var/lib/amavis ]; then root=/var/lib/amavis; elif [ -d /var/spool/amavisd ]; then root=/var/spool/amavisd; else echo "Mail filter storage not found" >&2; exit 1; fi'; +const amavisDataNames = "db quarantine virusmails .spamassassin"; + +export function buildMailBackupPayload( + domain: string, + flags: MailBackupFlags, + flavor: MailEngineFlavor = "host", + options: MailBackupPlanOptions = {}, +): MailBackupPayload { + const maildir = mailHostPath(flavor, "/var/vmail"); + const dkim = mailHostPath(flavor, "/var/lib/dkim"); + const amavisPaths = + flavor === "container" + ? [mailHostPath(flavor, "/etc/amavis/conf.d/50-user")] + : HOST_AMAVIS_CONF_CANDIDATES.map((candidate) => candidate.path); + const statePath = options.statePath ?? `${HOST_STATE_DIR}/mail-state.json`; + const preflight = [ + "set -eu", + ...(flags.keys + ? [ + `test -d ${shellQuote(dkim)}`, + `test -s ${shellQuote(`${dkim}/${domain}.pem`)} || { echo "The mail server's primary DKIM key is missing or unreadable" >&2; exit 1; }`, + `test -s ${shellQuote(statePath)}`, + ] + : []), + ...(flags.messageData ? [`test -d ${shellQuote(maildir)}`] : []), + ].join("\n"); + const info = JSON.stringify({ version: 2, domain, ...flags, portableState: !!options.statePath }); + const produce = [ + "set -eu", + "umask 077", + 'tmp="$(mktemp -d)"', + "trap 'rm -rf \"$tmp\"' EXIT", + 'stage="$tmp/staging"', + 'mkdir -p "$stage"', + preflight, + `${mailPgDumpToStdout(flavor, dumpFlags)} > "$stage/vmail.sql"`, + 'mkdir -p "$stage/databases"', + ...AUXILIARY_DATABASES.map((db) => + [ + `if [ "$(${mailPsqlCommand(flavor, `SELECT 1 FROM pg_database WHERE datname='${db}'`)})" = 1 ]; then`, + ` ${mailPgDumpToStdout(flavor, dumpFlags, db)} > "$stage/databases/${db}.sql"`, + "fi", + ].join("\n"), + ), + `printf '%s' ${shellQuote(info)} > "$stage/mail-backup.json"`, + flags.keys + ? [ + 'mkdir -p "$stage/keys"', + `cp -a ${shellQuote(dkim)} "$stage/keys/dkim"`, + `for c in ${amavisPaths.map(shellQuote).join(" ")}; do`, + ' [ -f "$c" ] || continue', + ' cp -a "$c" "$stage/keys/amavis-conf"', + ' printf \'%s\' "$c" > "$stage/keys/amavis-conf.path"', + " break", + "done", + `cp -a ${shellQuote(statePath)} "$stage/keys/mail-state.json"`, + ].join("\n") + : "", + ...(flags.messageData + ? [ + `${mailEngineCommand(flavor, `tar -c -C /var/spool/postfix ${QUEUE_DIRECTORIES}`)} > "$stage/postfix.tar"`, + `${mailEngineCommand(flavor, `sh -c ${shellQuote(`${amavisRoot}; cd "$root"; for name in ${amavisDataNames}; do if [ -d "$name" ]; then printf '%s\\n' "$name"; fi; done | tar -c -T -`)}`)} > "$stage/amavis.tar"`, + ] + : []), + safePipelineFragment( + flags.messageData + ? `tar -c --transform=${shellQuote("flags=rh;s#^\\.$#vmail#;s#^\\./#vmail/#;s#^staging/#./#")} -C "$tmp" staging -C ${shellQuote(maildir)} .` + : 'tar -c -C "$stage" .', + "zstd -c -3", + { left: "tar (staging the mail archive)", right: "the compressor 'zstd'" }, + ), + ] + .filter(Boolean) + .join("\n"); + + const restore = [ + "set -eu", + "umask 077", + 'tmp="$(mktemp -d)"', + "trap 'rm -rf \"$tmp\"' EXIT", + safePipelineFragment("zstd -d", 'tar -x -C "$tmp"', { + left: "the decompressor 'zstd -d'", + right: "tar (extracting the mail archive)", + }), + // Refuse an empty or unrelated archive before touching live data. + '[ -s "$tmp/vmail.sql" ] && [ -s "$tmp/mail-backup.json" ] || { echo "Invalid mail backup: required data is missing" >&2; exit 1; }', + flags.keys && options.restoreStatePath + ? '[ -s "$tmp/keys/mail-state.json" ] || { echo "Mail backup has no portable configuration" >&2; exit 1; }' + : "", + flags.messageData + ? '[ -d "$tmp/vmail" ] && [ -s "$tmp/postfix.tar" ] && [ -s "$tmp/amavis.tar" ] || { echo "Mail backup has no message store or queue" >&2; exit 1; }' + : "", + ...(flags.messageData + ? ['tar -tf "$tmp/postfix.tar" > /dev/null', 'tar -tf "$tmp/amavis.tar" > /dev/null'] + : []), + // Each database's schema and rows restore in one transaction. Roles and + // their passwords stay on the destination; the dump retains mail grants. + ...AUXILIARY_DATABASES.map( + (db) => + `if [ -f "$tmp/databases/${db}.sql" ]; then ${mailPsqlFromStdin(flavor, { singleTransaction: true, database: db })} < "$tmp/databases/${db}.sql"; fi`, + ), + `${mailPsqlFromStdin(flavor, { singleTransaction: true })} < "$tmp/vmail.sql"`, + `if [ -d "$tmp/keys/dkim" ]; then mkdir -p ${shellQuote(dkim)}; cp -a "$tmp/keys/dkim/." ${shellQuote(dkim)}/; fi`, + // The destination's own mail configuration/DB credentials stay authoritative. + // Finalization reuses normal DKIM/relay primitives after the data is restored. + options.restoreStatePath + ? [ + 'if [ -f "$tmp/keys/mail-state.json" ]; then', + ` install -m 0600 "$tmp/keys/mail-state.json" ${shellQuote(options.restoreStatePath)}`, + "fi", + ].join("\n") + : "", + flags.messageData + ? [ + // Keep bind-mount roots in place so the engine observes the replacement. + `find -H ${shellQuote(maildir)} -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +`, + `cp -a "$tmp/vmail/." ${shellQuote(maildir)}/`, + mailEngineCommand(flavor, "chown -R -H vmail:vmail /var/vmail"), + `${mailEngineCommand(flavor, `sh -c ${shellQuote(`set -eu; for name in ${QUEUE_DIRECTORIES}; do find "/var/spool/postfix/$name" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; done; tar -x --same-owner -C /var/spool/postfix`)}`, { stdin: true })} < "$tmp/postfix.tar"`, + `${mailEngineCommand(flavor, `sh -c ${shellQuote(`set -eu; ${amavisRoot}; for name in ${amavisDataNames}; do if [ -d "$root/$name" ]; then find "$root/$name" -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +; fi; done; tar -x --same-owner -C "$root"`)}`, { stdin: true })} < "$tmp/amavis.tar"`, + mailEngineCommand(flavor, "postsuper -s"), + ].join("\n") + : "", + options.restoreStatePath ? "" : mailDaemonReloadCommand(flavor), + ] + .filter(Boolean) + .join("\n"); + + return { + preflightCommand: `sudo -n sh -c ${shellQuote(preflight)}`, + payloadKind: "custom_command", + payloadConfig: { + produceCommand: `sudo -n sh -c ${shellQuote(produce)}`, + restoreCommand: `sudo -n sh -c ${shellQuote(restore)}`, + artifactName: "mail.tar.zst", + mail: { version: 2, ...flags }, + }, + }; +} diff --git a/packages/platform/src/engine/modules/mail/backup-state.ts b/packages/platform/src/engine/modules/mail/backup-state.ts new file mode 100644 index 000000000..0913db290 --- /dev/null +++ b/packages/platform/src/engine/modules/mail/backup-state.ts @@ -0,0 +1,97 @@ +import type { Encryption } from "@repo/platform"; +import { decrypt, encrypt } from "../../lib/encryption"; +import type { MailServerState } from "./mail-state"; + +export interface PortableMailState { + version: 1; + encoding: "plaintext"; + state: MailServerState; +} + +function mapCredentials( + state: MailServerState, + transform: (value: string, legacy: boolean) => string, +) { + const result = structuredClone(state); + if (result.platformMailbox) + result.platformMailbox.password = transform(result.platformMailbox.password, true); + for (const mailbox of Object.values(result.testMailboxes ?? {})) + mailbox.password = transform(mailbox.password, true); + if (result.outboundRelay?.passwordEncrypted) + result.outboundRelay.passwordEncrypted = transform( + result.outboundRelay.passwordEncrypted, + false, + ); + return result; +} + +/** The archive carries portable values; no source-instance cipher is copied to a new install. */ +export function exportMailState( + state: MailServerState, + cipher: Pick = { decrypt }, +): PortableMailState { + const portable = mapCredentials(state, (value, legacy) => { + if (!value) return value; + // Old generated mailbox passwords are shorter than an AES-GCM envelope. + // A full-size envelope that cannot be decrypted must fail capture explicitly. + if (legacy && Buffer.from(value, "base64").length < 32) return value; + return cipher.decrypt(value); + }); + // These belong to the source machine, and are never applied on the target. + // Carry only the postmaster password corresponding to the restored account. + portable.secrets = state.secrets.DOMAIN_ADMIN_PASSWD_PLAIN + ? { DOMAIN_ADMIN_PASSWD_PLAIN: state.secrets.DOMAIN_ADMIN_PASSWD_PLAIN } + : {}; + portable.completedSteps = {}; + portable.logs = []; + delete portable.webmail; + return { version: 1, encoding: "plaintext", state: portable }; +} + +/** Keep the destination's engine identity and DB-role credentials, while restoring + * the mailbox/relay credentials that correspond to the restored SQL account hashes. */ +export function importMailState( + portable: PortableMailState, + target: MailServerState, + cipher: Pick = { encrypt }, +): MailServerState { + if ( + portable?.version !== 1 || + portable.encoding !== "plaintext" || + portable.state?.version !== 1 || + portable.state.domain !== target.domain + ) + throw new Error("The mail backup configuration does not match the target's primary domain"); + const restored = mapCredentials(portable.state, (value) => + value ? cipher.encrypt(value) : value, + ); + const { DOMAIN_ADMIN_PASSWD_PLAIN: _oldPostmaster, ...targetSecrets } = target.secrets; + return { + ...restored, + serverId: target.serverId, + engineImage: target.engineImage, + containerName: target.containerName, + startedAt: target.startedAt, + completedSteps: target.completedSteps, + secrets: { + ...targetSecrets, + ...(restored.secrets.DOMAIN_ADMIN_PASSWD_PLAIN + ? { DOMAIN_ADMIN_PASSWD_PLAIN: restored.secrets.DOMAIN_ADMIN_PASSWD_PLAIN } + : {}), + }, + webmail: target.webmail, + dnsRecords: null, + dnsAcknowledged: false, + ptrAcknowledged: false, + additionalDomains: Object.fromEntries( + Object.entries(restored.additionalDomains ?? {}).map(([domain, settings]) => [ + domain, + { ...settings, acknowledgedAt: null }, + ]), + ), + logs: [], + resumeStep: target.resumeStep, + errorMessage: null, + updatedAt: new Date().toISOString(), + }; +} diff --git a/packages/platform/src/engine/modules/mail/mail-domain.ts b/packages/platform/src/engine/modules/mail/mail-domain.ts new file mode 100644 index 000000000..b1db1bc11 --- /dev/null +++ b/packages/platform/src/engine/modules/mail/mail-domain.ts @@ -0,0 +1,8 @@ +/** One domain contract for API input and names recovered from a mail database. */ +export const MAIL_DOMAIN_RE = /^[a-z0-9][a-z0-9-]*(\.[a-z0-9][a-z0-9-]*)+$/i; + +export function validateMailDomain(domain: string): void { + const normalized = domain.trim().toLowerCase(); + if (!MAIL_DOMAIN_RE.test(normalized) || normalized.length > 255) + throw new Error(`Invalid domain: ${domain}`); +} diff --git a/packages/platform/src/engine/modules/mail/mail-engine.ts b/packages/platform/src/engine/modules/mail/mail-engine.ts index a8b7517c9..dba8111ef 100644 --- a/packages/platform/src/engine/modules/mail/mail-engine.ts +++ b/packages/platform/src/engine/modules/mail/mail-engine.ts @@ -44,6 +44,7 @@ import { reportCaughtError as observeCaughtError, diagnostics as errorDiagnostics } from "@repo/core/diagnostics"; import { randomBytes } from "node:crypto"; import { dirname } from "node:path/posix"; +import { setTimeout as delay } from "node:timers/promises"; import { detectMailEngine, @@ -51,6 +52,7 @@ import { MAIL_DB_CONTAINER, MAIL_DB_NAME, MAIL_HOST_PATHS, + MAIL_CONTAINER_MOUNTS, privilegedExecutor, type CommandExecutor, type MailEngineFlavor, @@ -63,6 +65,16 @@ import { sshManager } from "../../lib/ssh-manager"; export type { MailEngineFlavor, MailEngineProbe }; +/** Resolve a mail engine path through the same mount map used by installation. */ +export function mailHostPath(flavor: MailEngineFlavor, path: string): string { + if (flavor === "host") return path; + const mount = MAIL_CONTAINER_MOUNTS.find( + (entry) => path === entry.container || path.startsWith(`${entry.container}/`), + ); + if (!mount) throw new Error(`No mail host mount for ${path}`); + return `${mount.host}${path.slice(mount.container.length)}`; +} + /** Either a server id (we acquire the executor) or an executor already in hand. */ export type MailTarget = string | CommandExecutor; @@ -513,8 +525,8 @@ export function mailPsqlCommand(flavor: MailEngineFlavor, sql: string): string { * Run a command where Postfix/Dovecot and their config live — `postmap`, * `postconf`, `postfix reload`. On a legacy box that's the host itself. */ -export function mailEngineCommand(flavor: MailEngineFlavor, cmd: string): string { - return flavor === "container" ? `docker exec ${MAIL_CONTAINER} ${cmd}` : cmd; +export function mailEngineCommand(flavor: MailEngineFlavor, cmd: string, options: { stdin?: boolean } = {}): string { + return flavor === "container" ? `docker exec${options.stdin ? " -i" : ""} ${MAIL_CONTAINER} ${cmd}` : cmd; } /** @@ -549,15 +561,15 @@ export async function resolveMailFlavor(target: MailTarget): Promise/dev/null 2>&1; then postfix stop; fi`; + return mailEngineCommand(flavor, `sh -ec ${sq( + action === "restart" ? `${stop}\nsupervisorctl start ${sq(unit)}` : stop, + )}`); + } return key === "postgresql" ? `docker ${action} ${sq(MAIL_DB_CONTAINER)}` : `docker exec ${sq(MAIL_CONTAINER)} supervisorctl ${action} ${sq(unit)}`; @@ -786,6 +807,45 @@ export function mailUnitActionCommand( return `systemctl --no-block ${action} ${sq(unit)}`; } +/** A consistent mail snapshot needs its writers stopped, not its PostgreSQL + * process. Reuse the normal unit matrix and prove each transition completed. */ +export async function pauseMailDaemons(exec: CommandExecutor, flavor: MailEngineFlavor) { + const { engineExec } = await resolveMailMutationAccess(exec, flavor, []); + const stopped: string[] = []; + const status = async (unit: string) => parseMailUnitProbe( + flavor, unit, unit, await engineExec.exec(mailUnitProbeCommand(flavor, unit, unit)), + ).status; + const waitFor = async (unit: string, wanted: "active" | "inactive") => { + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + if (await status(unit) === wanted) return; + await delay(100); + } + throw new Error(`Mail ${unit} did not become ${wanted}`); + }; + const resume = async () => { + for (const unit of [...stopped].reverse()) { + await engineExec.exec(mailUnitActionCommand(flavor, unit, unit, "start")); + await waitFor(unit, "active"); + stopped.splice(stopped.indexOf(unit), 1); + } + }; + try { + for (const unit of ["postfix", "dovecot", "amavis", "iredapd", ...(flavor === "host" ? ["amavisd"] : [])]) { + const current = await status(unit); + if (current === "missing" || current === "inactive") continue; + if (current !== "active") throw new Error(`Mail ${unit} has an unknown or unhealthy state; cannot safely capture it`); + stopped.push(unit); + await engineExec.exec(mailUnitActionCommand(flavor, unit, unit, "stop")); + await waitFor(unit, "inactive"); + } + return { resume, get stopped() { return stopped.length > 0; } }; + } catch (error) { + await resume(); + throw error; + } +} + /** * Build fail2ban-client command for inspecting or mutating banned IPs. */ diff --git a/packages/platform/src/engine/modules/mail/mail.service.ts b/packages/platform/src/engine/modules/mail/mail.service.ts index 3a87f8bfa..cb1b2eeaa 100644 --- a/packages/platform/src/engine/modules/mail/mail.service.ts +++ b/packages/platform/src/engine/modules/mail/mail.service.ts @@ -21,6 +21,7 @@ import { reportCaughtError as observeCaughtError } from "@repo/core/diagnostics"; import { randomBytes } from "node:crypto"; +import { validateMailDomain } from "./mail-domain"; import type { CommandExecutor, SystemLogCallback, SystemLog } from "@repo/adapters"; import { checkMailHealth, requiresMailComponent } from "./mail-health.service"; import { @@ -750,7 +751,7 @@ export async function stepDkimKeys( log(6, "info", "Retrieving DKIM keys..."); let rawOutput: string; try { - rawOutput = await exec.exec(`${amavisBin} showkeys 2>&1`); + rawOutput = await exec.exec(`${amavisBin} showkeys ${sq(domain)} 2>&1`); } catch (err) { observeCaughtError(err, "platform/engine/modules/mail/mail.service"); return { stepId: 6, success: false, message: `Failed to retrieve DKIM keys: ${errMsg(err)}` }; @@ -893,7 +894,10 @@ export async function stepDkimKeys( export async function provisionDomainDkim( exec: CommandExecutor, newDomain: string, + options: { reload?: boolean } = {}, ): Promise { + validateMailDomain(newDomain); + newDomain = newDomain.trim().toLowerCase(); // ── Step 1: resolve where amavis lives + which binary it ships ───────── const { bin, run, conf, flavor } = await resolveAmavis(exec); const { hostFiles, engineExec } = await resolveMailMutationAccess(exec, flavor, [ @@ -921,7 +925,7 @@ export async function provisionDomainDkim( } // ── Step 4: reload amavis so the new key is signed with ────────────── - await engineExec.exec(mailUnitActionCommand(flavor, "amavis", "amavis", "restart")).catch((diagnosticFailure) => { + if (options.reload !== false) await engineExec.exec(mailUnitActionCommand(flavor, "amavis", "amavis", "restart")).catch((diagnosticFailure) => { observeCaughtError(diagnosticFailure, "platform/engine/modules/mail/mail.service"); });