diff --git a/.cursor/environment.json b/.cursor/environment.json new file mode 100644 index 000000000..5964ed32d --- /dev/null +++ b/.cursor/environment.json @@ -0,0 +1,20 @@ +{ + "name": "Naruon dev (backend + frontend + Postgres/pgvector)", + "user": "ubuntu", + "install": "bash .cursor/install.sh", + "start": "bash .cursor/start.sh", + "terminals": [ + { + "name": "backend", + "command": "cd backend && . .venv/bin/activate && python scripts/start_backend.py --host 0.0.0.0 --port 8000" + }, + { + "name": "frontend", + "command": "cd frontend && corepack pnpm@11.5.3 dev --port 3000 --hostname 0.0.0.0" + } + ], + "ports": [ + { "name": "backend", "port": 8000 }, + { "name": "frontend", "port": 3000 } + ] +} diff --git a/.cursor/install.sh b/.cursor/install.sh new file mode 100755 index 000000000..61db7ac03 --- /dev/null +++ b/.cursor/install.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Idempotent repository bootstrap for Naruon Cloud Agent environments. +# +# Prepares durable, source-derived state after checkout: +# * system packages (PostgreSQL 16 + pgvector, Python venv/build tooling) +# * the backend virtualenv + pinned Python requirements +# * the frontend pnpm dependency tree +# +# Per-boot service startup (Postgres, schema migrations, dev secrets) lives in +# .cursor/start.sh so it re-runs on every VM boot, including builds. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +echo "==> [install] system packages (postgresql-16, pgvector, python venv, build tools)" +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get install -y -qq \ + postgresql-16 \ + postgresql-16-pgvector \ + postgresql-client-16 \ + python3.12-venv \ + python3.12-dev \ + build-essential + +echo "==> [install] backend virtualenv + requirements" +cd "$REPO_ROOT/backend" +if [ ! -x ".venv/bin/python" ]; then + python3 -m venv .venv +fi +# shellcheck disable=SC1091 +. .venv/bin/activate +python -m pip install --require-hashes -r requirements-hashes.txt + +echo "==> [install] frontend dependencies (pnpm@11.5.3)" +cd "$REPO_ROOT/frontend" +corepack enable +corepack prepare pnpm@11.5.3 --activate +corepack pnpm@11.5.3 install --frozen-lockfile + +echo "==> [install] done" \ No newline at end of file diff --git a/.cursor/start.sh b/.cursor/start.sh new file mode 100755 index 000000000..523051f6a --- /dev/null +++ b/.cursor/start.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Per-boot runtime reconciliation for Naruon Cloud Agent environments. +# +# Runs on every VM start (idempotent): brings up the PostgreSQL cluster, +# materializes a local dev .env with generated secrets on first boot, ensures +# the app database + pgvector extension exist, and applies Alembic migrations. +# +# Dependency installation lives in .cursor/install.sh; this script only +# reconciles per-boot state and then returns so the backend/frontend terminals +# can start. +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +ENV_FILE="$HOME/.env" +PY="$REPO_ROOT/backend/.venv/bin/python" + +echo "==> [start] ensuring PostgreSQL 16 cluster is online" +if ! pg_lsclusters -h 2>/dev/null | awk '{print $4}' | grep -q online; then + sudo pg_ctlcluster 16 main start || true +fi +# Wait for the server to accept connections before touching it. +for _ in $(seq 1 30); do + if sudo -u postgres pg_isready -q; then break; fi + sleep 1 +done +if ! sudo -u postgres pg_isready -q; then + echo "==> [start] PostgreSQL did not become ready" >&2 + exit 1 +fi + +echo "==> [start] generating local dev .env on first boot (secrets are per-VM)" +if [ ! -f "$ENV_FILE" ]; then + "$PY" - "$ENV_FILE" <<'PYGEN' +import os, secrets, sys +from pathlib import Path +from cryptography.fernet import Fernet + +env_path = Path(sys.argv[1]) +db_password = secrets.token_urlsafe(24) +hmac_secret = secrets.token_urlsafe(48) +enc_key = Fernet.generate_key().decode() + +env_path.write_text( + "# Naruon local dev environment (generated per-VM; not committed).\n" + f"DATABASE_URL=postgresql+asyncpg://postgres:{db_password}@127.0.0.1:5432/ai_email\n" + f"AUTH_SESSION_HMAC_SECRET={hmac_secret}\n" + f"ENCRYPTION_KEY={enc_key}\n" + "DEBUG=false\n" + "RUNTIME_ENVIRONMENT=development\n" + "ENABLE_PROMETHEUS_METRICS=false\n" + "ALLOWED_CORS_ORIGINS=http://localhost:3000,http://127.0.0.1:3000," + "http://localhost:8000,http://127.0.0.1:8000\n" + "SMTP_MODE=simulated\n" + "OPENAI_API_KEY=\n" + "OPENAI_EMBEDDING_MODEL=text-embedding-3-small\n" + "OPENAI_MODEL=gpt-4o\n", + encoding="utf-8", +) +os.chmod(env_path, 0o600) +print(f"wrote {env_path}") +PYGEN +fi + +echo "==> [start] reconciling database role, database, and pgvector extension" +# Keep the local postgres role secret in sync with DATABASE_URL without +# interpolating it into SQL or the process argument list. +"$PY" "$REPO_ROOT/backend/scripts/reconcile_local_postgres_role.py" --env-file "$ENV_FILE" +if ! sudo -u postgres psql -tAc "SELECT 1 FROM pg_database WHERE datname='ai_email'" | grep -q 1; then + sudo -u postgres createdb ai_email +fi +sudo -u postgres psql -d ai_email -v ON_ERROR_STOP=1 \ + -c "CREATE EXTENSION IF NOT EXISTS vector;" >/dev/null + +echo "==> [start] applying database migrations (alembic upgrade head)" +cd "$REPO_ROOT/backend" +"$PY" scripts/migrate_db.py + +echo "==> [start] done" diff --git a/AGENTS.md b/AGENTS.md index 9104dd1f4..a1f8b8c69 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -348,6 +348,12 @@ in this repo. embedding generation is unavailable. Tests must cover the local `embeddinggemma` path so Data workspace imports do not silently bypass the selected embedding model. +- PostgreSQL `text` / `hashtext()` cannot encode a NUL (`0x00`) octet. Do not + pass a raw `user_id\\x00organization_id` (or any other NUL-separated payload) + as a `pg_advisory_lock` bind value. Derive a NUL-free digest such as + SHA-256 hex and keep mocked/SQLite tests from asserting the raw NUL form — + those dialects skip the lock and hide `CharacterNotInRepertoireError` until + a real Postgres import. - Home/Today dashboard reply-wait surfaces must read signed `/api/emails/pending-replies` data instead of inferring pending replies from generic inbox fixtures or static copy. Tests and E2E mocks must verify the diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9d2cbba18..9e27c7ed1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -145,6 +145,12 @@ remains a local/dev compatibility path that creates the `vector` extension, metadata-defined tables for fresh local databases, and idempotent backfills for existing local databases. +Cloud Agent VMs use the same managed migration path through +`.cursor/start.sh` → `scripts/migrate_db.py`. Local role-secret alignment is +stdin dollar-quoted SQL (`scripts/reconcile_local_postgres_role.py`); do not +interpolate `DATABASE_URL` secrets into `psql -c`. See +[`docs/development/cloud-agent-environment.md`](docs/development/cloud-agent-environment.md). + ## Send boundary Outbound replies preserve `In-Reply-To` and `References` headers in the built diff --git a/CHANGELOG.md b/CHANGELOG.md index f31c701a5..1cb7130da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ ## [Unreleased] +- Cloud Agent `.cursor/start.sh`는 `DATABASE_URL` 역할 비밀을 `psql -c`에 보간하지 않고 `scripts/reconcile_local_postgres_role.py`의 dollar-quoted stdin으로 맞춥니다. `install.sh`는 `requirements-hashes.txt`를 `--require-hashes`로 설치합니다. 신선한 DB에서 폐기된 `emails` 테이블을 가정하던 `0011_email_read_state`는 테이블이 없으면 no-op이고, `0019_email_record_read_state`가 `0018_email_send_rate_buckets` 다음에 정식 `email_records.is_read DEFAULT true`를 가드로 맞춥니다. 가져오기 quota advisory lock 키는 NUL 없는 SHA-256 hex입니다. - EmailDetail 테스트가 지원하지 않는 스레드 병합/분리 버튼을 `textContent`뿐 아니라 `aria-label`과 `title` 접근 가능 이름으로도 검출하도록 바꿔, 아이콘 전용 버튼 회귀를 놓치지 않습니다. ### 캘린더 충돌 (Status-weighted conflicts) diff --git a/CLAUDE.md b/CLAUDE.md index be67bc80c..f68a237f6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,6 +98,17 @@ The other compose files are purpose-specific evaluation/evidence stacks: defaults anywhere; Compose/Kubernetes/operators must inject them, and startup fails closed when they are missing. +### Cloud Agent environment (`.cursor/`) + +Repo-managed Cloud Agent bootstrap lives in `.cursor/environment.json`. +`install.sh` installs PostgreSQL 16/pgvector and hashed Python deps; +`start.sh` brings Postgres online, mints a per-VM `~/.env`, and runs +`scripts/migrate_db.py`. Role-secret sync goes through +`backend/scripts/reconcile_local_postgres_role.py` (dollar-quoted stdin, never +`psql -c` interpolation). After boot, open http://127.0.0.1:3000 and sign in +with a minted HMAC session — see +`docs/development/cloud-agent-environment.md`. + ## Architecture Naruon is an AI email workspace: a web client/control plane over diff --git a/backend/alembic/versions/0011_email_read_state.py b/backend/alembic/versions/0011_email_read_state.py index 716590cd1..bbe4490f9 100644 --- a/backend/alembic/versions/0011_email_read_state.py +++ b/backend/alembic/versions/0011_email_read_state.py @@ -14,6 +14,19 @@ def upgrade() -> None: + # ``emails`` was the pre-reconciliation email table. ``email_records`` is now + # the single source of truth and already declares ``is_read`` via the model + # metadata created in 0001, so this legacy column add only applies to older + # databases that still carry the retired ``emails`` table. Guard on the table + # existing (matching the has_table/has_column pattern used by later + # revisions) so ``alembic upgrade head`` succeeds on fresh databases. + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table("emails"): + return + existing_columns = {column["name"] for column in inspector.get_columns("emails")} + if "is_read" in existing_columns: + return op.add_column( "emails", sa.Column( @@ -26,4 +39,7 @@ def upgrade() -> None: def downgrade() -> None: - op.drop_column("emails", "is_read") + # No-op: this revision only reconciles a retired ``emails`` table. Dropping + # ``is_read`` when the column is present would also remove a pre-existing + # column this revision did not create. + return diff --git a/backend/alembic/versions/0019_email_record_read_state.py b/backend/alembic/versions/0019_email_record_read_state.py new file mode 100644 index 000000000..6e35cbb82 --- /dev/null +++ b/backend/alembic/versions/0019_email_record_read_state.py @@ -0,0 +1,62 @@ +"""Guard email_records.is_read with NOT NULL DEFAULT true. + +Revision ID: 0019_email_record_read_state +Revises: 0018_email_send_rate_buckets + +``0011_email_read_state`` only mutates the retired ``emails`` table. Fresh +databases already receive ``email_records.is_read`` from ``0001`` +``create_all`` plus the current model ``server_default``. Existing databases +whose ``email_records`` row predates that column (or lacks a server default) +still need a guarded additive revision so raw INSERTs that omit ``is_read`` +succeed. +""" + +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa + +revision = "0019_email_record_read_state" +down_revision = "0018_email_send_rate_buckets" +branch_labels = None +depends_on = None + +_EMAIL_RECORDS_TABLE = "email_records" +_READ_STATE_COLUMN = "is_read" + + +def upgrade() -> None: + bind = op.get_bind() + inspector = sa.inspect(bind) + if not inspector.has_table(_EMAIL_RECORDS_TABLE): + return + columns = { + column["name"]: column + for column in inspector.get_columns(_EMAIL_RECORDS_TABLE) + } + if _READ_STATE_COLUMN not in columns: + op.add_column( + _EMAIL_RECORDS_TABLE, + sa.Column( + _READ_STATE_COLUMN, + sa.Boolean(), + nullable=False, + server_default=sa.text("true"), + ), + ) + return + if columns[_READ_STATE_COLUMN].get("default") is None: + op.alter_column( + _EMAIL_RECORDS_TABLE, + _READ_STATE_COLUMN, + existing_type=sa.Boolean(), + existing_nullable=False, + server_default=sa.text("true"), + ) + + +def downgrade() -> None: + # No-op: ``create_all`` and later model metadata may already own this + # column. Dropping it would remove a default this revision did not + # necessarily create. + return diff --git a/backend/db/models.py b/backend/db/models.py index 7380defb8..40db8ea00 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -18,6 +18,7 @@ Text, UniqueConstraint, func, + text, ) from sqlalchemy.orm import declarative_base, Mapped, mapped_column, relationship from sqlalchemy.types import TypeDecorator @@ -803,7 +804,12 @@ def owner_filters(cls, user_id: str, organization_id: str | None): date: Mapped[datetime.datetime] = mapped_column(DateTime(timezone=True), index=True) body: Mapped[str] = mapped_column(Text) # IMAP \Seen read state; defaults read so historical/file imports don't nag. - is_read: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + # A DB-level server_default keeps create_all/bootstrap_db consistent with the + # 0011_email_read_state migration intent so raw inserts that omit is_read + # (e.g. postgres smoke seeds) don't hit a NOT NULL violation. + is_read: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=True, server_default=text("true") + ) # Defer large pgvector payloads on default entity loads. embedding = mapped_column(Vector(1536), deferred=True) attachments: Mapped[list["Attachment"]] = relationship( diff --git a/backend/scripts/bootstrap_db.py b/backend/scripts/bootstrap_db.py index 1047103e8..3e579a053 100644 --- a/backend/scripts/bootstrap_db.py +++ b/backend/scripts/bootstrap_db.py @@ -186,10 +186,6 @@ def _get_create_indexes_statements() -> list[Executable]: "CREATE INDEX IF NOT EXISTS ix_email_records_owner_date " "ON email_records (user_id, organization_id, date)" ), - text( - "CREATE INDEX IF NOT EXISTS ix_emails_owner_date " - "ON emails (user_id, organization_id, date)" - ), text( "CREATE INDEX IF NOT EXISTS ix_sender_relationships_owner_source " "ON sender_relationships " diff --git a/backend/scripts/reconcile_local_postgres_role.py b/backend/scripts/reconcile_local_postgres_role.py new file mode 100644 index 000000000..1c652baad --- /dev/null +++ b/backend/scripts/reconcile_local_postgres_role.py @@ -0,0 +1,82 @@ +"""Reconcile the local Cloud Agent Postgres role secret without SQL interpolation. + +Cloud Agent ``start.sh`` keeps the ``postgres`` role aligned with the +generated ``DATABASE_URL``. The secret must travel only on ``psql`` stdin as +dollar-quoted SQL so a quote or backslash in an existing ``~/.env`` cannot +break out of ``ALTER USER ... PASSWORD`` and does not appear on the process +command line. +""" + +from __future__ import annotations + +import argparse +import secrets +import subprocess # nosec B404 -- fixed executable argv, shell=False, secret on stdin. +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any +from urllib.parse import unquote, urlsplit + +INCOMPLETE_LOCAL_DATABASE_CONFIG = "local database configuration is incomplete" +_ROLE_NAME = "postgres" + + +def database_url_password(env_text: str) -> str: + """Return the ``DATABASE_URL`` user secret, or fail closed when it is absent.""" + for line in env_text.splitlines(): + if line.startswith("DATABASE_URL="): + raw_url = line.split("=", 1)[1].strip() + secret = unquote(urlsplit(raw_url).password or "") + if not secret: + raise ValueError(INCOMPLETE_LOCAL_DATABASE_CONFIG) + return secret + raise ValueError(INCOMPLETE_LOCAL_DATABASE_CONFIG) + + +def build_alter_role_sql(secret: str, *, role_name: str = _ROLE_NAME) -> str: + """Build ``ALTER USER`` SQL that dollar-quotes ``secret``. + + The tag is regenerated until it is absent from the secret so the closer + cannot appear inside the quoted value. + """ + if not secret: + raise ValueError(INCOMPLETE_LOCAL_DATABASE_CONFIG) + tag = f"naruon{secrets.token_hex(16)}" + while tag in secret: + tag = f"naruon{secrets.token_hex(16)}" + return f"ALTER USER {role_name} WITH PASSWORD ${tag}${secret}${tag};\n" + + +def reconcile_local_postgres_role( + secret: str, + *, + runner: Callable[..., Any] = subprocess.run, +) -> None: + """Apply the role secret through ``psql`` stdin, never ``psql -c``.""" + sql = build_alter_role_sql(secret) + runner( + ["sudo", "-u", "postgres", "psql", "-v", "ON_ERROR_STOP=1"], + input=sql, + text=True, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def main(argv: list[str] | None = None) -> int: + """Read an env file and align the local ``postgres`` role secret.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--env-file", + required=True, + help="Path to the local env file that already contains DATABASE_URL.", + ) + args = parser.parse_args(argv) + env_text = Path(args.env_file).read_text(encoding="utf-8") + reconcile_local_postgres_role(database_url_password(env_text)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/backend/services/email_import_service.py b/backend/services/email_import_service.py index 1ff9a2bb3..5df9a5f41 100644 --- a/backend/services/email_import_service.py +++ b/backend/services/email_import_service.py @@ -243,6 +243,27 @@ def _session_uses_postgresql(session: AsyncSession) -> bool: return getattr(getattr(bind, "dialect", None), "name", None) == "postgresql" +def _owner_import_quota_lock_key(user_id: str, organization_id: str) -> str: + """Return a NUL-free SHA-256 digest for ``pg_advisory_lock(hashtext(...))``. + + PostgreSQL ``text`` cannot store a 0x00 octet, so passing + ``f"{user_id}\\x00{organization_id}"`` to ``hashtext()`` raises + ``CharacterNotInRepertoireError`` on real Postgres (PostgreSQL Global + Development Group, n.d.). Hash the NUL-separated payload instead and bind + only the hex digest. + + References + ---------- + PostgreSQL Global Development Group. (n.d.). *Character set support*. + PostgreSQL Documentation. + https://www.postgresql.org/docs/current/multibyte.html + """ + payload = "\x00".join((user_id, organization_id)) + return hashlib.sha256( + payload.encode("utf-8", errors="surrogatepass") + ).hexdigest() + + async def _acquire_owner_import_quota_lock( session: AsyncSession, *, user_id: str, organization_id: str ) -> bool: @@ -250,7 +271,7 @@ async def _acquire_owner_import_quota_lock( return False lock_params = { "namespace_key": EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE, - "owner_key": f"{user_id}\x00{organization_id}", + "owner_key": _owner_import_quota_lock_key(user_id, organization_id), } await session.execute( select( @@ -269,7 +290,7 @@ async def _release_owner_import_quota_lock( ) -> None: lock_params = { "namespace_key": EMAIL_IMPORT_QUOTA_LOCK_NAMESPACE, - "owner_key": f"{user_id}\x00{organization_id}", + "owner_key": _owner_import_quota_lock_key(user_id, organization_id), } await session.execute( select( diff --git a/backend/tests/test_alembic_migrations.py b/backend/tests/test_alembic_migrations.py index f8f3ffeae..f286b7259 100644 --- a/backend/tests/test_alembic_migrations.py +++ b/backend/tests/test_alembic_migrations.py @@ -438,6 +438,43 @@ def test_merge_revision_reconciles_newsdom_provider_branch(): assert "op.drop_column(" not in revision_text +def test_email_read_state_revision_is_retired_emails_table_only() -> None: + revision_path = BACKEND_ROOT / "alembic" / "versions" / "0011_email_read_state.py" + revision_text = revision_path.read_text() + + assert 'revision = "0011_email_read_state"' in revision_text + assert 'down_revision = "0009_project_graph_projection"' in revision_text + assert 'has_table("emails")' in revision_text + assert 'op.add_column(\n "emails"' in revision_text + assert 'op.add_column(\n "email_records"' not in revision_text + # Downgrade must not drop a pre-existing emails.is_read this revision did + # not create. Retired-table reconciliation is a documented no-op. + assert "op.drop_column(" not in revision_text + + +def test_email_record_read_state_revision_guards_canonical_table() -> None: + revision_path = ( + BACKEND_ROOT / "alembic" / "versions" / "0019_email_record_read_state.py" + ) + assert revision_path.exists() + revision_text = revision_path.read_text() + + assert 'revision = "0019_email_record_read_state"' in revision_text + assert 'down_revision = "0018_email_send_rate_buckets"' in revision_text + assert len("0019_email_record_read_state") <= 32 + assert '_EMAIL_RECORDS_TABLE = "email_records"' in revision_text + assert "has_table(_EMAIL_RECORDS_TABLE)" in revision_text + assert "_READ_STATE_COLUMN" in revision_text + assert '"is_read"' in revision_text + assert "server_default=sa.text(\"true\")" in revision_text or ( + 'server_default=sa.text("true")' in revision_text + ) + assert "op.add_column(" in revision_text + assert "sa.text(f" not in revision_text + # Additive default only; do not drop a column create_all may already own. + assert "op.drop_column(" not in revision_text + + def test_merge_revision_reconciles_newsdom_document_and_carddav_heads(): revision_path = ( BACKEND_ROOT / "alembic" / "versions" / "0017_merge_newsdom_carddav_heads.py" diff --git a/backend/tests/test_bootstrap_db.py b/backend/tests/test_bootstrap_db.py index 5af0540f0..07e0919c8 100644 --- a/backend/tests/test_bootstrap_db.py +++ b/backend/tests/test_bootstrap_db.py @@ -138,6 +138,8 @@ def test_schema_backfill_adds_email_indexes(monkeypatch): "create index if not exists ix_email_records_date" in statement for statement in statements ) + assert all(" on emails " not in statement for statement in statements) + assert all("ix_emails_owner_date" not in statement for statement in statements) def test_schema_backfill_adds_llm_provider_columns_and_indexes(monkeypatch): diff --git a/backend/tests/test_cloud_agent_environment.py b/backend/tests/test_cloud_agent_environment.py new file mode 100644 index 000000000..a6c0a0b26 --- /dev/null +++ b/backend/tests/test_cloud_agent_environment.py @@ -0,0 +1,38 @@ +"""Source contracts for the repo-managed Cloud Agent environment scripts.""" + +from __future__ import annotations + +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +INSTALL_SH = (REPO_ROOT / ".cursor" / "install.sh").read_text(encoding="utf-8") +START_SH = (REPO_ROOT / ".cursor" / "start.sh").read_text(encoding="utf-8") +ENVIRONMENT_JSON = (REPO_ROOT / ".cursor" / "environment.json").read_text( + encoding="utf-8" +) + + +def test_install_sh_pins_python_requirements_with_hashes() -> None: + assert "requirements-hashes.txt" in INSTALL_SH + assert "--require-hashes" in INSTALL_SH + assert "pip install -r requirements.txt" not in INSTALL_SH + assert "pip install --upgrade pip" not in INSTALL_SH + + +def test_start_sh_does_not_interpolate_role_secret_into_sql() -> None: + assert "PASSWORD '${DB_PASSWORD}'" not in START_SH + assert "ALTER USER postgres WITH PASSWORD '" not in START_SH + assert "reconcile_local_postgres_role.py" in START_SH + + +def test_start_sh_fails_closed_when_postgres_never_becomes_ready() -> None: + assert "pg_isready" in START_SH + assert "did not become ready" in START_SH + assert "exit 1" in START_SH + + +def test_environment_json_keeps_servers_in_terminals_not_install() -> None: + assert "bash .cursor/install.sh" in ENVIRONMENT_JSON + assert "bash .cursor/start.sh" in ENVIRONMENT_JSON + assert "scripts/start_backend.py" in ENVIRONMENT_JSON + assert "corepack pnpm@11.5.3 dev" in ENVIRONMENT_JSON \ No newline at end of file diff --git a/backend/tests/test_email_import_quota_lock_key.py b/backend/tests/test_email_import_quota_lock_key.py new file mode 100644 index 000000000..856b16d1b --- /dev/null +++ b/backend/tests/test_email_import_quota_lock_key.py @@ -0,0 +1,36 @@ +"""Contract tests for the Postgres import-quota advisory-lock owner key. + +PostgreSQL ``text`` / ``hashtext()`` cannot store a NUL (0x00) octet +(PostgreSQL Global Development Group, n.d.). The production helper must +therefore emit a NUL-free digest. The expected digest is computed here with +the standard-library hasher so a helper rewrite cannot silently change the +on-disk lock identity. +""" + +from __future__ import annotations + +import hashlib + +from services.email_import_service import _owner_import_quota_lock_key + +# Independent reference: SHA-256 of UTF-8 ``user_id + NUL + organization_id``. +_GOLDEN_TESTUSER_ORG_ACME = "3fbc5671f32a1608f88c1775c1008c26c53faaea0308b97c556eeceb2b4bb8d3" + + +def test_owner_import_quota_lock_key_matches_independent_sha256_digest() -> None: + independent = hashlib.sha256(b"testuser\x00org-acme").hexdigest() + assert independent == _GOLDEN_TESTUSER_ORG_ACME + assert _owner_import_quota_lock_key("testuser", "org-acme") == independent + assert "\x00" not in independent + + +def test_owner_import_quota_lock_key_is_nul_free_for_unicode_owners() -> None: + user_id = "유저" + organization_id = "org-서울" + independent = hashlib.sha256( + f"{user_id}\x00{organization_id}".encode("utf-8") + ).hexdigest() + actual = _owner_import_quota_lock_key(user_id, organization_id) + assert actual == independent + assert "\x00" not in actual + assert len(actual) == 64 diff --git a/backend/tests/test_emails_api.py b/backend/tests/test_emails_api.py index 3c06363b7..c0678696a 100644 --- a/backend/tests/test_emails_api.py +++ b/backend/tests/test_emails_api.py @@ -22,6 +22,7 @@ import datetime from unittest.mock import AsyncMock, patch from services.embedding import STORAGE_EMBEDDING_DIMENSION +from services.email_import_service import _owner_import_quota_lock_key from services.email_service import generate_email_fingerprint from services.email_send_rate_limiter import ( EmailSendRateLimitDecision, @@ -1281,14 +1282,18 @@ async def test_import_email_files_serializes_quota_with_postgres_owner_lock( assert "pg_advisory_unlock" in advisory_queries[-1] assert "hashtext(:namespace_key)" in advisory_queries[0] assert ":owner_key" in advisory_queries[0] + # PostgreSQL text/hashtext cannot encode NUL (0x00); the owner key must be a + # NUL-free digest, not a raw ``user\x00org`` separator. + expected_owner_key = _owner_import_quota_lock_key("testuser", "org-acme") + assert "\x00" not in expected_owner_key assert advisory_query_params(session) == [ { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, ] @@ -1344,14 +1349,16 @@ async def test_import_email_files_rejects_when_owner_quota_is_exhausted( advisory_queries = advisory_query_texts(session) assert "pg_advisory_lock" in advisory_queries[0] assert "pg_advisory_unlock" in advisory_queries[-1] + expected_owner_key = _owner_import_quota_lock_key("testuser", "org-acme") + assert "\x00" not in expected_owner_key assert advisory_query_params(session) == [ { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, { "namespace_key": "naruon-email-import-quota", - "owner_key": "testuser\x00org-acme", + "owner_key": expected_owner_key, }, ] diff --git a/backend/tests/test_reconcile_local_postgres_role.py b/backend/tests/test_reconcile_local_postgres_role.py new file mode 100644 index 000000000..2a31bfa46 --- /dev/null +++ b/backend/tests/test_reconcile_local_postgres_role.py @@ -0,0 +1,66 @@ +"""Tests for Cloud Agent local-Postgres role password reconciliation. + +The start path must reject an empty role secret and must never interpolate +that secret into a ``psql -c`` SQL string or process argv. +""" + +from __future__ import annotations + +import pytest + +from scripts.reconcile_local_postgres_role import ( + build_alter_role_sql, + database_url_password, + reconcile_local_postgres_role, +) + + +def test_database_url_password_rejects_empty_secret() -> None: + with pytest.raises(ValueError, match="local database configuration is incomplete"): + database_url_password("DATABASE_URL=postgresql+asyncpg://postgres@127.0.0.1:5432/ai_email\n") + + +def test_database_url_password_rejects_missing_url() -> None: + with pytest.raises(ValueError, match="local database configuration is incomplete"): + database_url_password("DEBUG=false\n") + + +def test_database_url_password_unquotes_url_encoded_secret() -> None: + assert ( + database_url_password( + "DATABASE_URL=postgresql+asyncpg://postgres:a%27b@127.0.0.1:5432/ai_email\n" + ) + == "a'b" + ) + + +def test_build_alter_role_sql_dollar_quotes_metacharacters() -> None: + secret = "x'; DROP ROLE postgres; --" + sql = build_alter_role_sql(secret) + assert "DROP ROLE postgres" in sql + assert "PASSWORD '" not in sql + assert sql.startswith("ALTER USER postgres WITH PASSWORD $") + assert sql.endswith(";\n") or sql.endswith(";") + + +def test_reconcile_local_postgres_role_keeps_secret_off_argv() -> None: + captured: dict[str, object] = {} + + def fake_run(argv, **kwargs): + captured["argv"] = list(argv) + captured["input"] = kwargs.get("input") + captured["check"] = kwargs.get("check") + + class _Completed: + returncode = 0 + + return _Completed() + + secret = "quote'me;and\\back" + reconcile_local_postgres_role(secret, runner=fake_run) + argv = captured["argv"] + assert isinstance(argv, list) + assert all(secret not in str(part) for part in argv) + assert "-c" not in argv + assert secret in str(captured["input"]) + assert captured["check"] is True diff --git a/docs/development/cloud-agent-environment.md b/docs/development/cloud-agent-environment.md new file mode 100644 index 000000000..ede732910 --- /dev/null +++ b/docs/development/cloud-agent-environment.md @@ -0,0 +1,71 @@ +# Cloud Agent development environment + +Use this when a Cloud Agent or a fresh Ubuntu workstation needs a running +Naruon control plane (FastAPI + Next.js + PostgreSQL 16/pgvector) without +Docker Compose. + +## Next action + +1. Confirm `.cursor/environment.json` is the environment source for the VM. +2. After `start.sh` exits, open http://127.0.0.1:3000 (frontend) and + http://127.0.0.1:8000/ (backend health). +3. Mint a member HMAC session against the generated `AUTH_SESSION_HMAC_SECRET` + in `~/.env`, then import `.eml` fixtures through + `POST /api/emails/import-files`. +4. If Postgres is down, re-run `bash .cursor/start.sh` — do not edit secrets + into SQL by hand. + +## Install versus start + +| Script | Lifetime | What to put here | +| --- | --- | --- | +| `.cursor/install.sh` | Durable, source-derived | `postgresql-16` + pgvector, hashed `requirements-hashes.txt`, `pnpm@11.5.3 --frozen-lockfile` | +| `.cursor/start.sh` | Every boot | Postgres cluster, per-VM `~/.env`, `ai_email` + `vector`, `alembic upgrade head` | +| `environment.json` terminals | Long-running | `scripts/start_backend.py` and `pnpm dev` | + +`install.sh` must use `python -m pip install --require-hashes -r requirements-hashes.txt`. +Unhashed `requirements.txt` is not an acceptable Cloud Agent supply-chain path. + +## Secret handling + +`start.sh` writes `~/.env` only when the file is missing. Generated values use +`secrets.token_urlsafe(48)` for `AUTH_SESSION_HMAC_SECRET` and +`Fernet.generate_key()` for `ENCRYPTION_KEY`, matching +`validate_auth_session_hmac_secret_value`. + +The `postgres` role secret is applied by +`backend/scripts/reconcile_local_postgres_role.py`: + +- empty `DATABASE_URL` user secrets fail closed +- the secret is dollar-quoted on `psql` stdin +- the secret never appears in `psql -c` or process argv + +Do not mount this `~/.env` as a Compose `env_file`. Compose interpolation still +resolves `NARUON_ENV_FILE` > `~/.env` > `./.env` without leaking the file into +the container environment wholesale. + +## Schema contract on a fresh database + +`0001_initial_control_plane` runs `Base.metadata.create_all`, so +`email_records.is_read` is present with `DEFAULT true` on a current model. +`0011_email_read_state` only touches the retired `emails` table and is a no-op +when that table is absent. `0019_email_record_read_state` follows the shared +`0018_email_send_rate_buckets` migration and adds or defaults `email_records.is_read` +on older databases that already have `email_records` but lack the column or its +server default. + +## Import quota lock + +Owner import serialization uses `pg_advisory_lock(hashtext(namespace), hashtext(owner_key))`. +PostgreSQL `text` cannot store a NUL octet, so the owner key is a SHA-256 hex +digest of `user_id + 0x00 + organization_id`, not the raw NUL-separated string +(PostgreSQL Global Development Group, n.d.). + +## References + +PostgreSQL Global Development Group. (n.d.). *Character set support*. +PostgreSQL Documentation. https://www.postgresql.org/docs/current/multibyte.html + +PostgreSQL Global Development Group. (n.d.). *psql — PostgreSQL interactive +terminal*. PostgreSQL Documentation. +https://www.postgresql.org/docs/current/app-psql.html