diff --git a/.github/workflows/dev-workflow-tests.yml b/.github/workflows/dev-workflow-tests.yml new file mode 100644 index 000000000..7a4f8261f --- /dev/null +++ b/.github/workflows/dev-workflow-tests.yml @@ -0,0 +1,60 @@ +name: Dev Workflow Tests + +on: + pull_request: + paths: + - "start-dev.sh" + - "stop-dev.sh" + - "scripts/setup-worktree.sh" + - "scripts/dev-infra.sh" + - "scripts/dev-infra.test.sh" + - "packages/plugins/lib/meiliIndexName.ts" + - "packages/plugins/lib/meiliIndexName.test.ts" + - "packages/plugins/search-meilisearch/**" + - "packages/plugins/vectorstore-meilisearch/**" + - "package.json" + - ".github/workflows/dev-workflow-tests.yml" + push: + branches: + - main + paths: + - "start-dev.sh" + - "stop-dev.sh" + - "scripts/setup-worktree.sh" + - "scripts/dev-infra.sh" + - "scripts/dev-infra.test.sh" + - "packages/plugins/lib/meiliIndexName.ts" + - "packages/plugins/lib/meiliIndexName.test.ts" + - "packages/plugins/search-meilisearch/**" + - "packages/plugins/vectorstore-meilisearch/**" + - "package.json" + - ".github/workflows/dev-workflow-tests.yml" + +permissions: + contents: read + +jobs: + shared-dev-infra: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + + - name: Setup + uses: ./tooling/github/setup + + - name: Check Bash syntax + run: | + for script in start-dev.sh stop-dev.sh scripts/setup-worktree.sh scripts/dev-infra.sh scripts/dev-infra.test.sh; do + bash -n "$script" + done + + - name: Run shared dev infrastructure tests + run: bash scripts/dev-infra.test.sh + + - name: Run Meilisearch namespace tests + run: pnpm --filter @karakeep/plugins test --run + + - name: Typecheck plugins + run: pnpm --filter @karakeep/plugins typecheck diff --git a/AGENTS.md b/AGENTS.md index 8f4eb4643..f5f2e8ccc 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,8 +15,12 @@ - Monorepo: Next.js, React, TypeScript, Hono, tRPC, Drizzle, SQLite, Meilisearch, pnpm, and Turborepo. - Runtime: Node 24 through `mise exec node@24 --`; pnpm 11.2.1 through Corepack. - Install with `pnpm install`, create the documented `.env` symlinks, then run `pnpm db:migrate`. -- Start local development with `pnpm dev:start`. Use `pnpm dev:start -d` for detached mode and `pnpm dev:stop` to stop it. +- Start local development with `pnpm dev:start`. Use `pnpm dev:start -d` for detached mode and `pnpm dev:stop` to stop only that workspace. +- Shared local infrastructure is machine-level: one Meilisearch at `http://localhost:7700` and one Chrome/CDP at `http://localhost:9222`. Manage it explicitly with `pnpm dev:infra:up`, `pnpm dev:infra:status`, and `pnpm dev:infra:down`. +- Parallel worktrees keep separate SQLite/assets data and unique web ports. `scripts/setup-worktree.sh` assigns each worktree a unique `MEILI_INDEX_PREFIX`; both `bookmarks` and `bookmarks_vectors` use that namespace on the shared Meilisearch server. +- `pnpm dev:start` defaults the main workspace namespace to `main_`. Outside the fork dev launcher, an unset `MEILI_INDEX_PREFIX` preserves the original `bookmarks` and `bookmarks_vectors` index names. - Run focused checks before broad checks when practical. Standard checks are `pnpm format:fix`, `pnpm lint`, `pnpm typecheck`, and `pnpm test`. +- Validate shared-dev shell behavior with `bash scripts/dev-infra.test.sh`. - Root `.env` is canonical. Do not print or commit secrets. - Guided Docker self-hosting: `docs/docs/02-installation/11-guided-docker-setup.md`. @@ -114,21 +118,41 @@ pnpm dev:start Useful variants: - `pnpm dev:start` - foreground - `pnpm dev:start -d` - detached -- `pnpm dev:stop` - stop detached services - -What it does: -- runs `web` + `workers` natively -- runs Meilisearch + headless Chrome in Docker +- `pnpm dev:stop` - stop only this workspace's web/workers processes +- `pnpm dev:infra:up` - explicitly start/reuse shared Meilisearch + Chrome +- `pnpm dev:infra:status` - inspect shared dev infrastructure +- `pnpm dev:infra:down` - explicitly remove shared containers while preserving Meilisearch data + +Local-dev ownership model: +- `web` + `workers` run natively per workspace +- one machine-level Meilisearch container is shared at `http://localhost:7700` +- one machine-level Chrome container is shared at `http://localhost:9222` +- `pnpm dev:start` automatically ensures those shared containers exist +- `pnpm dev:stop` never stops shared infrastructure because other worktrees may still use it +- the shared Chrome image is `ghcr.io/karakeep-app/karakeep-chrome:release` + +Parallel-worktree isolation: +- every worktree keeps its own `.data/local` SQLite/assets state and unique web port +- `scripts/setup-worktree.sh` points all worktrees at shared Meilisearch/Chrome endpoints +- every worktree receives a safe unique `MEILI_INDEX_PREFIX` derived from its normalized workspace name plus `WT_PORT_BASE` +- both `bookmarks` and `bookmarks_vectors` use that prefix, so separate SQLite states never share Meilisearch documents +- `pnpm dev:start` defaults the main workspace prefix to `main_` +- outside this fork's dev launcher, unset `MEILI_INDEX_PREFIX` preserves the original `bookmarks` / `bookmarks_vectors` names ### Direct commands +When bypassing `pnpm dev:start`, manual starts **must** set an explicit unique `MEILI_INDEX_PREFIX` for that workspace before starting web or workers. Use `main_` only for the main workspace; parallel worktrees need distinct prefixes. + ```bash +export MEILI_INDEX_PREFIX=main_ +pnpm dev:infra:up pnpm web pnpm workers ``` Notes: - Meilisearch and headless Chrome are optional for booting the app, but required for full search/crawling behavior. +- shared infra binds only to localhost; if ports `7700` or `9222` are occupied by something else, the helper fails rather than silently reusing an unknown service - If `next dev` crashes with a stale Turbopack/instrumentation issue, clear `apps/web/.next`. ### Pull prod state to local dev @@ -145,7 +169,7 @@ Optional root `.env` keys: - `KARAKEEP_PROD_COMPOSE_SERVICE` - `KARAKEEP_PROD_EXPORT_IMAGE` -Every pull restores the full `/data` volume because SQLite rows can reference stored assets. Do not use DB-only pulls or print `.env` secrets. +Every pull restores the full `/data` volume because SQLite rows can reference stored assets. Do not use DB-only pulls or print `.env` secrets. Meilisearch remains derived local state in that workspace's own index namespace. ## Deploy model for this fork @@ -175,6 +199,7 @@ Additional tooling used in this fork: - `pnpm doctor` - React health scan via react.doctor - `pnpm doctor:staged` - staged-file React scan - `bash scripts/install.test.sh` - guided installer shell-level validation +- `bash scripts/dev-infra.test.sh` - shared worktree-dev infrastructure validation Notes: - `react.doctor` is advisory in pre-commit and can emit noisy temp-package errors. @@ -209,6 +234,10 @@ pnpm test pnpm knip pnpm doctor bash scripts/install.test.sh +bash scripts/dev-infra.test.sh +pnpm dev:infra:up +pnpm dev:infra:status +pnpm dev:infra:down pnpm db:generate --name pnpm db:migrate pnpm web diff --git a/docs/fork-setup.md b/docs/fork-setup.md index a637e2fc1..7c5c5d165 100644 --- a/docs/fork-setup.md +++ b/docs/fork-setup.md @@ -48,24 +48,73 @@ pnpm dev:start Variants: - `pnpm dev:start` — foreground - `pnpm dev:start -d` — detached (`.dev/` logs, shell returns immediately) -- `pnpm dev:stop` — stop detached services +- `pnpm dev:stop` — stop only this workspace's detached web/workers processes +- `pnpm dev:infra:status` — show the shared Chrome/Meilisearch container status +- `pnpm dev:infra:up` — explicitly start/reuse shared Chrome + Meilisearch +- `pnpm dev:infra:down` — explicitly remove the shared containers while preserving the Meilisearch data volume + +`pnpm dev:start` automatically ensures the machine-level dev infrastructure is running, then starts only this workspace's native processes: -What this starts: - `web` - `workers` -- Meilisearch in Docker -- headless Chrome in Docker + +The machine-level infrastructure is shared across all local worktrees: + +- `karakeep-dev-meilisearch` on `127.0.0.1:7700` +- `karakeep-dev-chrome` on `127.0.0.1:9222` + +The Chrome helper uses `ghcr.io/karakeep-app/karakeep-chrome:release`, which is published for both `linux/amd64` and `linux/arm64`. The Meilisearch container uses the named volume `karakeep-dev-meilisearch-data`, which survives `pnpm dev:infra:down`. + +`pnpm dev:stop` never stops the shared containers. This is intentional: another worktree may still be using them. + +### Parallel worktrees + +Worktrees share the physical Chrome and Meilisearch containers, but application state stays isolated. + +`scripts/setup-worktree.sh` keeps these values per worktree: + +- unique `KARAKEEP_PORT` +- unique `DATA_DIR` (`/.data/local`) +- unique `API_URL` / `NEXTAUTH_URL` +- unique `MEILI_INDEX_PREFIX` + +Every generated worktree points at the same local infrastructure endpoints: + +```text +MEILI_ADDR=http://localhost:7700 +BROWSER_WEB_URL=http://localhost:9222 +``` + +The Meilisearch plugins prepend `MEILI_INDEX_PREFIX` to both index UIDs. For example, a worktree prefix `issue-123-7_` produces: + +```text +issue-123-7_bookmarks +issue-123-7_bookmarks_vectors +``` + +The generated prefix is based on the normalized worktree name plus `WT_PORT_BASE`, so two configured worktrees do not share search/vector state even though they use the same Meilisearch server. + +The main workspace uses `main_` when `pnpm dev:start` does not find an explicit `MEILI_INDEX_PREFIX`. When `MEILI_INDEX_PREFIX` is entirely unset outside this fork's dev launcher, the plugins retain the original production-compatible index names `bookmarks` and `bookmarks_vectors`. + +Do not share a worktree's `.data/local` directory with another worktree. SQLite rows and stored assets are authoritative per workspace; Meilisearch remains derived state and can be rebuilt into that workspace's namespace. ### Direct/manual start +If you intentionally bypass `pnpm dev:start`, manual starts **must** set an explicit unique `MEILI_INDEX_PREFIX` for that workspace. Use `main_` only for the main workspace; parallel worktrees need distinct prefixes. + ```bash +export MEILI_INDEX_PREFIX=main_ +pnpm dev:infra:up pnpm web pnpm workers ``` +Direct commands do not synthesize a namespace for you. Leaving the prefix unset selects the backward-compatible unprefixed indexes and can mix search/vector state when multiple manual worktrees share the same Meilisearch server. + Notes: - Meilisearch and headless Chrome are optional for booting the app, but required for full search/crawling behavior. - If `next dev` crashes with a stale Turbopack / `instrumentation.ts` parse issue, clear `apps/web/.next` and restart. +- If port `7700` is occupied by something other than `karakeep-dev-meilisearch`, or port `9222` by something other than `karakeep-dev-chrome`, `pnpm dev:infra:up` fails instead of silently reusing an unknown service. ### Verify the offline iPhone PWA @@ -86,7 +135,9 @@ The root `.env` is the source of truth, but several processes load `.env` from t The most important variables for local development are: - `DATA_DIR` - `NEXTAUTH_SECRET` -- `MEILI_ADDR` (if search should work) +- `MEILI_ADDR` (shared dev default: `http://localhost:7700`) +- `MEILI_INDEX_PREFIX` (per-worktree search/vector namespace; empty remains backward-compatible outside the dev launcher) +- `BROWSER_WEB_URL` (shared dev default: `http://localhost:9222`) - `OPENAI_API_KEY` (if AI tagging/summarization should work) ### Pull production state into local development @@ -110,6 +161,8 @@ Optional root `.env` keys: - `KARAKEEP_PROD_COMPOSE_SERVICE` - `KARAKEEP_PROD_EXPORT_IMAGE` +A production-state pull still populates only that workspace's SQLite/assets state. Its local search/vector data belongs to the workspace's own `MEILI_INDEX_PREFIX` namespace in the shared local Meilisearch container. + ## CI Primary workflow: @@ -125,8 +178,9 @@ It runs: Fork-specific notes: - this fork does **not** use Turbo remote cache - some CI jobs reclaim disk space before heavy steps because typecheck/tests can otherwise exhaust hosted-runner storage -- CI reads the exact Node runtime from `.nvmrc`; keep the temporary 24.18.1 pin until the Node v24 cleanup-hook fix tracked in [nodejs/node#65042](https://github.com/nodejs/node/pull/65042) ships in a usable Node 24 release +- local development and production use Node 24.18.1 from `.nvmrc`; the combined CI `tests` job temporarily overrides setup to Node 22.21.1 because Vitest + `better-sqlite3` can abort during Node 24 worker teardown; remove that override once the Node fix tracked in [nodejs/node#65042](https://github.com/nodejs/node/pull/65042) ships in a usable Node 24 release - `knip` and `react-doctor` run as **non-blocking** report jobs +- `.github/workflows/dev-workflow-tests.yml` validates the shared local-dev Bash lifecycle and Meilisearch namespace behavior when relevant files change ## Extra quality tooling diff --git a/docs/superpowers/plans/2026-08-15-shared-dev-infra.md b/docs/superpowers/plans/2026-08-15-shared-dev-infra.md new file mode 100644 index 000000000..0596a38ef --- /dev/null +++ b/docs/superpowers/plans/2026-08-15-shared-dev-infra.md @@ -0,0 +1,258 @@ +# Shared Local Dev Infrastructure Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Run one shared Chrome container and one shared Meilisearch container for all local Karakeep worktrees while keeping SQLite/assets and Meilisearch indexes isolated per worktree. + +**Architecture:** Add an optional `MEILI_INDEX_PREFIX` consumed by both Meilisearch plugins, then move Chrome/Meilisearch ownership from each workspace into a machine-level `scripts/dev-infra.sh` helper. Worktree setup keeps unique web/data state but points every workspace to localhost ports 7700/9222 and generates a stable per-worktree index prefix. `pnpm dev:start` ensures shared infra exists; `pnpm dev:stop` stops only current workspace processes. + +**Tech Stack:** Bash, pnpm, Docker, Meilisearch, Karakeep Chrome, TypeScript, Zod, Vitest, GitHub Actions. + +## Global Constraints + +- Shared dev endpoints are `http://localhost:7700` for Meilisearch and `http://localhost:9222` for Chrome/CDP. +- Shared containers are machine-level and must not be stopped by an individual worktree. +- SQLite/assets remain isolated in each workspace's `.data/local`. +- `MEILI_INDEX_PREFIX` defaults to an empty string so production/installer/E2E behavior remains unchanged. +- Worktree prefixes must be safe Meilisearch UID components and unique across simultaneously configured worktrees. +- Use `ghcr.io/karakeep-app/karakeep-chrome:release`, not the retired GCR Alpine Chrome image. +- Shared infra ports must fail closed when occupied by a foreign process/container. +- No automatic garbage collection of old worktree indexes in this change. + +--- + +### Task 1: Meilisearch index namespacing + +**Files:** +- Create: `packages/plugins/lib/meiliIndexName.ts` +- Create: `packages/plugins/lib/meiliIndexName.test.ts` +- Modify: `packages/plugins/search-meilisearch/src/env.ts` +- Modify: `packages/plugins/search-meilisearch/src/index.ts` +- Modify: `packages/plugins/vectorstore-meilisearch/src/env.ts` +- Modify: `packages/plugins/vectorstore-meilisearch/src/index.ts` + +**Interfaces:** +- Produces: `buildMeiliIndexName(baseName: string, prefix?: string): string` +- Consumes: optional `MEILI_INDEX_PREFIX` environment variable. + +- [ ] **Step 1: Write failing unit tests** + +```ts +import { describe, expect, it } from "vitest"; +import { buildMeiliIndexName } from "./meiliIndexName"; + +describe("buildMeiliIndexName", () => { + it("keeps existing index names when no prefix is configured", () => { + expect(buildMeiliIndexName("bookmarks")).toBe("bookmarks"); + expect(buildMeiliIndexName("bookmarks_vectors", "")).toBe("bookmarks_vectors"); + }); + + it("prefixes search and vector indexes consistently", () => { + expect(buildMeiliIndexName("bookmarks", "issue-123_")) + .toBe("issue-123_bookmarks"); + expect(buildMeiliIndexName("bookmarks_vectors", "issue-123_")) + .toBe("issue-123_bookmarks_vectors"); + }); +}); +``` + +- [ ] **Step 2: Run plugin tests and confirm RED** + +Run: `pnpm --filter @karakeep/plugins test --run` +Expected: FAIL because `./meiliIndexName` does not exist. + +- [ ] **Step 3: Implement the helper and environment parsing** + +```ts +export function buildMeiliIndexName(baseName: string, prefix = ""): string { + return `${prefix}${baseName}`; +} +``` + +Add `MEILI_INDEX_PREFIX` with an empty default to both Meilisearch env parsers and construct provider index names through the helper. + +- [ ] **Step 4: Run plugin tests/typecheck** + +Run: `pnpm --filter @karakeep/plugins test --run && pnpm --filter @karakeep/plugins typecheck` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/plugins +git commit -m "feat: namespace Meilisearch indexes" +``` + +### Task 2: Shared machine-level development infrastructure + +**Files:** +- Create: `scripts/dev-infra.sh` +- Create: `scripts/dev-infra.test.sh` +- Create: `.github/workflows/dev-workflow-tests.yml` + +**Interfaces:** +- Produces: `scripts/dev-infra.sh up|status|down` +- Owns containers `karakeep-dev-meilisearch` and `karakeep-dev-chrome`. + +- [ ] **Step 1: Write failing shell tests** + +Tests use a fake `docker`/port probe and assert: +- `up` creates the two stable containers with Meili on 7700 and Chrome on 9222; +- repeated `up` reuses owned containers; +- foreign occupancy of either port fails with an actionable error; +- `down` removes only the two shared infra containers; +- Chrome image is `ghcr.io/karakeep-app/karakeep-chrome:release`; +- Bash syntax is valid. + +- [ ] **Step 2: Run shell tests and confirm RED** + +Run: `bash scripts/dev-infra.test.sh` +Expected: FAIL because `scripts/dev-infra.sh` does not exist. + +- [ ] **Step 3: Implement `dev-infra.sh`** + +The helper must: +- require Docker and daemon access; +- bind only localhost (`127.0.0.1:7700:7700`, `127.0.0.1:9222:9222`); +- use one persistent named Meili volume; +- recognize only its stable container names as owned endpoints; +- fail rather than assume compatibility when a port is occupied by anything else; +- make `up` idempotent and `down` explicit. + +- [ ] **Step 4: Run shell tests** + +Run: `bash -n scripts/dev-infra.sh && bash scripts/dev-infra.test.sh` +Expected: PASS. + +- [ ] **Step 5: Add path-scoped GitHub Actions validation** + +The workflow runs Bash syntax plus `scripts/dev-infra.test.sh` when shared-dev scripts/workflow change. + +- [ ] **Step 6: Commit** + +```bash +git add scripts/dev-infra.sh scripts/dev-infra.test.sh .github/workflows/dev-workflow-tests.yml +git commit -m "feat: add shared local dev infrastructure" +``` + +### Task 3: Worktree/start/stop integration + +**Files:** +- Modify: `scripts/setup-worktree.sh` +- Modify: `start-dev.sh` +- Modify: `stop-dev.sh` +- Modify: `package.json` +- Extend: `scripts/dev-infra.test.sh` + +**Interfaces:** +- Consumes: `scripts/dev-infra.sh up|status|down`. +- Produces package commands `dev:infra:up`, `dev:infra:status`, `dev:infra:down`. + +- [ ] **Step 1: Add failing integration assertions** + +Assert generated worktree `.env` contains: + +```text +MEILI_ADDR=http://localhost:7700 +BROWSER_WEB_URL=http://localhost:9222 +MEILI_INDEX_PREFIX=-_ +``` + +and no longer derives Meili/Chrome ports from `WT_PORT_BASE`. Assert `start-dev.sh` delegates infra startup and `stop-dev.sh` contains no shared-container stop/remove operation. + +- [ ] **Step 2: Run shell tests and confirm RED** + +Run: `bash scripts/dev-infra.test.sh` +Expected: FAIL against current per-worktree infrastructure behavior. + +- [ ] **Step 3: Update worktree setup** + +Keep unique web port/data URL behavior, write shared endpoints, and derive a sanitized prefix from `WT_WORKSPACE_NAME` plus `WT_PORT_BASE`. The main workspace defaults to `main_` when no generated prefix exists. + +- [ ] **Step 4: Update start/stop lifecycle** + +`start-dev.sh` invokes `scripts/dev-infra.sh up`, then starts only web/workers and migrations. `stop-dev.sh` kills only current workspace processes and prints how to stop shared infra explicitly. + +- [ ] **Step 5: Add package scripts** + +```json +"dev:infra:up": "bash scripts/dev-infra.sh up", +"dev:infra:status": "bash scripts/dev-infra.sh status", +"dev:infra:down": "bash scripts/dev-infra.sh down" +``` + +- [ ] **Step 6: Run shell tests and syntax checks** + +Run: `bash -n start-dev.sh stop-dev.sh scripts/setup-worktree.sh scripts/dev-infra.sh && bash scripts/dev-infra.test.sh` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add package.json start-dev.sh stop-dev.sh scripts/setup-worktree.sh scripts/dev-infra.test.sh +git commit -m "feat: reuse dev infrastructure across worktrees" +``` + +### Task 4: Documentation and assistant guidance + +**Files:** +- Modify: `docs/fork-setup.md` +- Modify: `AGENTS.md` +- Modify: PR description after implementation. + +**Interfaces:** +- Documents the exact commands and lifecycle implemented in Tasks 1-3. + +- [ ] **Step 1: Update fork developer docs** + +Document shared infra architecture, `dev:infra:*` commands, automatic infra startup from `dev:start`, per-worktree SQLite/assets isolation, `MEILI_INDEX_PREFIX`, and the fact that `dev:stop` leaves shared infra running. + +- [ ] **Step 2: Update AGENTS.md** + +Keep the assistant-facing local-dev summary aligned with `docs/fork-setup.md`; do not change production installer guidance. + +- [ ] **Step 3: Check CLAUDE/GEMINI representation** + +If they are references/symlinks to `AGENTS.md`, do not duplicate edits. If independent copies, update them consistently. + +- [ ] **Step 4: Commit** + +```bash +git add docs/fork-setup.md AGENTS.md +git commit -m "docs: explain shared worktree dev infrastructure" +``` + +### Task 5: Full validation and PR readiness + +**Files:** +- Review all PR #32 changed files. + +- [ ] **Step 1: Run focused validation** + +```bash +pnpm --filter @karakeep/plugins test --run +pnpm --filter @karakeep/plugins typecheck +bash -n start-dev.sh stop-dev.sh scripts/setup-worktree.sh scripts/dev-infra.sh +bash scripts/dev-infra.test.sh +``` + +- [ ] **Step 2: Run repository quality checks** + +```bash +pnpm format +pnpm lint +pnpm typecheck +pnpm test +``` + +- [ ] **Step 3: Inspect PR diff for scope/secrets** + +Confirm production/installer behavior stays unchanged, no credentials are present, and only local-dev Meili indexes are namespaced when the prefix is set. + +- [ ] **Step 4: Update PR #32 description and mark ready** + +Summarize architecture, validation, and backward compatibility. Keep base branch `main`. + +- [ ] **Step 5: Inspect current GitHub Actions** + +Do not call the PR complete until the current head's required CI and new dev-workflow tests are green, or report the exact blocker from logs. diff --git a/docs/superpowers/specs/2026-08-14-shared-dev-infra-design.md b/docs/superpowers/specs/2026-08-14-shared-dev-infra-design.md new file mode 100644 index 000000000..c56aca47a --- /dev/null +++ b/docs/superpowers/specs/2026-08-14-shared-dev-infra-design.md @@ -0,0 +1,264 @@ +# Shared Local Dev Infrastructure Across Worktrees + +## Goal + +Reduce duplicated local development infrastructure when multiple Karakeep worktrees run at the same time, while preserving per-worktree application/data isolation. + +The desired local model is: + +- one shared headless Chrome container per developer machine +- one shared Meilisearch container per developer machine +- one web process, workers process, SQLite/data directory, and Meilisearch index namespace per worktree +- stopping one worktree must not stop infrastructure still used by other worktrees + +This design is local-development-specific. It does not change the guided production installer or production service-isolation guidance. + +## Current behavior + +`start-dev.sh` currently starts a worktree-specific Meilisearch container and Chrome container, with ports derived from the worktree environment. `scripts/setup-worktree.sh` assigns unique web, Meilisearch, and Chrome ports to each worktree. `stop-dev.sh` stops and removes the worktree-specific infrastructure containers. + +Application data is already isolated per worktree through `.data/local`. + +The Meilisearch plugins currently use fixed index UIDs: + +- search: `bookmarks` +- vector store: `bookmarks_vectors` + +Because different worktrees have different SQLite state, multiple worktrees cannot safely share one Meilisearch server unless those index UIDs are namespaced. + +## Approaches considered + +### 1. Keep dedicated infrastructure per worktree + +This preserves strong isolation and requires no application changes, but every worktree consumes an additional Chrome and Meilisearch container. It scales poorly for parallel worktree development and keeps duplicated local state that is not valuable. + +### 2. Share Chrome and Meilisearch without namespacing + +This minimizes container count, but is unsafe. Every worktree would write to the same `bookmarks` and `bookmarks_vectors` indexes while using a different SQLite database. Reindexing, deletions, settings updates, and vector data would cross worktree boundaries. + +### 3. Share physical infrastructure and namespace Meilisearch indexes + +**Selected approach.** Chrome is naturally shareable for local development because each worker creates browser contexts within the shared browser process. Meilisearch is shared at the server/container level while each worktree receives distinct index UIDs. SQLite/assets remain worktree-local. + +This gives the resource benefit of shared infrastructure without mixing application state. + +## Architecture + +The machine-level infrastructure owns two stable endpoints: + +- Chrome/CDP: `http://localhost:9222` +- Meilisearch: `http://localhost:7700` + +Each worktree receives: + +- a unique web port, as today +- `BROWSER_WEB_URL=http://localhost:9222` +- `MEILI_ADDR=http://localhost:7700` +- a unique `MEILI_INDEX_PREFIX` +- its own `.data/local` directory + +Example: + +```text +shared dev infra +├── Chrome :9222 +└── Meilisearch :7700 + ├── main_bookmarks + ├── main_bookmarks_vectors + ├── issue-123-1_bookmarks + ├── issue-123-1_bookmarks_vectors + ├── feature-x-2_bookmarks + └── feature-x-2_bookmarks_vectors + +worktree main +├── web :3000 +├── workers +└── .data/local + +worktree issue-123 +├── web :3001 +├── workers +└── .data/local +``` + +## Meilisearch index namespace + +Introduce an optional environment variable: + +```text +MEILI_INDEX_PREFIX +``` + +Default: empty string. + +This preserves all existing production/upstream-compatible behavior when unset: + +```text +bookmarks +bookmarks_vectors +``` + +When set to `issue-123-1_`, the plugins use: + +```text +issue-123-1_bookmarks +issue-123-1_bookmarks_vectors +``` + +Both the search Meilisearch plugin and vector-store Meilisearch plugin must consume the same prefix source. + +The prefix is an index namespace only. It does not change `MEILI_ADDR`, credentials, or server-level configuration. + +### Worktree prefix generation + +`scripts/setup-worktree.sh` derives a stable safe slug from the worktree identity and combines it with `WT_PORT_BASE`, which is already unique per configured worktree. It writes: + +```text +MEILI_INDEX_PREFIX=-_ +``` + +Using both values prevents collisions when two workspace names normalize to the same slug. Unsupported characters are normalized to `-`; secrets and absolute paths are never included. + +The main workspace uses a stable `main_` prefix so it cannot collide with worktrees while sharing the same server. `start-dev.sh` supplies `main_` only when `MEILI_INDEX_PREFIX` is otherwise unset, so an explicit developer override remains possible. + +## Shared infrastructure lifecycle + +Add a dedicated machine-level helper, `scripts/dev-infra.sh`, and expose it through package scripts: + +```bash +pnpm dev:infra:up +pnpm dev:infra:status +pnpm dev:infra:down +``` + +The helper owns stable container names: + +```text +karakeep-dev-meilisearch +karakeep-dev-chrome +``` + +It starts: + +- `getmeili/meilisearch:v1.41.0` on host port `7700` +- `ghcr.io/karakeep-app/karakeep-chrome:release` on host port `9222` + +Chrome must use the maintained Karakeep image/configuration rather than the retired `gcr.io/zenika-hub/alpine-chrome:124` image. + +The shared Meilisearch container keeps one machine-level Docker volume so index state survives restarts. Individual worktree indexes remain logically isolated within it. + +`dev:infra:down` is an explicit machine-level action. Individual worktree stop commands must never call it automatically. + +## `start-dev.sh` behavior + +`pnpm dev:start` remains the preferred developer entry point. + +Before starting web/workers, it should: + +1. verify Docker and pnpm prerequisites as today +2. ensure shared dev infrastructure is running, starting it through `scripts/dev-infra.sh up` when needed +3. use the workspace's `MEILI_ADDR`, `BROWSER_WEB_URL`, and `MEILI_INDEX_PREFIX` +4. for the main workspace only, default an unset `MEILI_INDEX_PREFIX` to `main_` +5. run migrations against that workspace's isolated data directory +6. start only that workspace's web and workers processes + +It must not create worktree-specific Chrome or Meilisearch containers. + +The shared endpoints default to `localhost:7700` and `localhost:9222`. Explicit `.env` overrides remain possible, but when the default shared endpoints are used they must be owned by the shared-infra helper rather than an unrelated process. + +## `stop-dev.sh` behavior + +`pnpm dev:stop` stops only the current workspace's web/workers processes and removes their pidfiles. + +It must not stop or remove shared Chrome or Meilisearch. + +The output should tell the developer that shared infrastructure remains running and can be stopped explicitly with: + +```bash +pnpm dev:infra:down +``` + +## Worktree setup behavior + +`scripts/setup-worktree.sh` continues to isolate: + +- web port +- `DATA_DIR` +- `API_URL` +- `NEXTAUTH_URL` + +It no longer allocates per-worktree Meilisearch or Chrome ports. Every generated worktree `.env` points to: + +```text +MEILI_ADDR=http://localhost:7700 +BROWSER_WEB_URL=http://localhost:9222 +MEILI_INDEX_PREFIX=-_ +``` + +Production-state pulls remain per-worktree because the SQLite/assets directory remains isolated. A pulled production snapshot does not imply sharing local Meilisearch indexes; the worktree's own index can be rebuilt from its local application state. + +## Port/conflict handling + +The shared infra helper must fail clearly if ports `7700` or `9222` are already occupied by a process/container it does not own instead of silently assuming compatibility. + +If the expected Karakeep shared container already owns the port, `up` is idempotent and reuses it. + +This avoids accidentally connecting development worktrees to an unrelated local Meilisearch or CDP endpoint. + +## Backward compatibility + +`MEILI_INDEX_PREFIX` is optional and defaults to empty, so production, E2E, installer-generated deployments, and users who do not use the fork's worktree helpers keep the existing `bookmarks` / `bookmarks_vectors` names. + +The shared-infra behavior is limited to this fork's local development scripts and docs. + +## Tests + +Add focused regression coverage for: + +1. search plugin index UID defaults to `bookmarks` when no prefix is set +2. search plugin applies `MEILI_INDEX_PREFIX` +3. vector plugin defaults to `bookmarks_vectors` +4. vector plugin applies the same prefix +5. worktree setup produces shared Meili/Chrome endpoints and a unique index prefix while retaining isolated web/data values +6. `start-dev.sh` delegates shared infra startup instead of creating per-worktree infra containers +7. `stop-dev.sh` does not stop/remove shared infrastructure +8. shared infra helper is idempotent for its own containers and rejects foreign port conflicts +9. Bash syntax validation for modified shell scripts + +Where existing test structure makes direct shell invocation awkward, extract small pure shell helpers rather than weakening assertions. + +## Documentation + +Update the canonical fork-development documentation: + +- `docs/fork-setup.md` +- `AGENTS.md` + +Document: + +- two shared infra containers per machine regardless of worktree count +- per-worktree SQLite/assets and Meilisearch index namespaces +- `pnpm dev:start` / `pnpm dev:stop` lifecycle +- `pnpm dev:infra:up`, `pnpm dev:infra:status`, and `pnpm dev:infra:down` +- `MEILI_INDEX_PREFIX` and its backward-compatible default + +Update `CLAUDE.md` / `GEMINI.md` only if they are independent copies rather than references to `AGENTS.md`. + +## Non-goals + +- sharing SQLite or asset directories between worktrees +- sharing one un-namespaced Meilisearch index between worktrees +- changing production or guided-installer service ownership +- introducing automatic garbage collection of old worktree indexes in this PR +- exposing Chrome/Meilisearch beyond localhost +- adding orchestration beyond the existing Bash/pnpm developer workflow + +## Success criteria + +With three simultaneous local worktrees: + +- exactly one shared Chrome container and one shared Meilisearch container are needed +- each worktree has an independent web port and SQLite/assets state +- each worktree uses distinct Meilisearch search/vector indexes +- stopping one worktree leaves the other worktrees and shared infra running +- normal deployments remain unaffected when `MEILI_INDEX_PREFIX` is unset diff --git a/package.json b/package.json index d6cd850d1..3a2bdce6d 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,9 @@ "dev": "turbo --no-daemon dev --parallel", "dev:start": "bash start-dev.sh", "dev:stop": "bash stop-dev.sh", + "dev:infra:up": "bash scripts/dev-infra.sh up", + "dev:infra:status": "bash scripts/dev-infra.sh status", + "dev:infra:down": "bash scripts/dev-infra.sh down", "assets:marka": "node scripts/generate-marka-assets.mjs", "test:marka-assets": "node --test scripts/generate-marka-assets.test.mjs", "clean": "git clean -xdf node_modules", diff --git a/packages/plugins/lib/meiliIndexName.test.ts b/packages/plugins/lib/meiliIndexName.test.ts new file mode 100644 index 000000000..9f4083577 --- /dev/null +++ b/packages/plugins/lib/meiliIndexName.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { buildMeiliIndexName } from "./meiliIndexName"; + +describe("buildMeiliIndexName", () => { + it("keeps existing index names when no prefix is configured", () => { + expect(buildMeiliIndexName("bookmarks")).toBe("bookmarks"); + expect(buildMeiliIndexName("bookmarks_vectors", "")).toBe( + "bookmarks_vectors", + ); + }); + + it("prefixes search and vector indexes consistently", () => { + expect(buildMeiliIndexName("bookmarks", "issue-123_")).toBe( + "issue-123_bookmarks", + ); + expect(buildMeiliIndexName("bookmarks_vectors", "issue-123_")).toBe( + "issue-123_bookmarks_vectors", + ); + }); +}); diff --git a/packages/plugins/lib/meiliIndexName.ts b/packages/plugins/lib/meiliIndexName.ts new file mode 100644 index 000000000..83190c346 --- /dev/null +++ b/packages/plugins/lib/meiliIndexName.ts @@ -0,0 +1,3 @@ +export function buildMeiliIndexName(baseName: string, prefix = ""): string { + return `${prefix}${baseName}`; +} diff --git a/packages/plugins/search-meilisearch/src/env.ts b/packages/plugins/search-meilisearch/src/env.ts index 508009503..e9dd63b1a 100644 --- a/packages/plugins/search-meilisearch/src/env.ts +++ b/packages/plugins/search-meilisearch/src/env.ts @@ -4,6 +4,10 @@ export const envConfig = z .object({ MEILI_ADDR: z.string().optional(), MEILI_MASTER_KEY: z.string().default(""), + MEILI_INDEX_PREFIX: z + .string() + .regex(/^[A-Za-z0-9_-]*$/) + .default(""), MEILI_BATCH_SIZE: z.coerce.number().int().positive().default(50), MEILI_BATCH_TIMEOUT_MS: z.coerce.number().int().positive().default(500), }) diff --git a/packages/plugins/search-meilisearch/src/index.ts b/packages/plugins/search-meilisearch/src/index.ts index e66533b39..f1da2538e 100644 --- a/packages/plugins/search-meilisearch/src/index.ts +++ b/packages/plugins/search-meilisearch/src/index.ts @@ -14,6 +14,7 @@ import { PluginProvider } from "@karakeep/shared/plugins"; import { envConfig } from "./env"; import { BatchingDocumentQueue } from "../../lib/batchingDocumentQueue"; +import { buildMeiliIndexName } from "../../lib/meiliIndexName"; function filterToMeiliSearchFilter(filter: FilterQuery): string { switch (filter.type) { @@ -123,7 +124,10 @@ export class MeiliSearchProvider implements PluginProvider { private client: Meilisearch | undefined; private indexClient: SearchIndexClient | undefined; private initPromise: Promise | undefined; - private readonly indexName = "bookmarks"; + private readonly indexName = buildMeiliIndexName( + "bookmarks", + envConfig.MEILI_INDEX_PREFIX, + ); constructor() { if (MeiliSearchProvider.isConfigured()) { diff --git a/packages/plugins/vectorstore-meilisearch/src/env.ts b/packages/plugins/vectorstore-meilisearch/src/env.ts index dc8215fd5..13b37f022 100644 --- a/packages/plugins/vectorstore-meilisearch/src/env.ts +++ b/packages/plugins/vectorstore-meilisearch/src/env.ts @@ -4,6 +4,10 @@ const rawConfig = z .object({ MEILI_ADDR: z.string().optional(), MEILI_MASTER_KEY: z.string().optional(), + MEILI_INDEX_PREFIX: z + .string() + .regex(/^[A-Za-z0-9_-]*$/) + .default(""), // Dedicated Meilisearch instance for the vector store. When unset, the // vector store falls back to the main (search) Meilisearch instance. MEILI_VECTOR_ADDR: z.string().optional(), @@ -17,6 +21,7 @@ export const envConfig = { MEILI_ADDR: rawConfig.MEILI_VECTOR_ADDR ?? rawConfig.MEILI_ADDR, MEILI_MASTER_KEY: rawConfig.MEILI_VECTOR_MASTER_KEY ?? rawConfig.MEILI_MASTER_KEY ?? "", + MEILI_INDEX_PREFIX: rawConfig.MEILI_INDEX_PREFIX, MEILI_BATCH_SIZE: rawConfig.MEILI_BATCH_SIZE, MEILI_BATCH_TIMEOUT_MS: rawConfig.MEILI_BATCH_TIMEOUT_MS, }; diff --git a/packages/plugins/vectorstore-meilisearch/src/index.ts b/packages/plugins/vectorstore-meilisearch/src/index.ts index 68602c517..1d8fc8308 100644 --- a/packages/plugins/vectorstore-meilisearch/src/index.ts +++ b/packages/plugins/vectorstore-meilisearch/src/index.ts @@ -14,6 +14,7 @@ import { PluginProvider } from "@karakeep/shared/plugins"; import { envConfig } from "./env"; import { BatchingDocumentQueue } from "../../lib/batchingDocumentQueue"; +import { buildMeiliIndexName } from "../../lib/meiliIndexName"; // Meilisearch document type that includes the vector field interface MeiliVectorDocument { @@ -80,7 +81,7 @@ class MeiliSearchVectorClient implements VectorStoreClient { const result = await this.index.search("", { vector: options.vector, hybrid: { - semanticRatio: 1.0, // Pure vector search + semanticRatio: 1.0, embedder: "default", }, filter: options.filter?.map((f) => filterToMeiliSearchFilter(f)), @@ -150,7 +151,10 @@ export class MeiliSearchVectorProvider implements PluginProvider | undefined; - private readonly indexName = "bookmarks_vectors"; + private readonly indexName = buildMeiliIndexName( + "bookmarks_vectors", + envConfig.MEILI_INDEX_PREFIX, + ); constructor() { if (MeiliSearchVectorProvider.isConfigured()) { @@ -238,7 +242,6 @@ export class MeiliSearchVectorProvider implements PluginProvider %s\n' "$*"; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +require_docker() { + command -v docker >/dev/null 2>&1 || die "Docker is not installed. Install Docker first." + docker info >/dev/null 2>&1 || die "Docker is installed but the current user cannot reach the Docker daemon. Start Docker and retry." +} + +container_exists() { + docker inspect "$1" >/dev/null 2>&1 +} + +container_running() { + [[ "$(docker inspect -f '{{.State.Running}}' "$1" 2>/dev/null || true)" == "true" ]] +} + +port_in_use() { + local port="$1" + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP:"$port" -sTCP:LISTEN >/dev/null 2>&1 + return + fi + if command -v nc >/dev/null 2>&1; then + nc -z 127.0.0.1 "$port" >/dev/null 2>&1 + return + fi + return 1 +} + +ensure_available_port() { + local port="$1" owner="$2" + if port_in_use "$port"; then + die "Port $port is already in use by something other than $owner. Stop the conflicting service before starting shared Karakeep dev infrastructure." + fi +} + +ensure_meilisearch() { + if container_exists "$MEILI_CONTAINER"; then + if container_running "$MEILI_CONTAINER"; then + info "Reusing shared Meilisearch on http://localhost:$MEILI_PORT" + return + fi + ensure_available_port "$MEILI_PORT" "$MEILI_CONTAINER" + docker start "$MEILI_CONTAINER" >/dev/null || die "Failed to start existing $MEILI_CONTAINER container." + info "Started existing shared Meilisearch on http://localhost:$MEILI_PORT" + return + fi + + ensure_available_port "$MEILI_PORT" "$MEILI_CONTAINER" + docker run -d \ + --name "$MEILI_CONTAINER" \ + --restart unless-stopped \ + -p "127.0.0.1:$MEILI_PORT:7700" \ + -e MEILI_NO_ANALYTICS=true \ + -v "$MEILI_VOLUME:/meili_data" \ + "$MEILI_IMAGE" >/dev/null || die "Failed to start shared Meilisearch. Check whether port $MEILI_PORT became occupied and retry." + info "Started shared Meilisearch on http://localhost:$MEILI_PORT" +} + +ensure_chrome() { + if container_exists "$CHROME_CONTAINER"; then + if container_running "$CHROME_CONTAINER"; then + info "Reusing shared Chrome on http://localhost:$CHROME_PORT" + return + fi + ensure_available_port "$CHROME_PORT" "$CHROME_CONTAINER" + docker start "$CHROME_CONTAINER" >/dev/null || die "Failed to start existing $CHROME_CONTAINER container." + info "Started existing shared Chrome on http://localhost:$CHROME_PORT" + return + fi + + ensure_available_port "$CHROME_PORT" "$CHROME_CONTAINER" + docker run -d \ + --name "$CHROME_CONTAINER" \ + --restart unless-stopped \ + --init \ + -p "127.0.0.1:$CHROME_PORT:9222" \ + "$CHROME_IMAGE" \ + --disable-gpu \ + --disable-dev-shm-usage \ + --hide-scrollbars \ + --disable-blink-features=AutomationControlled \ + --window-size=1440,900 >/dev/null || die "Failed to start shared Chrome. Check whether port $CHROME_PORT became occupied and retry." + info "Started shared Chrome on http://localhost:$CHROME_PORT" +} + +up() { + require_docker + ensure_meilisearch + ensure_chrome +} + +status_one() { + local name="$1" endpoint="$2" + if ! container_exists "$name"; then + printf '%-28s %s\n' "$name" "not created" + elif container_running "$name"; then + printf '%-28s running %s\n' "$name" "$endpoint" + else + printf '%-28s %s\n' "$name" "stopped" + fi +} + +status() { + require_docker + status_one "$MEILI_CONTAINER" "http://localhost:$MEILI_PORT" + status_one "$CHROME_CONTAINER" "http://localhost:$CHROME_PORT" +} + +down() { + require_docker + local removed=0 + if container_exists "$MEILI_CONTAINER"; then + docker rm -f "$MEILI_CONTAINER" >/dev/null + removed=1 + fi + if container_exists "$CHROME_CONTAINER"; then + docker rm -f "$CHROME_CONTAINER" >/dev/null + removed=1 + fi + if ((removed)); then + info "Stopped shared Karakeep dev infrastructure. Meilisearch data volume $MEILI_VOLUME was preserved." + else + info "Shared Karakeep dev infrastructure is not running." + fi +} + +usage() { + cat <<'EOF' +Usage: scripts/dev-infra.sh up|status|down + + up Start or reuse the shared local Meilisearch and Chrome containers. + status Show whether the shared containers are running. + down Remove the shared containers while preserving the Meilisearch volume. +EOF +} + +case "${1:-}" in + up) up ;; + status) status ;; + down) down ;; + -h|--help) usage ;; + *) usage >&2; exit 1 ;; +esac diff --git a/scripts/dev-infra.test.sh b/scripts/dev-infra.test.sh new file mode 100644 index 000000000..925ebf831 --- /dev/null +++ b/scripts/dev-infra.test.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +INFRA="$SCRIPT_DIR/dev-infra.sh" +SETUP_WORKTREE="$SCRIPT_DIR/setup-worktree.sh" +START_DEV="$REPO_ROOT/start-dev.sh" +STOP_DEV="$REPO_ROOT/stop-dev.sh" +PACKAGE_JSON="$REPO_ROOT/package.json" + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + exit 1 +} + +assert_contains() { + local file_or_text="$1" expected="$2" + if [[ -f "$file_or_text" ]]; then + grep -Fq -- "$expected" "$file_or_text" || fail "Expected '$expected' in $file_or_text" + else + grep -Fq -- "$expected" <<<"$file_or_text" || fail "Expected '$expected' in output" + fi +} + +assert_not_contains() { + local file_or_text="$1" unexpected="$2" + if [[ -f "$file_or_text" ]]; then + ! grep -Fq -- "$unexpected" "$file_or_text" || fail "Did not expect '$unexpected' in $file_or_text" + else + ! grep -Fq -- "$unexpected" <<<"$file_or_text" || fail "Did not expect '$unexpected' in output" + fi +} + +root="$(mktemp -d)" +trap 'rm -rf "$root"' EXIT +fake_bin="$root/bin" +state_dir="$root/docker-state" +mkdir -p "$fake_bin" "$state_dir" + +cat >"$fake_bin/docker" <<'EOF_DOCKER' +#!/usr/bin/env bash +set -Eeuo pipefail +state_dir="${FAKE_DOCKER_STATE:?}" +log="${FAKE_DOCKER_LOG:?}" +printf '%q ' "$@" >>"$log" +printf '\n' >>"$log" + +case "${1:-}" in + info) + exit 0 + ;; + inspect) + if [[ "${2:-}" == "-f" ]]; then + name="${4:-}" + [[ -f "$state_dir/$name" ]] || exit 1 + printf '%s\n' "$(cat "$state_dir/$name")" + exit 0 + fi + name="${2:-}" + [[ -f "$state_dir/$name" ]] + ;; + start) + name="${2:?}" + [[ -f "$state_dir/$name" ]] || exit 1 + printf 'true\n' >"$state_dir/$name" + ;; + run) + name="" + args=("$@") + for ((i = 1; i < ${#args[@]}; i++)); do + if [[ "${args[$i]}" == "--name" ]]; then + name="${args[$((i + 1))]}" + break + fi + done + [[ -n "$name" ]] || exit 2 + printf 'true\n' >"$state_dir/$name" + printf 'fake-container-id\n' + ;; + rm) + name="${@: -1}" + rm -f "$state_dir/$name" + ;; + ps) + for file in "$state_dir"/*; do + [[ -e "$file" ]] || continue + if [[ "$(cat "$file")" == "true" ]]; then + basename "$file" + fi + done + ;; + *) + exit 0 + ;; +esac +EOF_DOCKER +chmod +x "$fake_bin/docker" + +cat >"$fake_bin/lsof" <<'EOF_LSOF' +#!/usr/bin/env bash +set -Eeuo pipefail +port="" +for arg in "$@"; do + case "$arg" in + -iTCP:*) port="${arg#-iTCP:}" ;; + -i:*) port="${arg#-i:}" ;; + esac +done +case ",${FAKE_BUSY_PORTS:-}," in + *",$port,"*) exit 0 ;; + *) exit 1 ;; +esac +EOF_LSOF +chmod +x "$fake_bin/lsof" + +export PATH="$fake_bin:$PATH" +export FAKE_DOCKER_STATE="$state_dir" +export FAKE_DOCKER_LOG="$root/docker.log" +: >"$FAKE_DOCKER_LOG" + +[[ -f "$INFRA" ]] || fail "Missing scripts/dev-infra.sh" +for script in "$INFRA" "$SETUP_WORKTREE" "$START_DEV" "$STOP_DEV"; do + bash -n "$script" +done + +# Shared infra starts exactly one stable Meilisearch and Chrome container. +bash "$INFRA" up >/dev/null +assert_contains "$FAKE_DOCKER_LOG" "karakeep-dev-meilisearch" +assert_contains "$FAKE_DOCKER_LOG" "127.0.0.1:7700:7700" +assert_contains "$FAKE_DOCKER_LOG" "getmeili/meilisearch:v1.41.0" +assert_contains "$FAKE_DOCKER_LOG" "karakeep-dev-chrome" +assert_contains "$FAKE_DOCKER_LOG" "127.0.0.1:9222:9222" +assert_contains "$FAKE_DOCKER_LOG" "ghcr.io/karakeep-app/karakeep-chrome:release" + +first_run_count="$(grep -c '^run ' "$FAKE_DOCKER_LOG" || true)" +bash "$INFRA" up >/dev/null +second_run_count="$(grep -c '^run ' "$FAKE_DOCKER_LOG" || true)" +[[ "$first_run_count" == "$second_run_count" ]] || fail "Repeated infra up created duplicate containers" + +# A foreign listener blocks creation instead of being silently reused. +rm -f "$state_dir/karakeep-dev-meilisearch" "$state_dir/karakeep-dev-chrome" +: >"$FAKE_DOCKER_LOG" +if FAKE_BUSY_PORTS=7700 bash "$INFRA" up >"$root/foreign.out" 2>&1; then + fail "Shared infra unexpectedly reused a foreign listener on port 7700" +fi +assert_contains "$root/foreign.out" "Port 7700 is already in use" + +# Worktrees share infra endpoints but retain unique web/data state and a Meilisearch-safe namespace. +main_root="$root/main" +workspace="$root/worktree" +mkdir -p "$main_root" "$workspace" +cat >"$main_root/.env" <<'EOF_ROOT_ENV' +NEXTAUTH_SECRET=dev-secret +MEILI_ADDR=https://old-meili.example +MEILI_MASTER_KEY=old-meili-secret +MEILI_VECTOR_ADDR=https://old-vector.example +MEILI_VECTOR_MASTER_KEY=old-vector-secret +BROWSER_WEB_URL=http://localhost:9333 +BROWSER_WEBSOCKET_URL=ws://localhost:9334 +BROWSERLESS_URL=https://old-browserless.example +BROWSERLESS_TOKEN=old-browserless-secret +BROWSER_CONNECT_ONDEMAND=true +EOF_ROOT_ENV +WT_ROOT_PATH="$main_root" \ +WT_WORKSPACE_PATH="$workspace" \ +WT_WORKSPACE_NAME='Issue/ABC.weird' \ +WT_PORT_BASE=7 \ +"$SETUP_WORKTREE" >/dev/null +assert_contains "$workspace/.env" "KARAKEEP_PORT=3007" +assert_contains "$workspace/.env" "DATA_DIR=$workspace/.data/local" +assert_contains "$workspace/.env" "MEILI_ADDR=http://localhost:7700" +assert_contains "$workspace/.env" "MEILI_MASTER_KEY=" +assert_contains "$workspace/.env" "BROWSER_WEB_URL=http://localhost:9222" +assert_contains "$workspace/.env" "BROWSER_CONNECT_ONDEMAND=false" +assert_contains "$workspace/.env" "MEILI_INDEX_PREFIX=issue-abc-weird-7_" +assert_not_contains "$workspace/.env" "MEILI_INDEX_PREFIX=issue-abc.weird-7_" +assert_not_contains "$workspace/.env" "old-meili.example" +assert_not_contains "$workspace/.env" "old-meili-secret" +assert_not_contains "$workspace/.env" "old-vector.example" +assert_not_contains "$workspace/.env" "old-vector-secret" +assert_not_contains "$workspace/.env" "old-browserless.example" +assert_not_contains "$workspace/.env" "old-browserless-secret" +assert_not_contains "$workspace/.env" "http://localhost:9333" +assert_not_contains "$workspace/.env" "ws://localhost:9334" +assert_not_contains "$workspace/.env" "http://localhost:7707" +assert_not_contains "$workspace/.env" "http://localhost:9229" + +# Workspace lifecycle delegates shared infra startup, pins service selection, and never tears it down implicitly. +assert_contains "$START_DEV" 'scripts/dev-infra.sh" up' +assert_contains "$START_DEV" "unset MEILI_VECTOR_ADDR MEILI_VECTOR_MASTER_KEY" +assert_contains "$START_DEV" "unset BROWSER_WEBSOCKET_URL BROWSERLESS_URL BROWSERLESS_TOKEN" +assert_contains "$START_DEV" "export BROWSER_CONNECT_ONDEMAND=false" +assert_not_contains "$START_DEV" "gcr.io/zenika-hub/alpine-chrome:124" +assert_not_contains "$STOP_DEV" 'docker stop "$MEILI_CONTAINER"' +assert_not_contains "$STOP_DEV" 'docker stop "$CHROME_CONTAINER"' +assert_not_contains "$STOP_DEV" 'docker rm "$MEILI_CONTAINER"' +assert_not_contains "$STOP_DEV" 'docker rm "$CHROME_CONTAINER"' +assert_contains "$PACKAGE_JSON" '"dev:infra:up": "bash scripts/dev-infra.sh up"' +assert_contains "$PACKAGE_JSON" '"dev:infra:status": "bash scripts/dev-infra.sh status"' +assert_contains "$PACKAGE_JSON" '"dev:infra:down": "bash scripts/dev-infra.sh down"' + +# Explicit down owns only the shared infra containers. +: >"$FAKE_DOCKER_LOG" +printf 'true\n' >"$state_dir/karakeep-dev-meilisearch" +printf 'true\n' >"$state_dir/karakeep-dev-chrome" +bash "$INFRA" down >/dev/null +assert_contains "$FAKE_DOCKER_LOG" "rm -f karakeep-dev-meilisearch" +assert_contains "$FAKE_DOCKER_LOG" "rm -f karakeep-dev-chrome" + +printf 'Shared dev infrastructure tests passed.\n' diff --git a/scripts/setup-worktree.sh b/scripts/setup-worktree.sh index 96b1a4e46..ceda1c26a 100755 --- a/scripts/setup-worktree.sh +++ b/scripts/setup-worktree.sh @@ -10,6 +10,7 @@ workspace_env="$WT_WORKSPACE_PATH/.env" workspace_data_dir="$WT_WORKSPACE_PATH/.data/local" refresh_data="${WT_REFRESH_DATA:-false}" data_source="${WT_DATA_SOURCE:-main}" +workspace_name="${WT_WORKSPACE_NAME:-$(basename "$WT_WORKSPACE_PATH")}" [[ -f "$root_env" ]] || { echo "error: missing root environment file: $root_env" >&2 @@ -23,15 +24,16 @@ case "$WT_PORT_BASE" in ;; esac +workspace_slug="$(printf '%s' "$workspace_name" | tr '[:upper:]' '[:lower:]' | tr -cs '[:alnum:]_-' '-' | sed 's/^-*//; s/-*$//')" +[[ -n "$workspace_slug" ]] || workspace_slug="worktree" web_port=$((3000 + WT_PORT_BASE)) -meili_port=$((7700 + WT_PORT_BASE)) -chrome_port=$((9222 + WT_PORT_BASE)) +meili_index_prefix="${workspace_slug}-${WT_PORT_BASE}_" tmp_env="$(mktemp)" trap 'rm -f "$tmp_env"' EXIT while IFS= read -r line || [[ -n "$line" ]]; do case "$line" in - DATA_DIR=* | KARAKEEP_PORT=* | API_URL=* | NEXTAUTH_URL=* | MEILI_ADDR=* | BROWSER_WEB_URL=* | BROWSER_WEBSOCKET_URL=*) + DATA_DIR=* | KARAKEEP_PORT=* | API_URL=* | NEXTAUTH_URL=* | MEILI_ADDR=* | MEILI_MASTER_KEY=* | MEILI_VECTOR_ADDR=* | MEILI_VECTOR_MASTER_KEY=* | MEILI_INDEX_PREFIX=* | BROWSER_WEB_URL=* | BROWSER_WEBSOCKET_URL=* | BROWSERLESS_URL=* | BROWSERLESS_TOKEN=* | BROWSER_CONNECT_ONDEMAND=*) ;; *) printf '%s\n' "$line" >>"$tmp_env" @@ -44,8 +46,11 @@ DATA_DIR=$workspace_data_dir KARAKEEP_PORT=$web_port API_URL=http://localhost:$web_port NEXTAUTH_URL=http://localhost:$web_port -MEILI_ADDR=http://localhost:$meili_port -BROWSER_WEB_URL=http://localhost:$chrome_port +MEILI_ADDR=http://localhost:7700 +MEILI_MASTER_KEY= +MEILI_INDEX_PREFIX=$meili_index_prefix +BROWSER_WEB_URL=http://localhost:9222 +BROWSER_CONNECT_ONDEMAND=false ENV mv "$tmp_env" "$workspace_env" @@ -85,4 +90,4 @@ case "$data_source" in ;; esac -echo "Configured worktree ports: web $web_port, Meilisearch $meili_port, Chrome $chrome_port" +echo "Configured worktree: web $web_port, shared Meilisearch 7700, shared Chrome 9222, Meilisearch prefix $meili_index_prefix" diff --git a/start-dev.sh b/start-dev.sh index 99f492d5c..c373ec877 100755 --- a/start-dev.sh +++ b/start-dev.sh @@ -1,9 +1,12 @@ #!/usr/bin/env bash +set -euo pipefail # Karakeep dev launcher. -# ./start-dev.sh foreground — logs in this terminal, Ctrl+C stops everything -# ./start-dev.sh -d detached — frees the shell, logs in .dev/, stop with ./stop-dev.sh +# ./start-dev.sh foreground - logs in this terminal, Ctrl+C stops this workspace +# ./start-dev.sh -d detached - frees the shell, logs in .dev/, stop with ./stop-dev.sh +# Shared Meilisearch + Chrome remain machine-level infrastructure managed by scripts/dev-infra.sh. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DETACH=0 case "${1:-}" in -d|--detach) DETACH=1 ;; @@ -15,112 +18,63 @@ esac DEV_DIR=".dev" mkdir -p "$DEV_DIR" -# Worktrees receive KARAKEEP_PORT, MEILI_ADDR, and BROWSER_WEB_URL from their -# generated .env. The main workspace retains the conventional defaults. -WORKSPACE_NAME="${WT_WORKSPACE_NAME:-main}" -WORKSPACE_SLUG="$(printf '%s' "$WORKSPACE_NAME" | tr -cs '[:alnum:]_.-' '-')" -MEILI_CONTAINER="karakeep-${WORKSPACE_SLUG}-meilisearch" -CHROME_CONTAINER="karakeep-${WORKSPACE_SLUG}-chrome" -WEB_PORT="${KARAKEEP_PORT:-3000}" -MEILI_PORT=7700 - -if [ -f ".env" ]; then - _web_port=$(grep "^KARAKEEP_PORT=" .env | cut -d'=' -f2-) - case "$_web_port" in - ''|*[!0-9]*) ;; - *) WEB_PORT="$_web_port" ;; - esac - - _meili_addr=$(grep "^MEILI_ADDR=" .env | cut -d'=' -f2-) - _meili_port="${_meili_addr##*:}"; _meili_port="${_meili_port%%/*}" - case "$_meili_port" in - ''|*[!0-9]*) ;; - *) MEILI_PORT="$_meili_port" ;; - esac -fi - -case "$WEB_PORT" in - ''|*[!0-9]*) echo "Error: KARAKEEP_PORT must be a port number."; exit 1 ;; -esac - -# Function to check if a command exists command_exists() { command -v "$1" >/dev/null 2>&1 } -# Function to check if a port is in use -port_in_use() { - lsof -i :"$1" >/dev/null 2>&1 +env_value() { + local key="$1" + if [ -f ".env" ]; then + grep -m1 "^${key}=" .env 2>/dev/null | cut -d'=' -f2- || true + fi } -# Recursively terminate a process and all its descendants (portable: macOS + Linux). -# pnpm spawns next/tsx children, so killing just the pnpm pid can leave orphans. kill_tree() { local pid="$1" child for child in $(pgrep -P "$pid" 2>/dev/null); do kill_tree "$child" done - kill "$pid" 2>/dev/null + kill "$pid" 2>/dev/null || true } -# Headless-Chrome host port: derived from BROWSER_WEB_URL in .env (default 9222). -CHROME_PORT=9222 -if [ -f ".env" ]; then - _bw=$(grep "^BROWSER_WEB_URL=" .env | cut -d'=' -f2-) - _p="${_bw##*:}"; _p="${_p%%/*}" - case "$_p" in - ''|*[!0-9]*) ;; - *) CHROME_PORT="$_p" ;; - esac -fi - -# Check if Docker is installed if ! command_exists docker; then echo "Error: Docker is not installed. Please install Docker first." exit 1 fi -# Check if pnpm is installed if ! command_exists pnpm; then echo "Error: pnpm is not installed. Please install pnpm first." exit 1 fi -# Start Meilisearch if not already running. -if ! port_in_use "$MEILI_PORT"; then - echo "Starting Meilisearch on port $MEILI_PORT..." - docker run -d -p "$MEILI_PORT:7700" --name "$MEILI_CONTAINER" getmeili/meilisearch:v1.41.0 -else - echo "Meilisearch is already running on port $MEILI_PORT" -fi +WEB_PORT="${KARAKEEP_PORT:-$(env_value KARAKEEP_PORT)}" +WEB_PORT="${WEB_PORT:-3000}" +case "$WEB_PORT" in + ''|*[!0-9]*) echo "Error: KARAKEEP_PORT must be a port number."; exit 1 ;; +esac -# Start Chrome if not already running -# Start Chrome if not already running. -if ! port_in_use "$CHROME_PORT"; then - echo "Starting headless Chrome on port $CHROME_PORT..." - docker run -d -p "$CHROME_PORT:9222" --name "$CHROME_CONTAINER" gcr.io/zenika-hub/alpine-chrome:124 \ - --no-sandbox \ - --disable-gpu \ - --disable-dev-shm-usage \ - --remote-debugging-address=0.0.0.0 \ - --remote-debugging-port=9222 \ - --hide-scrollbars -else - echo "Port $CHROME_PORT already in use; assuming a compatible Chrome/CDP is there" -fi +MEILI_ADDR="${MEILI_ADDR:-$(env_value MEILI_ADDR)}" +MEILI_ADDR="${MEILI_ADDR:-http://localhost:7700}" +BROWSER_WEB_URL="${BROWSER_WEB_URL:-$(env_value BROWSER_WEB_URL)}" +BROWSER_WEB_URL="${BROWSER_WEB_URL:-http://localhost:9222}" +MEILI_INDEX_PREFIX="${MEILI_INDEX_PREFIX:-$(env_value MEILI_INDEX_PREFIX)}" +MEILI_INDEX_PREFIX="${MEILI_INDEX_PREFIX:-main_}" +unset MEILI_VECTOR_ADDR MEILI_VECTOR_MASTER_KEY +unset BROWSER_WEBSOCKET_URL BROWSERLESS_URL BROWSERLESS_TOKEN +export MEILI_ADDR BROWSER_WEB_URL MEILI_INDEX_PREFIX +export BROWSER_CONNECT_ONDEMAND=false + +bash "$SCRIPT_DIR/scripts/dev-infra.sh" up -# Install dependencies if node_modules doesn't exist if [ ! -d "node_modules" ]; then echo "Installing dependencies..." pnpm install fi -# Get DATA_DIR from environment or .env file if [ -z "${DATA_DIR:-}" ] && [ -f ".env" ]; then - DATA_DIR=$(grep "^DATA_DIR=" .env | cut -d'=' -f2) + DATA_DIR="$(env_value DATA_DIR)" fi -# Create DATA_DIR if it doesn't exist if [ -n "${DATA_DIR:-}" ] && [ ! -d "$DATA_DIR" ]; then echo "Creating DATA_DIR at $DATA_DIR..." mkdir -p "$DATA_DIR" @@ -140,48 +94,49 @@ fi echo "$WEB_PID" > "$DEV_DIR/web.pid" echo "$WORKERS_PID" > "$DEV_DIR/workers.pid" -# Function to handle script termination (foreground mode) cleanup() { echo "" - echo "Shutting down services..." + echo "Shutting down this workspace..." kill_tree "$WEB_PID" kill_tree "$WORKERS_PID" - docker stop "$MEILI_CONTAINER" "$CHROME_CONTAINER" 2>/dev/null - docker rm "$MEILI_CONTAINER" "$CHROME_CONTAINER" 2>/dev/null rm -f "$DEV_DIR/web.pid" "$DEV_DIR/workers.pid" + echo "Shared Meilisearch and Chrome are still running. Stop them explicitly with: pnpm dev:infra:down" exit 0 } -# Wait for web app to be ready (max 30 seconds) -echo "Waiting for web app to start..." -ATTEMPT=0 -while [ $ATTEMPT -lt 30 ]; do - if nc -z localhost "$WEB_PORT" 2>/dev/null; then - break +if command_exists nc; then + echo "Waiting for web app to start..." + ATTEMPT=0 + while [ $ATTEMPT -lt 30 ]; do + if nc -z localhost "$WEB_PORT" 2>/dev/null; then + break + fi + sleep 1 + ATTEMPT=$((ATTEMPT + 1)) + done + if [ $ATTEMPT -eq 30 ]; then + echo "Warning: Web app may not have started properly after 30 seconds" fi - sleep 1 - ATTEMPT=$((ATTEMPT + 1)) -done -if [ $ATTEMPT -eq 30 ]; then - echo "Warning: Web app may not have started properly after 30 seconds" +else + echo "Skipping web readiness probe because 'nc' is not installed." fi echo "" echo "Development environment is running!" -echo " Web app: http://localhost:$WEB_PORT" -echo " Meilisearch: http://localhost:$MEILI_PORT" -echo " Chrome debugger: http://localhost:$CHROME_PORT" +echo " Web app: http://localhost:$WEB_PORT" +echo " Meilisearch: $MEILI_ADDR" +echo " Meili index prefix: $MEILI_INDEX_PREFIX" +echo " Chrome debugger: $BROWSER_WEB_URL" +echo " Shared infra: pnpm dev:infra:status" if [ "$DETACH" -eq 1 ]; then echo "" echo "Running detached (your shell is free)." echo " Logs: tail -f $DEV_DIR/web.log $DEV_DIR/workers.log" - echo " Stop: ./stop-dev.sh" + echo " Stop this workspace: ./stop-dev.sh" exit 0 else - # Set up trap to catch termination signals trap cleanup SIGINT SIGTERM - echo " Press Ctrl+C to stop all services" - # Wait for user interrupt + echo " Press Ctrl+C to stop this workspace (shared infra stays running)" wait fi diff --git a/stop-dev.sh b/stop-dev.sh index 3cd9de0e7..d976043c8 100755 --- a/stop-dev.sh +++ b/stop-dev.sh @@ -1,22 +1,17 @@ #!/usr/bin/env bash +set -euo pipefail -# Stop the Karakeep dev environment started by ./start-dev.sh -# (works for detached `-d` runs; also tears down the meilisearch + chrome containers). +# Stop only the Karakeep web/workers processes for this workspace. +# Shared Meilisearch + Chrome are machine-level infrastructure and stay running. DEV_DIR=".dev" -WORKSPACE_NAME="${WT_WORKSPACE_NAME:-main}" -WORKSPACE_SLUG="$(printf '%s' "$WORKSPACE_NAME" | tr -cs '[:alnum:]_.-' '-')" -MEILI_CONTAINER="karakeep-${WORKSPACE_SLUG}-meilisearch" -CHROME_CONTAINER="karakeep-${WORKSPACE_SLUG}-chrome" - -# Recursively terminate a process and all its descendants (portable: macOS + Linux). kill_tree() { local pid="$1" child for child in $(pgrep -P "$pid" 2>/dev/null); do kill_tree "$child" done - kill "$pid" 2>/dev/null + kill "$pid" 2>/dev/null || true } stop_proc() { @@ -38,8 +33,5 @@ stop_proc() { stop_proc "web app" "web.pid" stop_proc "workers" "workers.pid" -echo "Stopping Meilisearch + Chrome containers..." -docker stop "$MEILI_CONTAINER" "$CHROME_CONTAINER" 2>/dev/null -docker rm "$MEILI_CONTAINER" "$CHROME_CONTAINER" 2>/dev/null - -echo "Done. Dev environment stopped." +echo "Done. This workspace is stopped." +echo "Shared Meilisearch + Chrome are still running. Stop them explicitly with: pnpm dev:infra:down"