feat: integrate mailing list service (CM-1318)

.github/workflows/backend-lint.yaml

@@ -98,6 +98,48 @@ jobs:
         if: steps.changes.outputs.python_changed == 'true'
         run: uv sync --group dev --frozen
 
+      - name: Check Python linting and formatting
+        if: steps.changes.outputs.python_changed == 'true'
+        run: |
+          uv run ruff check src/ --output-format=github
+
+  lint-python-mailing-list-integration:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./services/apps/mailing_list_integration
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for Python file changes
+        id: changes
+        run: |
+          if git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q "^services/apps/mailing_list_integration/.*\.py$"; then
+            echo "python_changed=true" >> $GITHUB_OUTPUT
+          else
+            echo "python_changed=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Install uv
+        if: steps.changes.outputs.python_changed == 'true'
+        uses: astral-sh/setup-uv@v3
+        with:
+          enable-cache: true
+          cache-dependency-glob: "services/apps/mailing_list_integration/uv.lock"
+
+      - name: Set up Python
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv python install 3.13
+
+      - name: Install dependencies
+        if: steps.changes.outputs.python_changed == 'true'
+        run: uv sync --group dev --frozen
+
       - name: Check Python linting and formatting
         if: steps.changes.outputs.python_changed == 'true'
         run: |

backend/src/database/migrations/V1784048135__mailinglist-schema.sql

@@ -0,0 +1,50 @@
+-- Create the mailinglist schema
+CREATE SCHEMA IF NOT EXISTS mailinglist;
+
+-- Mailing lists (lore/public-inbox lists onboarded into CDP)
+CREATE TABLE mailinglist.lists (
+    id UUID PRIMARY KEY NOT NULL DEFAULT uuid_generate_v4(),
+    "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    "deletedAt" TIMESTAMP WITH TIME ZONE,
+
+    name TEXT NOT NULL,
+    "sourceUrl" TEXT NOT NULL,
+
+    "segmentId" UUID NOT NULL REFERENCES segments (id),
+    "integrationId" UUID NOT NULL REFERENCES public."integrations" (id),
+
+    UNIQUE ("sourceUrl")
+);
+
+-- Per-list processing state (public-inbox shard head tracking)
+CREATE TABLE mailinglist."listProcessing" (
+    "listId" UUID PRIMARY KEY NOT NULL REFERENCES mailinglist.lists (id) ON DELETE CASCADE,
+
+    "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+    "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+
+    state VARCHAR(50) NOT NULL,
+    priority INTEGER NOT NULL DEFAULT 2, -- 0=urgent, 1=high, 2=normal
+
+    "lockedAt" TIMESTAMP WITH TIME ZONE,
+    "lastProcessedAt" TIMESTAMP WITH TIME ZONE,
+    "lastProcessedHeads" JSONB NOT NULL DEFAULT '{}'
+);
+
+-- Indexes for optimal query performance
+
+CREATE INDEX "ix_mailinglist_lists_segmentId" ON mailinglist.lists ("segmentId");
+CREATE INDEX "ix_mailinglist_lists_integrationId" ON mailinglist.lists ("integrationId");
+
+CREATE INDEX "ix_mailinglist_listProcessing_state" ON mailinglist."listProcessing" (state);
+CREATE INDEX "ix_mailinglist_listProcessing_state_priority" ON mailinglist."listProcessing" (state, priority);
+
+-- Comments for documentation
+COMMENT ON SCHEMA mailinglist IS 'Schema for mailing list integration system that manages mailing list processing state';
+COMMENT ON TABLE mailinglist.lists IS 'Stores onboarded mailing lists (public-inbox/lore) and their segment/integration associations';
+COMMENT ON TABLE mailinglist."listProcessing" IS 'Per-list processing state including public-inbox shard head tracking';
+
+COMMENT ON COLUMN mailinglist."listProcessing".priority IS 'Processing priority: 0=urgent, 1=high, 2=normal';
+COMMENT ON COLUMN mailinglist."listProcessing".state IS 'Current processing state of the list';
+COMMENT ON COLUMN mailinglist."listProcessing"."lastProcessedHeads" IS 'JSONB map of shard index to last processed commit SHA, e.g. {"0": sha, "1": sha}';

scripts/builders/mailing-list-integration.env

@@ -0,0 +1,4 @@
+DOCKERFILE="./services/docker/Dockerfile.mailing_list_integration"
+CONTEXT="../"
+REPO="sjc.ocir.io/axbydjxa5zuh/mailing-list-integration"
+SERVICES="mailing-list-integration"

scripts/cli

@@ -1189,7 +1189,7 @@ while test $# -gt 0; do
         exit
         ;;
     clean-start-fe-dev)
-        IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker")
+        IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "cache-worker" "categorization-worker" "cron-service" "data-sink-worker" "git-integration" "mailing-list-integration" "integration-run-worker" "integration-stream-worker" "nango-webhook-api" "nango-worker" "script-executor-worker" "search-sync-api" "search-sync-worker" "security-best-practices-worker" "snowflake-connectors-worker")
         CLEAN_START=1
         DEV=1
         start

scripts/services/docker/Dockerfile.mailing_list_integration

@@ -0,0 +1,69 @@
+# Base image for both stages
+FROM python:3.13.5-slim-bullseye AS base
+
+# Builder stage: install build dependencies, uv, and dependencies
+FROM base AS builder
+
+# Install build dependencies
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy uv binary from official image
+COPY --from=ghcr.io/astral-sh/uv:0.7.17 /uv /usr/local/bin/uv
+
+WORKDIR /usr/crowd/app
+
+ENV UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    UV_PYTHON_DOWNLOADS=never \
+    UV_PYTHON=python3.13 \
+    UV_PROJECT_ENVIRONMENT=/usr/crowd/app/.venv \
+    UV_VENV_PATH=/usr/crowd/app/.venv
+
+# Copy only lock, pyproject.toml and License for dependency install caching
+COPY ./services/apps/mailing_list_integration/pyproject.toml ./services/apps/mailing_list_integration/uv.lock ./LICENSE ./
+
+# Install dependencies excluding the project itself for better caching
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project --no-dev
+
+# Copy full source code
+COPY ./services/apps/mailing_list_integration ./LICENSE ./
+
+# Sync full project including the project itself
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-dev
+
+
+# Runner: minimal image with runtime deps and virtualenv only
+FROM base AS runner
+
+# Install runtime dependencies, including public-inbox for mirroring lists
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    git \
+    public-inbox \
+    --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=off
+
+WORKDIR /usr/crowd/app
+
+# Copy virtual environment and app source from builder
+COPY --from=builder /usr/crowd/app/.venv /usr/crowd/app/.venv
+COPY --from=builder /usr/crowd/app /usr/crowd/app
+
+# Add virtual environment bin to PATH
+ENV PATH="/usr/crowd/app/.venv/bin:$PATH"
+
+# Make runner script executable
+RUN chmod +x ./src/runner.sh
+
+EXPOSE 8085
+
+CMD ["./src/runner.sh"]

scripts/services/mailing-list-integration.yaml

@@ -0,0 +1,50 @@
+
+x-env-args: &env-args
+  DOCKER_BUILDKIT: 1
+  NODE_ENV: docker
+  SERVICE: mailing-list-integration
+  SHELL: /bin/sh
+
+services:
+  mailing-list-integration:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.mailing_list_integration
+    working_dir: /usr/crowd/app/services/apps/mailing_list_integration
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    restart: always
+    networks:
+      - crowd-bridge
+    ports:
+      - '8086:8085'
+
+  mailing-list-integration-dev:
+    build:
+      context: ../../
+      dockerfile: ./scripts/services/docker/Dockerfile.mailing_list_integration
+    command: ["./src/runner.sh", "local"]
+    working_dir: /usr/crowd/app/services/apps/mailing_list_integration
+    env_file:
+      - ../../backend/.env.dist.local
+      - ../../backend/.env.dist.composed
+      - ../../backend/.env.override.local
+      - ../../backend/.env.override.composed
+    environment:
+      <<: *env-args
+    hostname: mailing-list-integration
+    ports:
+      - '8086:8085'
+    networks:
+      - crowd-bridge
+    volumes:
+      - ../../services/apps/mailing_list_integration/src:/usr/crowd/app/services/apps/mailing_list_integration/src
+
+networks:
+  crowd-bridge:
+    external: true

services/apps/mailing_list_integration/LICENSE

@@ -0,0 +1 @@
+../../../LICENSE

services/apps/mailing_list_integration/Makefile

@@ -0,0 +1,116 @@
+.DEFAULT_GOAL := help
+
+help: ## Shows this help message
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m  %s\033[0m\n\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+ifeq (cmd,$(firstword $(MAKECMDGOALS)))
+  CMD_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  $(eval $(CMD_ARGS):;@:)
+endif
+
+COMPOSE_FILE = docker-compose -f ../../../scripts/services/mailing-list-integration.yaml
+
+# Check if local setup is complete
+check_setup:
+	@if ! command -v uv >/dev/null 2>&1; then \
+		echo "❌ uv not found. Run 'make setup' first."; \
+		exit 1; \
+	fi
+	@if ! uv run ruff --version >/dev/null 2>&1; then \
+		echo "📦 Installing dev dependencies..."; \
+		uv sync --group dev; \
+	fi
+
+##@ 🚀 Setup and installation
+
+setup: ## Install uv and dev dependencies (only if not already installed)
+	@echo "🔧 Checking if uv is installed..."
+	@if command -v uv >/dev/null 2>&1; then \
+		echo "✅ uv is already installed at $$(which uv)"; \
+		echo "📦 Installing dev dependencies..."; \
+		uv sync --group dev; \
+		echo "✅ Setup complete"; \
+	else \
+		echo "📦 uv is required to:"; \
+		echo "   - Add/remove packages to pyproject.toml"; \
+		echo "   - Manage dependency versions and conflicts"; \
+		echo "   - Generate uv.lock for reproducible builds"; \
+		echo "   - Install dependencies locally for development"; \
+		echo ""; \
+		echo "Installing uv..."; \
+		curl -LsSf https://astral.sh/uv/install.sh | sh; \
+		echo "Adding uv to PATH..."; \
+		echo 'export PATH="$$HOME/.cargo/bin:$$PATH"' >> ~/.zshrc; \
+		echo 'export PATH="$$HOME/.cargo/bin:$$PATH"' >> ~/.bash_profile; \
+		echo 'export PATH="$$HOME/.cargo/bin:$$PATH"' >> ~/.bashrc; \
+		export PATH="$$HOME/.cargo/bin:$$PATH"; \
+		echo "📦 Installing dev dependencies..."; \
+		uv sync --group dev; \
+		echo "✅ uv installed and dev dependencies ready"; \
+		echo "📝 Next steps:"; \
+		echo "   - Restart your terminal or run: source ~/.zshrc"; \
+		echo "   - Use: make add_package name=<package> to add dependencies"; \
+	fi
+
+##@ 🛠️  Local development tools
+
+run: ## Start the mailing-list-integration service in interactive mode
+	$(COMPOSE_FILE) run --service-ports --rm mailing-list-integration-dev
+
+cmd: ## Run any command inside the container, example: make cmd -- ls -la
+	$(COMPOSE_FILE) run --rm --no-deps mailing-list-integration-dev $(CMD_ARGS)
+
+lint: check_setup ## Check code with ruff
+	@uv run ruff check src/
+
+format: check_setup ## Format code with ruff
+	@uv run ruff check --fix --select I src/
+	@uv run ruff format src/
+	$(MAKE) lint
+
+fix-all: check_setup ## Fix all lint issues, format everything, and run final lint check
+	@echo "🔧 Fixing all linting issues..."
+	@uv run ruff check --fix src/
+	@echo "✨ Formatting all code..."
+	@uv run ruff format src/
+	@echo "🔍 Running final lint check..."
+	@uv run ruff check src/
+	@echo "✅ All done! Code is linted and formatted."
+
+rebuild: ## Rebuild the container
+	$(COMPOSE_FILE) build mailing-list-integration-dev
+	@echo "✅ Container rebuilt"
+
+##@ 📦 Dependency management
+
+add_package: check_setup ## Add new package to pyproject.toml, example: make add_package name=requests
+	@echo "Adding $(name) to dependencies..."
+	@uv add $(name)
+	@echo "Rebuilding container..."
+	$(COMPOSE_FILE) build mailing-list-integration-dev
+	@echo "✅ Package $(name) added and container rebuilt"
+
+remove_package: check_setup ## Remove package from pyproject.toml, example: make remove_package name=requests
+	@echo "Removing $(name) from dependencies..."
+	@uv remove $(name)
+	@echo "Rebuilding container..."
+	$(COMPOSE_FILE) build mailing-list-integration-dev
+	@echo "✅ Package $(name) removed and container rebuilt"
+
+update_deps: check_setup ## Update all dependencies
+	@echo "Updating dependencies..."
+	@uv lock
+	@echo "Rebuilding container..."
+	$(COMPOSE_FILE) build mailing-list-integration-dev
+	@echo "✅ Dependencies updated and container rebuilt"
+
+lock_deps: check_setup ## Generate/update uv.lock file
+	@echo "Generating uv.lock file..."
+	@uv lock
+	@echo "✅ uv.lock updated"
+
+##@ 🧪 Testing
+
+test: check_setup ## Run tests
+	@echo "🧪 Running tests..."
+	@uv run pytest src/test/ -v