[SHOT-231] Add scheduled seeded database workflow

.github/workflows/build-seeded-databases.yml

@@ -0,0 +1,229 @@
+name: Build Seeded Database Images
+
+on:
+  push:
+    branches: [feat/seeder-build-pipeline]
+    paths:
+      - .github/workflows/build-seeded-databases.yml
+      - util/SeederUtility/**
+      - util/Seeder/**
+  schedule:
+    - cron: "0 2 * * 0"
+  workflow_dispatch:
+    inputs:
+      preset:
+        description: "Preset name to build (empty = curated default list, all = every preset)"
+        required: false
+        type: string
+      database:
+        description: "Database type to build (all = default matrix)"
+        required: false
+        default: all
+        type: choice
+        options:
+          - all
+          - postgres
+          - mysql
+          - mariadb
+          - mssql
+
+env:
+  _AZ_REGISTRY: bitwardenprod.azurecr.io
+  _DEFAULT_PRESETS: >-
+    ["qa.dunder-mifflin-enterprise-full",
+    "scale.md-balanced-sterling-cooper",
+    "scale.lg-balanced-wayne-enterprises",
+    "scale.lg-highperm-tyrell-corp",
+    "scale.xl-broad-initech"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  setup:
+    name: Determine build matrix
+    runs-on: ubuntu-22.04
+    outputs:
+      presets: ${{ steps.matrix.outputs.presets }}
+      databases: ${{ steps.matrix.outputs.databases }}
+    steps:
+      - name: Check out repo
+        if: inputs.preset == 'all'
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up .NET
+        if: inputs.preset == 'all'
+        uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0
+        with:
+          cache: true
+          cache-dependency-path: "**/*.csproj"
+
+      - name: Build SeederUtility
+        if: inputs.preset == 'all'
+        run: dotnet build util/SeederUtility/SeederUtility.csproj
+
+      - name: Determine matrix
+        id: matrix
+        env:
+          INPUT_PRESET: ${{ inputs.preset }}
+          INPUT_DATABASE: ${{ inputs.database }}
+        run: |
+          if [[ "${INPUT_PRESET}" == "all" ]]; then
+            presets=$(dotnet run --project util/SeederUtility --no-build -- preset --list --output json \
+              | jq -c '[.organization[], .individual[]]')
+          elif [[ -n "${INPUT_PRESET}" ]]; then
+            presets="[\"${INPUT_PRESET}\"]"
+          else
+            presets="${_DEFAULT_PRESETS}"
+          fi
+
+          if [[ -n "${INPUT_DATABASE}" && "${INPUT_DATABASE}" != "all" ]]; then
+            databases="[\"${INPUT_DATABASE}\"]"
+          else
+            databases='["postgres","mysql","mariadb","mssql"]'
+          fi
+
+          echo "presets=${presets}" >> "$GITHUB_OUTPUT"
+          echo "databases=${databases}" >> "$GITHUB_OUTPUT"
+          echo "Preset matrix:   ${presets}"
+          echo "Database matrix: ${databases}"
+
+  build:
+    name: ${{ matrix.database }} / ${{ matrix.preset }}
+    needs: setup
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      id-token: write
+    strategy:
+      matrix:
+        preset: ${{ fromJson(needs.setup.outputs.presets) }}
+        database: ${{ fromJson(needs.setup.outputs.databases) }}
+      fail-fast: false
+
+    steps:
+      - name: Check out repo
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Set up .NET
+        uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0
+        with:
+          cache: true
+          cache-dependency-path: "**/*.csproj"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
+
+      - name: Restore .NET local tools
+        run: dotnet tool restore
+
+      - name: Log in to Azure
+        uses: bitwarden/gh-actions/azure-login@main
+        with:
+          subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+          client_id: ${{ secrets.AZURE_CLIENT_ID }}
+
+      - name: Retrieve data protection key
+        id: retrieve-secret
+        uses: bitwarden/gh-actions/get-keyvault-secrets@main
+        with:
+          keyvault: gh-org-bitwarden
+          secrets: "DP-KEY-XML"
+
+      - name: Log out from Azure
+        uses: bitwarden/gh-actions/azure-logout@main
+
+      - name: Build seeded image
+        env:
+          PUSH: "false"
+          REGISTRY: ${{ env._AZ_REGISTRY }}
+          GIT_SHA: ${{ github.sha }}
+          MATRIX_PRESET: ${{ matrix.preset }}
+          MATRIX_DATABASE: ${{ matrix.database }}
+          DP_KEY_XML: ${{ steps.retrieve-secret.outputs.DP-KEY-XML }}
+        run: |
+          GIT_SHA="${GIT_SHA:0:7}" \
+          bash util/SeederUtility/scripts/build-seeded-image.sh \
+            "${MATRIX_PRESET}" \
+            "${MATRIX_DATABASE}"
+
+      - name: Save image as tarball
+        env:
+          MATRIX_PRESET: ${{ matrix.preset }}
+          MATRIX_DATABASE: ${{ matrix.database }}
+        run: |
+          TAG="${MATRIX_PRESET//./-}"
+          DB="${MATRIX_DATABASE}"
+          SHA_SHORT="${GITHUB_SHA:0:7}"
+          docker save \
+            "${_AZ_REGISTRY}/shot/seeded-${DB}:${TAG}-latest" \
+            "${_AZ_REGISTRY}/shot/seeded-${DB}:${TAG}-${SHA_SHORT}" \
+            -o "seeded-${DB}-${TAG}-${SHA_SHORT}.tar"
+
+      - name: Upload image artifact
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: seeded-${{ matrix.database }}-${{ matrix.preset }}
+          path: seeded-*.tar
+          retention-days: 7
+          if-no-files-found: error
+
+      - name: Upload core bundle
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: seeded-core-${{ matrix.database }}-${{ matrix.preset }}
+          path: util/SeederUtility/docker/bundles/seeded-core-*.tar.gz
+          retention-days: 7
+          if-no-files-found: error
+
+  summary:
+    name: Image summary
+    needs: [setup, build]
+    if: always()
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Render image table
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PRESETS: ${{ needs.setup.outputs.presets }}
+          DATABASES: ${{ needs.setup.outputs.databases }}
+          GIT_SHA: ${{ github.sha }}
+        run: |
+          SHA_SHORT="${GIT_SHA:0:7}"
+          results=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/jobs" \
+            --paginate --jq '.jobs[] | [.name, .conclusion] | @tsv')
+
+          {
+            echo "### Seeded images for \`${SHA_SHORT}\`"
+            echo ""
+            echo "| Preset | Database | Image |"
+            echo "| --- | --- | --- |"
+            while read -r db; do
+              while read -r preset; do
+                tag="${preset//./-}"
+                conclusion=$(printf '%s\n' "${results}" \
+                  | awk -F'\t' -v name="${db} / ${preset}" '$1 == name { print $2; exit }')
+                if [ "${conclusion}" != "success" ]; then
+                  continue
+                fi
+                echo "| ${preset} | ${db} | \`${_AZ_REGISTRY}/shot/seeded-${db}:${tag}-${SHA_SHORT}\` |"
+              done < <(echo "${PRESETS}" | jq -r '.[]')
+            done < <(echo "${DATABASES}" | jq -r '.[]')
+            echo ""
+            echo "Images are built, not pushed. Download them from this run's artifacts."
+          } >> "$GITHUB_STEP_SUMMARY"

util/SeederUtility/.gitignore

@@ -0,0 +1,9 @@
+docker/dp-keys/
+docker/licenses/
+docker/bundles/
+docker/seed.sql
+docker/**/seed.sql
+docker/**/seed.bak
+docker/**/*.mdf
+docker/**/*.ldf
+docker/*/build/

util/SeederUtility/Configuration/ServiceCollectionExtension.cs

@@ -3,6 +3,7 @@
 using Bit.Core.Entities;
 using Bit.Core.Services;
 using Bit.Core.Settings;
+using Bit.Core.Utilities;
 using Bit.Seeder.Pipeline;
 using Bit.Seeder.Services;
 using Bit.SharedWeb.Utilities;
@@ -34,7 +35,11 @@ public static void ConfigureServices(ServiceCollection services, bool enableMang
         services.AddSingleton<IPasswordHasher<User>, PasswordHasher<User>>();
         services.TryAddSingleton<ISeedReader, SeedReader>();
 
-        services.AddDataProtection().SetApplicationName("Bitwarden");
+        var dpBuilder = services.AddDataProtection().SetApplicationName("Bitwarden");
+        if (CoreHelpers.SettingHasValue(globalSettings.DataProtection.Directory))
+        {
+            dpBuilder.PersistKeysToFileSystem(new DirectoryInfo(globalSettings.DataProtection.Directory));
+        }
 
         services.AddAttachmentStorageService(globalSettings);
 

util/SeederUtility/docker/mariadb/Dockerfile

@@ -0,0 +1,16 @@
+FROM mariadb:12
+
+ARG PRESET_NAME=unknown
+ARG PRESET_CATEGORY=unknown
+ARG GIT_SHA=unknown
+ARG BUILD_DATE=unknown
+
+LABEL bitwarden.seeder.preset="${PRESET_NAME}"
+LABEL bitwarden.seeder.category="${PRESET_CATEGORY}"
+LABEL org.opencontainers.image.revision="${GIT_SHA}"
+LABEL org.opencontainers.image.created="${BUILD_DATE}"
+
+ENV MARIADB_DATABASE=vault_dev
+ENV MARIADB_ROOT_PASSWORD=Password1!
+
+COPY seed.sql /docker-entrypoint-initdb.d/seed.sql

util/SeederUtility/docker/mssql/Dockerfile

@@ -0,0 +1,29 @@
+FROM mcr.microsoft.com/mssql/server:2025-CU5-ubuntu-24.04
+
+ARG PRESET_NAME=unknown
+ARG PRESET_CATEGORY=unknown
+ARG GIT_SHA=unknown
+ARG BUILD_DATE=unknown
+
+LABEL bitwarden.seeder.preset="${PRESET_NAME}"
+LABEL bitwarden.seeder.category="${PRESET_CATEGORY}"
+LABEL org.opencontainers.image.revision="${GIT_SHA}"
+LABEL org.opencontainers.image.created="${BUILD_DATE}"
+
+ENV ACCEPT_EULA=Y
+ENV MSSQL_PID=Developer
+
+USER root
+RUN mkdir -p /seed
+COPY vault_dev.mdf /seed/vault_dev.mdf
+COPY vault_dev_log.ldf /seed/vault_dev_log.ldf
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+# Healthy only once the seed is attached, so dependents can wait on it rather than
+# racing the attach and migrating an empty database into place
+HEALTHCHECK --interval=10s --timeout=15s --retries=90 --start-period=30s \
+    CMD /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "${SA_PASSWORD}" -C \
+        -d vault -b -Q "SET NOCOUNT ON; SELECT TOP 1 1 FROM [User]" > /dev/null 2>&1
+
+ENTRYPOINT ["/docker-entrypoint.sh"]

util/SeederUtility/docker/mssql/docker-entrypoint.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+# Starts SQL Server, waits for it to be ready, then attaches the seeded database files.
+set -e
+
+/opt/mssql/bin/sqlservr &
+SQLSERVR_PID=$!
+
+sqlcmd() {
+    /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "${SA_PASSWORD}" -C "$@"
+}
+
+# Polls the given command every 2s, failing the container if it never succeeds
+wait_for() {
+    local what="$1" attempts="$2"
+    shift 2
+    echo "Waiting for ${what}..."
+    for _ in $(seq 1 "${attempts}"); do
+        if "$@"; then
+            echo "${what}: ready."
+            return 0
+        fi
+        sleep 2
+    done
+    echo "ERROR: timed out waiting for ${what}"
+    return 1
+}
+
+accepts_connections() {
+    sqlcmd -Q "SELECT 1" > /dev/null 2>&1
+}
+
+system_databases_online() {
+    local offline
+    offline=$(sqlcmd -h -1 \
+        -Q "SET NOCOUNT ON; SELECT COUNT(*) FROM sys.databases WHERE database_id <= 4 AND state_desc <> 'ONLINE'" \
+        2>/dev/null | tr -d '[:space:]')
+    [ "${offline}" = "0" ]
+}
+
+# Creating and dropping a database proves the data directory is writable
+data_directory_writable() {
+    sqlcmd -Q "CREATE DATABASE [__attach_ready]; DROP DATABASE [__attach_ready]" > /dev/null 2>&1
+}
+
+wait_for "SQL Server connections" 60 accepts_connections
+wait_for "system databases online" 60 system_databases_online
+wait_for "writable data directory" 30 data_directory_writable
+
+DATA_PATH=$(sqlcmd -h -1 \
+    -Q "SET NOCOUNT ON; SELECT CAST(SERVERPROPERTY('InstanceDefaultDataPath') AS NVARCHAR(512))" \
+    2>/dev/null | tr -d '\r\n ')
+echo "MSSQL default data path: ${DATA_PATH}"
+
+database_exists() {
+    local count
+    count=$(sqlcmd -h -1 \
+        -Q "SET NOCOUNT ON; SELECT COUNT(*) FROM sys.databases WHERE name = 'vault'" \
+        2>/dev/null | tr -d '[:space:]')
+    [ "${count}" = "1" ]
+}
+
+database_seeded() {
+    sqlcmd -b -d vault -Q "SET NOCOUNT ON; SELECT TOP 1 1 FROM [User]" > /dev/null 2>&1
+}
+
+# The data directory is usually a mounted volume, so vault survives a restart
+if database_exists; then
+    if ! database_seeded; then
+        echo "ERROR: a 'vault' database exists but holds no seeded data."
+        echo "Something created it before this image could attach the seed. Start the database"
+        echo "and wait for it to report healthy before starting anything that migrates."
+        exit 1
+    fi
+    echo "Database 'vault' is already attached. Leaving it as is."
+else
+    echo "Copying database files to data directory..."
+    cp /seed/vault_dev.mdf "${DATA_PATH}vault.mdf"
+    cp /seed/vault_dev_log.ldf "${DATA_PATH}vault_log.ldf"
+
+    # -b exits non-zero on a T-SQL error so a failed attach does not log success
+    echo "Attaching seeded database..."
+    sqlcmd -b -Q "CREATE DATABASE [vault] ON (FILENAME = '${DATA_PATH}vault.mdf'), (FILENAME = '${DATA_PATH}vault_log.ldf') FOR ATTACH"
+
+    echo "Attach complete."
+fi
+
+wait "${SQLSERVR_PID}"