Split Grafana into public and private instances

apps/monitoring/grafana/helmrelease-private.yaml

@@ -0,0 +1,24 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: grafana-private
+spec:
+  interval: 1m
+  chart:
+    spec:
+      chart: grafana
+      version: "10.5.15"
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+      interval: 1m
+  install:
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+  valuesFrom:
+    - kind: ConfigMap
+      name: grafana-private-config
+      valuesKey: values.yaml

apps/monitoring/grafana/values-private.yaml

@@ -0,0 +1,181 @@
+# Grafana Helm Chart Values — private instance (grafana-private)
+# https://github.com/grafana/helm-charts/tree/main/charts/grafana
+#
+# Admin-only instance: anonymous access is disabled, so nothing is visible
+# without logging in. Serves the private dashboards; the public instance
+# (values.yaml) serves the public ones.
+
+# Persistence: RWO block storage for SQLite. CephFS/RWX cannot reliably
+# provide the file locks SQLite needs and caused recurring CrashLoopBackOff
+# (database is locked / hung /api/health probes).
+persistence:
+  enabled: true
+  type: pvc
+  size: 20Gi
+  storageClassName: ${singlenode_storage_class}
+  accessModes:
+    - ReadWriteOnce
+
+# Ingress configuration
+ingress:
+  enabled: ${enable_ingresses}
+  ingressClassName: public
+  path: /grafana-private
+  pathType: Prefix
+  hosts:
+    - cms.geddes.rcac.purdue.edu
+
+# Service configuration
+service:
+  type: ClusterIP
+  port: 80
+  targetPort: 3000
+
+# Security context
+securityContext:
+  runAsNonRoot: true
+  runAsUser: 472
+  fsGroup: 472
+  fsGroupChangePolicy: OnRootMismatch
+
+# Disable init chown; rely on fsGroup on RBD
+initChownData:
+  enabled: false
+
+# Temp dirs stay on emptyDir. Plugins live on the RWO PVC so they are not
+# reinstalled (and rewritten into SQLite) on every restart.
+extraVolumes:
+  - name: grafana-png-tmp
+    emptyDir: {}
+  - name: grafana-pdf-tmp
+    emptyDir: {}
+  - name: grafana-csv-tmp
+    emptyDir: {}
+
+extraVolumeMounts:
+  - name: grafana-png-tmp
+    mountPath: /var/lib/grafana/png
+  - name: grafana-pdf-tmp
+    mountPath: /var/lib/grafana/pdf
+  - name: grafana-csv-tmp
+    mountPath: /var/lib/grafana/csv
+
+extraConfigmapMounts:
+  - name: grafana-plugins-provisioning
+    mountPath: /etc/grafana/provisioning/plugins
+    configMap: grafana-plugins-provisioning
+    readOnly: true
+
+# Resources
+resources:
+  requests:
+    cpu: 1000m
+    memory: 2Gi
+  limits:
+    cpu: 1000m
+    memory: 2Gi
+
+# RWO + SQLite: never run two pods against the same DB.
+deploymentStrategy:
+  type: Recreate
+
+# Grafana configuration
+grafana.ini:
+  server:
+    root_url: https://cms.geddes.rcac.purdue.edu/grafana-private/
+    serve_from_sub_path: true
+  database:
+    wal: true
+    query_retries: 10
+  dashboards:
+    default_home_dashboard_path: /var/lib/grafana/dashboards/private/users-overview.json
+  auth.anonymous:
+    enabled: false
+
+# Dashboard and datasource provisioning
+dashboardProviders:
+  dashboardproviders.yaml:
+    apiVersion: 1
+    providers:
+      - name: "private"
+        orgId: 1
+        folder: "Private Dashboards"
+        type: file
+        disableDeletion: false
+        updateIntervalSeconds: 10
+        allowUiUpdates: true
+        options:
+          path: /var/lib/grafana/dashboards/private
+
+# Use ConfigMap for dashboards
+dashboardsConfigMaps:
+  private: grafana-private-dashboards
+
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+      - name: prometheus
+        uid: prometheus
+        type: prometheus
+        url: http://prometheus-server:9090
+        access: proxy
+        isDefault: true
+      - name: prometheus-rancher
+        uid: prometheus-rancher
+        type: prometheus
+        url: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090
+        access: proxy
+        isDefault: false
+      - name: prometheus-hammer
+        uid: prometheus-hammer
+        type: prometheus
+        url: http://grafana.cms.rcac.purdue.edu:9090
+        access: proxy
+        isDefault: false
+      - name: tempo
+        uid: tempo
+        type: tempo
+        url: http://tempo.cms.svc.cluster.local:3100
+        access: proxy
+        isDefault: false
+        basicAuth: false
+        jsonData:
+          httpMethod: GET
+          timeInterval: 5s
+          tlsSkipVerify: true
+          nodeGraph:
+            enabled: true
+          serviceMap:
+            datasourceUid: prometheus
+          tracesToMetrics:
+            datasourceUid: prometheus
+          tracesToLogs:
+            datasourceUid: loki
+      - name: thanos
+        uid: thanos
+        type: prometheus
+        url: http://thanos-query.cms.svc.cluster.local:10902
+        access: proxy
+        isDefault: false
+        jsonData:
+          httpMethod: POST
+          timeInterval: 15s
+          manageAlerts: false
+      - name: pyroscope
+        type: grafana-pyroscope-datasource
+        url: http://pyroscope.cms.svc.cluster.local:4040
+        jsonData:
+          minStep: 15s
+      - name: loki
+        uid: loki
+        type: loki
+        url: http://loki.cms.svc.cluster.local:3100
+        access: proxy
+        isDefault: false
+        jsonData:
+          maxLines: 1000
+
+plugins:
+  - grafana-llm-app
+  - grafana-pyroscope-app

apps/monitoring/grafana/values.yaml

@@ -1,5 +1,9 @@
-# Grafana Helm Chart Values
+# Grafana Helm Chart Values — public instance (grafana)
 # https://github.com/grafana/helm-charts/tree/main/charts/grafana
+#
+# Anonymous-viewer instance serving the public dashboards; private
+# dashboards live in the admin-only grafana-private instance
+# (values-private.yaml).
 
 # Persistence: RWO block storage for SQLite. CephFS/RWX cannot reliably
 # provide the file locks SQLite needs and caused recurring CrashLoopBackOff
@@ -178,20 +182,10 @@ dashboardProviders:
         allowUiUpdates: true
         options:
           path: /var/lib/grafana/dashboards/public
-      - name: "private"
-        orgId: 1
-        folder: "Private Dashboards"
-        type: file
-        disableDeletion: false
-        updateIntervalSeconds: 10
-        allowUiUpdates: true
-        options:
-          path: /var/lib/grafana/dashboards/private
 
 # Use ConfigMap for dashboards
 dashboardsConfigMaps:
   public: grafana-public-dashboards
-  private: grafana-private-dashboards
 
 datasources:
   datasources.yaml:

deploy/core-geddes2/kustomization.yaml

@@ -27,6 +27,7 @@ resources:
   - ../../apps/monitoring/af-monitoring/af-node-monitor-rbac.yaml
   - ../../apps/infrastructure/af-node-monitor-storage.yaml
   - ../../apps/monitoring/grafana/helmrelease.yaml
+  - ../../apps/monitoring/grafana/helmrelease-private.yaml
   - ../../apps/monitoring/prometheus/helmrelease.yaml
   - ../../apps/monitoring/prometheus/servicemonitors.yaml
   - ../../apps/monitoring/prometheus/pod-monitor-svc.yaml
@@ -87,6 +88,10 @@ configMapGenerator:
     files:
       - values.yaml=../../apps/monitoring/grafana/values.yaml
 
+  - name: grafana-private-config
+    files:
+      - values.yaml=../../apps/monitoring/grafana/values-private.yaml
+
   - name: grafana-public-dashboards
     files:
       - ../../apps/monitoring/grafana/dashboards/default.json

deploy/core-production/kustomization.yaml

@@ -28,6 +28,7 @@ resources:
   - ../../apps/monitoring/helmrepo-prometheus.yaml
   - ../../apps/monitoring/af-monitoring/services.yaml
   - ../../apps/monitoring/grafana/helmrelease.yaml
+  - ../../apps/monitoring/grafana/helmrelease-private.yaml
   - ../../apps/monitoring/grafana/plugins-provisioning-cm.yaml
   - ../../apps/monitoring/grafana/announcements-nginx-conf.yaml
   - ../../apps/monitoring/prometheus/helmrelease.yaml
@@ -86,6 +87,10 @@ configMapGenerator:
     files:
       - values.yaml=../../apps/monitoring/grafana/values.yaml
 
+  - name: grafana-private-config
+    files:
+      - values.yaml=../../apps/monitoring/grafana/values-private.yaml
+
   - name: grafana-public-dashboards
     files:
       - ../../apps/monitoring/grafana/dashboards/default.json