Split Grafana into public and private instances

apps/monitoring/grafana/dashboards/pyroscope.json

@@ -24,7 +24,7 @@
     {
       "datasource": {
         "type": "grafana-pyroscope-datasource",
-        "uid": "PAF2BCAF32F82481F"
+        "uid": "pyroscope"
       },
       "fieldConfig": {
         "defaults": {},
@@ -43,7 +43,7 @@
         {
           "datasource": {
             "type": "grafana-pyroscope-datasource",
-            "uid": "PAF2BCAF32F82481F"
+            "uid": "pyroscope"
           },
           "groupBy": [],
           "labelSelector": "",

apps/monitoring/grafana/datasources.yaml

@@ -0,0 +1,73 @@
+# Shared Grafana datasource provisioning, merged into both instances
+# (grafana and grafana-private) via a second valuesFrom entry on each
+# HelmRelease — single source of truth so the instances cannot drift.
+# Instance-specific datasources (e.g. Infinity on the public instance)
+# live in that instance's values file under a different provisioning
+# filename, which the values merge keeps alongside this one.
+datasources:
+  datasources.yaml:
+    apiVersion: 1
+    datasources:
+      - name: prometheus
+        uid: prometheus
+        type: prometheus
+        url: http://prometheus-server:9090
+        access: proxy
+        isDefault: true
+      - name: prometheus-rancher
+        uid: prometheus-rancher
+        type: prometheus
+        url: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090
+        access: proxy
+        isDefault: false
+      - name: prometheus-hammer
+        uid: prometheus-hammer
+        type: prometheus
+        url: http://grafana.cms.rcac.purdue.edu:9090
+        access: proxy
+        isDefault: false
+      - name: tempo
+        uid: tempo
+        type: tempo
+        url: http://tempo.cms.svc.cluster.local:3100
+        access: proxy
+        isDefault: false
+        basicAuth: false
+        jsonData:
+          httpMethod: GET
+          timeInterval: 5s
+          tlsSkipVerify: true
+          nodeGraph:
+            enabled: true
+          serviceMap:
+            datasourceUid: prometheus
+          tracesToMetrics:
+            datasourceUid: prometheus
+          tracesToLogs:
+            datasourceUid: loki
+      - name: thanos
+        uid: thanos
+        type: prometheus
+        url: http://thanos-query.cms.svc.cluster.local:10902
+        access: proxy
+        isDefault: false
+        jsonData:
+          httpMethod: POST
+          timeInterval: 15s
+          manageAlerts: false
+      # uid pinned so dashboards can reference it stably across instances
+      # (a fresh database would otherwise generate a random uid)
+      - name: pyroscope
+        uid: pyroscope
+        type: grafana-pyroscope-datasource
+        url: http://pyroscope.cms.svc.cluster.local:4040
+        jsonData:
+          minStep: 15s
+      - name: loki
+        uid: loki
+        type: loki
+        url: http://loki.cms.svc.cluster.local:3100
+        access: proxy
+        isDefault: false
+        jsonData:
+          maxLines: 1000

apps/monitoring/grafana/helmrelease-private.yaml

@@ -0,0 +1,27 @@
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: grafana-private
+spec:
+  interval: 1m
+  chart:
+    spec:
+      chart: grafana
+      version: "10.5.15"
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+      interval: 1m
+  install:
+    remediation:
+      retries: -1
+  upgrade:
+    remediation:
+      retries: -1
+  valuesFrom:
+    - kind: ConfigMap
+      name: grafana-private-config
+      valuesKey: values.yaml
+    - kind: ConfigMap
+      name: grafana-datasources
+      valuesKey: values.yaml

apps/monitoring/grafana/helmrelease.yaml

@@ -22,3 +22,6 @@ spec:
     - kind: ConfigMap
       name: grafana-config
       valuesKey: values.yaml
+    - kind: ConfigMap
+      name: grafana-datasources
+      valuesKey: values.yaml

apps/monitoring/grafana/values-private.yaml

@@ -0,0 +1,109 @@
+# Grafana Helm Chart Values — private instance (grafana-private)
+# https://github.com/grafana/helm-charts/tree/main/charts/grafana
+#
+# Admin-only instance: anonymous access is disabled, so nothing is visible
+# without logging in. Serves the private dashboards; the public instance
+# (values.yaml) serves the public ones.
+
+# Persistence: RWO block storage for SQLite. CephFS/RWX cannot reliably
+# provide the file locks SQLite needs and caused recurring CrashLoopBackOff
+# (database is locked / hung /api/health probes).
+persistence:
+  enabled: true
+  type: pvc
+  size: 20Gi
+  storageClassName: ${singlenode_storage_class}
+  accessModes:
+    - ReadWriteOnce
+
+# Ingress configuration
+ingress:
+  enabled: ${enable_ingresses}
+  ingressClassName: public
+  path: /grafana-private
+  pathType: Prefix
+  hosts:
+    - cms.geddes.rcac.purdue.edu
+
+# Service configuration
+service:
+  type: ClusterIP
+  port: 80
+  targetPort: 3000
+
+# Security context
+securityContext:
+  runAsNonRoot: true
+  runAsUser: 472
+  fsGroup: 472
+  fsGroupChangePolicy: OnRootMismatch
+
+# Disable init chown; rely on fsGroup on RBD
+initChownData:
+  enabled: false
+
+# Temp dirs stay on emptyDir.
+extraVolumes:
+  - name: grafana-png-tmp
+    emptyDir: {}
+  - name: grafana-pdf-tmp
+    emptyDir: {}
+  - name: grafana-csv-tmp
+    emptyDir: {}
+
+extraVolumeMounts:
+  - name: grafana-png-tmp
+    mountPath: /var/lib/grafana/png
+  - name: grafana-pdf-tmp
+    mountPath: /var/lib/grafana/pdf
+  - name: grafana-csv-tmp
+    mountPath: /var/lib/grafana/csv
+
+# Resources
+resources:
+  requests:
+    cpu: 1000m
+    memory: 2Gi
+  limits:
+    cpu: 1000m
+    memory: 2Gi
+
+# RWO + SQLite: never run two pods against the same DB.
+deploymentStrategy:
+  type: Recreate
+
+# Grafana configuration
+grafana.ini:
+  server:
+    root_url: https://cms.geddes.rcac.purdue.edu/grafana-private/
+    serve_from_sub_path: true
+  database:
+    wal: true
+    query_retries: 10
+  dashboards:
+    default_home_dashboard_path: /var/lib/grafana/dashboards/private/users-overview.json
+  auth.anonymous:
+    enabled: false
+
+# Dashboard and datasource provisioning
+dashboardProviders:
+  dashboardproviders.yaml:
+    apiVersion: 1
+    providers:
+      - name: "private"
+        orgId: 1
+        folder: "Private Dashboards"
+        type: file
+        disableDeletion: false
+        updateIntervalSeconds: 10
+        allowUiUpdates: true
+        options:
+          path: /var/lib/grafana/dashboards/private
+
+# Use ConfigMap for dashboards
+dashboardsConfigMaps:
+  private: grafana-private-dashboards
+
+# Datasources come from the shared datasources.yaml (second valuesFrom
+# entry on the HelmRelease). The private dashboards use only core panels
+# and datasource types, so no plugins are installed on this instance.