diff --git a/apps/monitoring/grafana/dashboards/pyroscope.json b/apps/monitoring/grafana/dashboards/pyroscope.json index 704f29bf6..2ab1a865e 100644 --- a/apps/monitoring/grafana/dashboards/pyroscope.json +++ b/apps/monitoring/grafana/dashboards/pyroscope.json @@ -24,7 +24,7 @@ { "datasource": { "type": "grafana-pyroscope-datasource", - "uid": "PAF2BCAF32F82481F" + "uid": "pyroscope" }, "fieldConfig": { "defaults": {}, @@ -43,7 +43,7 @@ { "datasource": { "type": "grafana-pyroscope-datasource", - "uid": "PAF2BCAF32F82481F" + "uid": "pyroscope" }, "groupBy": [], "labelSelector": "", diff --git a/apps/monitoring/grafana/datasources.yaml b/apps/monitoring/grafana/datasources.yaml new file mode 100644 index 000000000..ffac1b519 --- /dev/null +++ b/apps/monitoring/grafana/datasources.yaml @@ -0,0 +1,73 @@ +# Shared Grafana datasource provisioning, merged into both instances +# (grafana and grafana-private) via a second valuesFrom entry on each +# HelmRelease — single source of truth so the instances cannot drift. +# Instance-specific datasources (e.g. Infinity on the public instance) +# live in that instance's values file under a different provisioning +# filename, which the values merge keeps alongside this one. +datasources: + datasources.yaml: + apiVersion: 1 + datasources: + - name: prometheus + uid: prometheus + type: prometheus + url: http://prometheus-server:9090 + access: proxy + isDefault: true + - name: prometheus-rancher + uid: prometheus-rancher + type: prometheus + url: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090 + access: proxy + isDefault: false + - name: prometheus-hammer + uid: prometheus-hammer + type: prometheus + url: http://grafana.cms.rcac.purdue.edu:9090 + access: proxy + isDefault: false + - name: tempo + uid: tempo + type: tempo + url: http://tempo.cms.svc.cluster.local:3100 + access: proxy + isDefault: false + basicAuth: false + jsonData: + httpMethod: GET + timeInterval: 5s + tlsSkipVerify: true + nodeGraph: + enabled: true + serviceMap: + datasourceUid: prometheus + tracesToMetrics: + datasourceUid: prometheus + tracesToLogs: + datasourceUid: loki + - name: thanos + uid: thanos + type: prometheus + url: http://thanos-query.cms.svc.cluster.local:10902 + access: proxy + isDefault: false + jsonData: + httpMethod: POST + timeInterval: 15s + manageAlerts: false + # uid pinned so dashboards can reference it stably across instances + # (a fresh database would otherwise generate a random uid) + - name: pyroscope + uid: pyroscope + type: grafana-pyroscope-datasource + url: http://pyroscope.cms.svc.cluster.local:4040 + jsonData: + minStep: 15s + - name: loki + uid: loki + type: loki + url: http://loki.cms.svc.cluster.local:3100 + access: proxy + isDefault: false + jsonData: + maxLines: 1000 diff --git a/apps/monitoring/grafana/helmrelease-private.yaml b/apps/monitoring/grafana/helmrelease-private.yaml new file mode 100644 index 000000000..56c6c123a --- /dev/null +++ b/apps/monitoring/grafana/helmrelease-private.yaml @@ -0,0 +1,27 @@ +apiVersion: helm.toolkit.fluxcd.io/v2beta1 +kind: HelmRelease +metadata: + name: grafana-private +spec: + interval: 1m + chart: + spec: + chart: grafana + version: "10.5.15" + sourceRef: + kind: HelmRepository + name: grafana + interval: 1m + install: + remediation: + retries: -1 + upgrade: + remediation: + retries: -1 + valuesFrom: + - kind: ConfigMap + name: grafana-private-config + valuesKey: values.yaml + - kind: ConfigMap + name: grafana-datasources + valuesKey: values.yaml diff --git a/apps/monitoring/grafana/helmrelease.yaml b/apps/monitoring/grafana/helmrelease.yaml index b4dd50eba..ecf864bfa 100644 --- a/apps/monitoring/grafana/helmrelease.yaml +++ b/apps/monitoring/grafana/helmrelease.yaml @@ -22,3 +22,6 @@ spec: - kind: ConfigMap name: grafana-config valuesKey: values.yaml + - kind: ConfigMap + name: grafana-datasources + valuesKey: values.yaml diff --git a/apps/monitoring/grafana/plugins-provisioning-cm.yaml b/apps/monitoring/grafana/plugins-provisioning-cm.yaml deleted file mode 100644 index 98e2c7aa9..000000000 --- a/apps/monitoring/grafana/plugins-provisioning-cm.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# configmap.yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: grafana-plugins-provisioning -data: - apps.yaml: | - apiVersion: 1 - apps: - - type: grafana-llm-app - disabled: false - jsonData: - openAI: - # YOUR custom OpenAI-compatible base URL (Azure OpenAI, vLLM, OpenRouter/gateway, etc.) - url: https://genai.rcac.purdue.edu/api - # Optional but recommended: default model “roles” used by features like Explain Flame Graph - modelMappings: - base: gpt-oss-120b - large: gpt-oss:120b - secureJsonData: - # Env var injected by Helm values, interpolated by *Grafana* at - # runtime. The doubled dollar escapes Flux postBuild substitution. - openAIKey: $${OPENAI_API_KEY} diff --git a/apps/monitoring/grafana/values-private.yaml b/apps/monitoring/grafana/values-private.yaml new file mode 100644 index 000000000..5d40aa036 --- /dev/null +++ b/apps/monitoring/grafana/values-private.yaml @@ -0,0 +1,109 @@ +# Grafana Helm Chart Values — private instance (grafana-private) +# https://github.com/grafana/helm-charts/tree/main/charts/grafana +# +# Admin-only instance: anonymous access is disabled, so nothing is visible +# without logging in. Serves the private dashboards; the public instance +# (values.yaml) serves the public ones. + +# Persistence: RWO block storage for SQLite. CephFS/RWX cannot reliably +# provide the file locks SQLite needs and caused recurring CrashLoopBackOff +# (database is locked / hung /api/health probes). +persistence: + enabled: true + type: pvc + size: 20Gi + storageClassName: ${singlenode_storage_class} + accessModes: + - ReadWriteOnce + +# Ingress configuration +ingress: + enabled: ${enable_ingresses} + ingressClassName: public + path: /grafana-private + pathType: Prefix + hosts: + - cms.geddes.rcac.purdue.edu + +# Service configuration +service: + type: ClusterIP + port: 80 + targetPort: 3000 + +# Security context +securityContext: + runAsNonRoot: true + runAsUser: 472 + fsGroup: 472 + fsGroupChangePolicy: OnRootMismatch + +# Disable init chown; rely on fsGroup on RBD +initChownData: + enabled: false + +# Temp dirs stay on emptyDir. +extraVolumes: + - name: grafana-png-tmp + emptyDir: {} + - name: grafana-pdf-tmp + emptyDir: {} + - name: grafana-csv-tmp + emptyDir: {} + +extraVolumeMounts: + - name: grafana-png-tmp + mountPath: /var/lib/grafana/png + - name: grafana-pdf-tmp + mountPath: /var/lib/grafana/pdf + - name: grafana-csv-tmp + mountPath: /var/lib/grafana/csv + +# Resources +resources: + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 1000m + memory: 2Gi + +# RWO + SQLite: never run two pods against the same DB. +deploymentStrategy: + type: Recreate + +# Grafana configuration +grafana.ini: + server: + root_url: https://cms.geddes.rcac.purdue.edu/grafana-private/ + serve_from_sub_path: true + database: + wal: true + query_retries: 10 + dashboards: + default_home_dashboard_path: /var/lib/grafana/dashboards/private/users-overview.json + auth.anonymous: + enabled: false + +# Dashboard and datasource provisioning +dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: "private" + orgId: 1 + folder: "Private Dashboards" + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards/private + +# Use ConfigMap for dashboards +dashboardsConfigMaps: + private: grafana-private-dashboards + +# Datasources come from the shared datasources.yaml (second valuesFrom +# entry on the HelmRelease). The private dashboards use only core panels +# and datasource types, so no plugins are installed on this instance. diff --git a/apps/monitoring/grafana/values.yaml b/apps/monitoring/grafana/values.yaml index 7a9f3cb3e..1edd88fbb 100644 --- a/apps/monitoring/grafana/values.yaml +++ b/apps/monitoring/grafana/values.yaml @@ -1,5 +1,9 @@ -# Grafana Helm Chart Values +# Grafana Helm Chart Values — public instance (grafana) # https://github.com/grafana/helm-charts/tree/main/charts/grafana +# +# Anonymous-viewer instance serving the public dashboards; private +# dashboards live in the admin-only grafana-private instance +# (values-private.yaml). # Persistence: RWO block storage for SQLite. CephFS/RWX cannot reliably # provide the file locks SQLite needs and caused recurring CrashLoopBackOff @@ -51,6 +55,10 @@ extraVolumes: emptyDir: {} - name: grafana-csv-tmp emptyDir: {} + # TRANSITIONAL — empty dir backing the private-dashboards cleanup + # provider (see dashboardProviders); remove together with it. + - name: private-dashboards-cleanup + emptyDir: {} - name: work existingClaim: af-shared-storage - name: announcements-nginx-conf @@ -67,40 +75,43 @@ extraVolumeMounts: mountPath: /var/lib/grafana/pdf - name: grafana-csv-tmp mountPath: /var/lib/grafana/csv + - name: private-dashboards-cleanup + mountPath: /var/lib/grafana/dashboards/private - name: work mountPath: /work/ mountPropagation: HostToContainer readOnly: true -# Add an init container that sets permissive mode on the PVC root without chown -# extraInitContainers: -# - name: init-chmod-data -# image: geddes-registry.rcac.purdue.edu/docker-hub-cache/library/busybox:1.36 -# imagePullPolicy: IfNotPresent -# command: -# - sh -# - -c -# - >- -# mkdir -p /var/lib/grafana && -# chmod -R 0777 /var/lib/grafana || true && -# ls -ld /var/lib/grafana || true -# securityContext: -# runAsUser: 0 -# runAsNonRoot: false -# allowPrivilegeEscalation: false -# readOnlyRootFilesystem: true -# seccompProfile: -# type: RuntimeDefault -# resources: -# requests: -# cpu: 50m -# memory: 64Mi -# limits: -# cpu: 200m -# memory: 128Mi -# volumeMounts: -# - name: storage -# mountPath: /var/lib/grafana +# TRANSITIONAL — remove together with the private-dashboards cleanup +# provider above. GF_INSTALL_PLUGINS only installs plugins; the llm-app +# and pyroscope-app removed from the plugins list stay installed on the +# PVC (and keep loading) until deleted from it. +extraInitContainers: + - name: remove-unused-plugins + image: geddes-registry.rcac.purdue.edu/docker-hub-cache/library/busybox:1.36 + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - >- + rm -rf /var/lib/grafana/plugins/grafana-llm-app + /var/lib/grafana/plugins/grafana-pyroscope-app + securityContext: + runAsNonRoot: true + runAsUser: 472 + allowPrivilegeEscalation: false + seccompProfile: + type: RuntimeDefault + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + cpu: 200m + memory: 128Mi + volumeMounts: + - name: storage + mountPath: /var/lib/grafana extraContainers: | - name: announcements-server @@ -123,18 +134,6 @@ extraContainers: | cpu: 1000m memory: 2Gi -extraConfigmapMounts: - - name: grafana-plugins-provisioning - mountPath: /etc/grafana/provisioning/plugins - configMap: grafana-plugins-provisioning - readOnly: true - -# envValueFrom: -# OPENAI_API_KEY: -# secretKeyRef: -# name: a2rchi-openai-api-key -# key: OPENAI_API_KEY - # Resources resources: requests: @@ -178,6 +177,14 @@ dashboardProviders: allowUiUpdates: true options: path: /var/lib/grafana/dashboards/public + # TRANSITIONAL — remove after this has deployed once. The private + # dashboards moved to the grafana-private instance, but dashboards + # already provisioned into this instance's database are not removed + # by simply dropping their provider. Keeping the provider pointed at + # an empty directory (emptyDir below) makes Grafana's sync delete + # them (disableDeletion: false). Afterwards this provider and the + # private-dashboards-cleanup volume can be deleted; the leftover + # empty "Private Dashboards" folder can be removed in the UI. - name: "private" orgId: 1 folder: "Private Dashboards" @@ -191,72 +198,14 @@ dashboardProviders: # Use ConfigMap for dashboards dashboardsConfigMaps: public: grafana-public-dashboards - private: grafana-private-dashboards +# Shared datasources come from datasources.yaml (second valuesFrom entry +# on the HelmRelease); only public-instance-specific ones live here, under +# a distinct provisioning filename so the values merge keeps both. datasources: - datasources.yaml: + infinity-datasources.yaml: apiVersion: 1 datasources: - - name: prometheus - uid: prometheus - type: prometheus - url: http://prometheus-server:9090 - access: proxy - isDefault: true - - name: prometheus-rancher - uid: prometheus-rancher - type: prometheus - url: http://rancher-monitoring-prometheus.cattle-monitoring-system:9090 - access: proxy - isDefault: false - - name: prometheus-hammer - uid: prometheus-hammer - type: prometheus - url: http://grafana.cms.rcac.purdue.edu:9090 - access: proxy - isDefault: false - - name: tempo - uid: tempo - type: tempo - url: http://tempo.cms.svc.cluster.local:3100 - access: proxy - isDefault: false - basicAuth: false - jsonData: - httpMethod: GET - timeInterval: 5s - tlsSkipVerify: true - nodeGraph: - enabled: true - serviceMap: - datasourceUid: prometheus - tracesToMetrics: - datasourceUid: prometheus - tracesToLogs: - datasourceUid: loki - - name: thanos - uid: thanos - type: prometheus - url: http://thanos-query.cms.svc.cluster.local:10902 - access: proxy - isDefault: false - jsonData: - httpMethod: POST - timeInterval: 15s - manageAlerts: false - - name: pyroscope - type: grafana-pyroscope-datasource - url: http://pyroscope.cms.svc.cluster.local:4040 - jsonData: - minStep: 15s - - name: loki - uid: loki - type: loki - url: http://loki.cms.svc.cluster.local:3100 - access: proxy - isDefault: false - jsonData: - maxLines: 1000 - name: Infinity type: yesoreyeram-infinity-datasource uid: infinity @@ -267,7 +216,5 @@ datasources: - "http://localhost:8080" plugins: - - grafana-llm-app - - grafana-pyroscope-app - marcusolsson-dynamictext-panel - yesoreyeram-infinity-datasource diff --git a/deploy/core-geddes2/kustomization.yaml b/deploy/core-geddes2/kustomization.yaml index b3acbc5a5..b095de52d 100644 --- a/deploy/core-geddes2/kustomization.yaml +++ b/deploy/core-geddes2/kustomization.yaml @@ -27,6 +27,8 @@ resources: - ../../apps/monitoring/af-monitoring/af-node-monitor-rbac.yaml - ../../apps/infrastructure/af-node-monitor-storage.yaml - ../../apps/monitoring/grafana/helmrelease.yaml + - ../../apps/monitoring/grafana/helmrelease-private.yaml + - ../../apps/monitoring/grafana/announcements-nginx-conf.yaml - ../../apps/monitoring/prometheus/helmrelease.yaml - ../../apps/monitoring/prometheus/servicemonitors.yaml - ../../apps/monitoring/prometheus/pod-monitor-svc.yaml @@ -87,6 +89,14 @@ configMapGenerator: files: - values.yaml=../../apps/monitoring/grafana/values.yaml + - name: grafana-private-config + files: + - values.yaml=../../apps/monitoring/grafana/values-private.yaml + + - name: grafana-datasources + files: + - values.yaml=../../apps/monitoring/grafana/datasources.yaml + - name: grafana-public-dashboards files: - ../../apps/monitoring/grafana/dashboards/default.json diff --git a/deploy/core-production/kustomization.yaml b/deploy/core-production/kustomization.yaml index f6c60abe6..12cbcd0fe 100644 --- a/deploy/core-production/kustomization.yaml +++ b/deploy/core-production/kustomization.yaml @@ -28,7 +28,7 @@ resources: - ../../apps/monitoring/helmrepo-prometheus.yaml - ../../apps/monitoring/af-monitoring/services.yaml - ../../apps/monitoring/grafana/helmrelease.yaml - - ../../apps/monitoring/grafana/plugins-provisioning-cm.yaml + - ../../apps/monitoring/grafana/helmrelease-private.yaml - ../../apps/monitoring/grafana/announcements-nginx-conf.yaml - ../../apps/monitoring/prometheus/helmrelease.yaml - ../../apps/monitoring/prometheus/servicemonitors.yaml @@ -86,6 +86,14 @@ configMapGenerator: files: - values.yaml=../../apps/monitoring/grafana/values.yaml + - name: grafana-private-config + files: + - values.yaml=../../apps/monitoring/grafana/values-private.yaml + + - name: grafana-datasources + files: + - values.yaml=../../apps/monitoring/grafana/datasources.yaml + - name: grafana-public-dashboards files: - ../../apps/monitoring/grafana/dashboards/default.json