From bb393f2b39504367338a681d7ce2b1b2bd022ef7 Mon Sep 17 00:00:00 2001 From: Rick Henry Date: Wed, 22 Apr 2026 10:17:54 +0100 Subject: [PATCH 1/4] feat: move tsdb retention config to file for prom>=3.11 Signed-off-by: Rick Henry --- roles/prometheus/templates/prometheus.service.j2 | 4 ++++ roles/prometheus/templates/prometheus.yml.j2 | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/roles/prometheus/templates/prometheus.service.j2 b/roles/prometheus/templates/prometheus.service.j2 index 17a556d24..286c4cfb0 100644 --- a/roles/prometheus/templates/prometheus.service.j2 +++ b/roles/prometheus/templates/prometheus.service.j2 @@ -16,12 +16,16 @@ ExecReload=/bin/kill -HUP $MAINPID ExecStart={{ prometheus_binary_install_dir }}/prometheus \ {% if not prometheus_agent_mode %} --storage.tsdb.path={{ prometheus_db_dir }} \ +{# If using Prometheus 3.11 or newer, don't set the TSDB retention flags at #} +{# all, instead use the config file. #} +{% if prometheus_version is version('3.11.0', '<') %} {% if prometheus_version is version('2.7.0', '>=') %} --storage.tsdb.retention.time={{ prometheus_storage_retention }} \ --storage.tsdb.retention.size={{ prometheus_storage_retention_size }} \ {% else %} --storage.tsdb.retention={{ prometheus_storage_retention }} \ {% endif %} +{% endif %} {% else %} {% if prometheus_version is version('3.0.0', '>=') %} --agent \ diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index 8f569165d..9ccda82d2 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -7,6 +7,14 @@ global: external_labels: {{ prometheus_external_labels | to_nice_yaml(indent=2,sort_keys=False) | indent(4, False) }} +{% if prometheus_version is version('3.11.0', '>=') %} +storage: + tsdb: + retention: + time: {{ prometheus_storage_retention }} + size: {{ prometheus_storage_retention_size }} +{% endif %} + {% if prometheus_remote_write != [] %} remote_write: {{ prometheus_remote_write | to_nice_yaml(indent=2,sort_keys=False) | indent(2, False) }} From 721ab8dd89c689d2fcf7d7755b712c1be1a52002 Mon Sep 17 00:00:00 2001 From: Rick Henry Date: Wed, 22 Apr 2026 10:28:30 +0100 Subject: [PATCH 2/4] feat: add prometheus_storage_retention_percentage This adds a variable to configure Prometheus' percentage-based retention. This feature was added in 3.11.0. Signed-off-by: Rick Henry --- roles/prometheus/defaults/main.yml | 4 ++++ roles/prometheus/meta/argument_specs.yml | 6 ++++++ roles/prometheus/templates/prometheus.yml.j2 | 1 + 3 files changed, 11 insertions(+) diff --git a/roles/prometheus/defaults/main.yml b/roles/prometheus/defaults/main.yml index 8d31598fc..a302b0fb4 100644 --- a/roles/prometheus/defaults/main.yml +++ b/roles/prometheus/defaults/main.yml @@ -23,6 +23,10 @@ prometheus_storage_retention: "30d" # [EXPERIMENTAL] Maximum number of bytes that can be stored for blocks. Units # supported: KB, MB, GB, TB, PB. prometheus_storage_retention_size: "0" +# Available since Prometheus 3.11.0 +# [EXPERIMENTAL] Maximum percentage of total disk space allowed for block +# storage. Set to 0 to disable percentage-based retention. +prometheus_storage_retention_percentage: "0" # The Agent mode optimizes Prometheus for the remote write use case: https://prometheus.io/blog/2021/11/16/agent/ prometheus_agent_mode: false diff --git a/roles/prometheus/meta/argument_specs.yml b/roles/prometheus/meta/argument_specs.yml index bff650693..e9a545208 100644 --- a/roles/prometheus/meta/argument_specs.yml +++ b/roles/prometheus/meta/argument_specs.yml @@ -55,6 +55,12 @@ argument_specs: - "Maximum number of bytes that can be stored for blocks." - "Units supported: KB, MB, GB, TB, PB." default: "0" + prometheus_storage_retention_percentage: + description: + - "Maximum percent of total disk space allowed for block storage" + - "Set to 0 to disable percentage-based retention" + - "Supported only on Prometheus >= 3.11.0" + default: "0" prometheus_agent_mode: description: - "The Agent mode optimizes Prometheus for the remote write use case. It disables querying, alerting, and local storage, and replaces it with a customized TSDB WAL. L(Everything,https://prometheus.io/docs/prometheus/latest/feature_flags/#prometheus-agent) else stays the same." diff --git a/roles/prometheus/templates/prometheus.yml.j2 b/roles/prometheus/templates/prometheus.yml.j2 index 9ccda82d2..bbc7486a1 100644 --- a/roles/prometheus/templates/prometheus.yml.j2 +++ b/roles/prometheus/templates/prometheus.yml.j2 @@ -13,6 +13,7 @@ storage: retention: time: {{ prometheus_storage_retention }} size: {{ prometheus_storage_retention_size }} + percentage: {{ prometheus_storage_retention_percentage }} {% endif %} {% if prometheus_remote_write != [] %} From 27301048e420c58492a8f629161a09150580a9b7 Mon Sep 17 00:00:00 2001 From: Rick Henry Date: Fri, 10 Jul 2026 09:44:12 +0100 Subject: [PATCH 3/4] ci: update Prometheus alt to modern Prometheus version The `--alertmanager.timeout` flag was deprecated in 2.25.0 and removed in 3.0.0, hence the removal here. Signed-off-by: Rick Henry --- roles/prometheus/molecule/alternative/molecule.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/roles/prometheus/molecule/alternative/molecule.yml b/roles/prometheus/molecule/alternative/molecule.yml index dc9d00e88..39c9991dc 100644 --- a/roles/prometheus/molecule/alternative/molecule.yml +++ b/roles/prometheus/molecule/alternative/molecule.yml @@ -13,7 +13,6 @@ provisioner: prometheus_storage_retention: "60d" prometheus_storage_retention_size: "1GB" prometheus_config_flags_extra: - alertmanager.timeout: 10s web.enable-admin-api: enable-feature: - promql-at-modifier @@ -88,7 +87,7 @@ provisioner: target_label: instance - target_label: __address__ replacement: 127.0.0.1:9115 # Blackbox exporter. - prometheus_version: 2.25.2 + prometheus_version: 3.13.0 prometheus_stop_timeout: 1min prometheus_binary_url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version\ \ }}/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz" From bb78acf21cb04699ebe1774af2a7331430586332 Mon Sep 17 00:00:00 2001 From: Rick Henry Date: Fri, 10 Jul 2026 10:11:30 +0100 Subject: [PATCH 4/4] ci: add storage percentage for Prometheus alt test Signed-off-by: Rick Henry --- roles/prometheus/molecule/alternative/molecule.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/prometheus/molecule/alternative/molecule.yml b/roles/prometheus/molecule/alternative/molecule.yml index 39c9991dc..1add7ae3d 100644 --- a/roles/prometheus/molecule/alternative/molecule.yml +++ b/roles/prometheus/molecule/alternative/molecule.yml @@ -12,6 +12,7 @@ provisioner: - /etc prometheus_storage_retention: "60d" prometheus_storage_retention_size: "1GB" + prometheus_storage_retention_percentage: 70 prometheus_config_flags_extra: web.enable-admin-api: enable-feature: