chore: add test scenarios and docs for redis read replicas

docker-compose.yml

@@ -336,6 +336,65 @@ services:
     profiles:
       - dev
 
+  # Opt-in replica nodes, one per master, enabled with the redis-cluster-replicas profile:
+  #   docker compose --profile dev --profile redis-cluster-replicas up -d redis-cluster-configure
+  # redis-cluster-create.sh picks up whichever of these are running and joins them with
+  # --cluster-replicas 1, which is what makes read_only=true routing testable locally. Without the
+  # profile the cluster stays three masters with no replicas.
+  redis-cluster-node-4:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      - primary
+    ports:
+      - '7004:6379'
+      - '16374:16379'
+    volumes:
+      - ./docker/redis/redis-cluster.conf:/usr/local/etc/redis/redis.conf
+    healthcheck:
+      test: ['CMD', 'redis-cli', '-p', '6379', 'ping']
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    profiles:
+      - redis-cluster-replicas
+
+  redis-cluster-node-5:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      - primary
+    ports:
+      - '7005:6379'
+      - '16375:16379'
+    volumes:
+      - ./docker/redis/redis-cluster.conf:/usr/local/etc/redis/redis.conf
+    healthcheck:
+      test: ['CMD', 'redis-cli', '-p', '6379', 'ping']
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    profiles:
+      - redis-cluster-replicas
+
+  redis-cluster-node-6:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      - primary
+    ports:
+      - '7006:6379'
+      - '16376:16379'
+    volumes:
+      - ./docker/redis/redis-cluster.conf:/usr/local/etc/redis/redis.conf
+    healthcheck:
+      test: ['CMD', 'redis-cli', '-p', '6379', 'ping']
+      interval: 10s
+      timeout: 5s
+      retries: 3
+    profiles:
+      - redis-cluster-replicas
+
   redis-cluster-configure:
     image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
     command: /usr/local/etc/redis/redis-cluster-create.sh
@@ -348,6 +407,18 @@ services:
         condition: service_healthy
       redis-cluster-node-3:
         condition: service_healthy
+      redis-cluster-node-4:
+        condition: service_healthy
+        # Absent unless the redis-cluster-replicas profile is enabled.
+        required: false
+      redis-cluster-node-5:
+        condition: service_healthy
+        # Absent unless the redis-cluster-replicas profile is enabled.
+        required: false
+      redis-cluster-node-6:
+        condition: service_healthy
+        # Absent unless the redis-cluster-replicas profile is enabled.
+        required: false
     volumes:
       - ./docker/redis/:/usr/local/etc/redis/
     restart: on-failure:1
@@ -433,4 +504,7 @@ volumes:
   redis-cluster-node-1:
   redis-cluster-node-2:
   redis-cluster-node-3:
+  redis-cluster-node-4:
+  redis-cluster-node-5:
+  redis-cluster-node-6:
   plugin-registry:

docker/redis/redis-cluster-create.sh

@@ -1,23 +1,56 @@
 # wait for the docker-compose depends_on to spin up the redis nodes usually takes this long
 sleep 10
 
-node_1_ip=$(getent hosts redis-cluster-node-1 | awk '{ print $1 }')
-node_2_ip=$(getent hosts redis-cluster-node-2 | awk '{ print $1 }')
-node_3_ip=$(getent hosts redis-cluster-node-3 | awk '{ print $1 }')
+resolve() {
+  getent hosts "$1" | awk '{ print $1 }'
+}
+
+master_ips=""
+for node in redis-cluster-node-1 redis-cluster-node-2 redis-cluster-node-3; do
+  ip=$(resolve $node)
+  if [ -z "$ip" ]; then
+    echo "$node did not resolve, cannot create the cluster"
+    exit 1
+  fi
+  master_ips="$master_ips $ip"
+done
+
+# Nodes 4-6 only run when the redis-cluster-replicas compose profile is enabled, so they are
+# picked up when they resolve and left out of the cluster otherwise.
+replica_ips=""
+replica_count=0
+for node in redis-cluster-node-4 redis-cluster-node-5 redis-cluster-node-6; do
+  ip=$(resolve $node)
+  if [ -z "$ip" ]; then
+    continue
+  fi
+  replica_ips="$replica_ips $ip"
+  replica_count=$((replica_count + 1))
+done
+
+# redis-cli spreads replicas evenly over the masters, so it takes one each or none at all.
+replicas_per_master=0
+if [ "$replica_count" -eq 3 ]; then
+  replicas_per_master=1
+elif [ "$replica_count" -ne 0 ]; then
+  echo "Only $replica_count of the 3 replica nodes are running, creating the cluster without replicas"
+  replica_ips=""
+fi
+
+node_ips="$master_ips $replica_ips"
 
 # Prepare the nodes for the cluster
-for ip in $node_1_ip $node_2_ip $node_3_ip; do
+for ip in $node_ips; do
   echo "Emptying db 0 of Redis node at $ip and resetting cluster"
   redis-cli -h $ip -p 6379 FLUSHDB
   redis-cli -h $ip -p 6379 CLUSTER RESET
   redis-cli -h $ip -p 6379 CONFIG SET cluster-announce-ip "$ip"
 done
 
-# Create the cluster
+# Create the cluster. The masters come first, so the replica nodes that follow become their
+# replicas, which is what read_only=true routing needs to be exercised.
 redis-cli --cluster create \
-  $node_1_ip:6379 \
-  $node_2_ip:6379 \
-  $node_3_ip:6379 \
-  --cluster-replicas 0 --cluster-yes
+  $(for ip in $node_ips; do printf '%s:6379 ' "$ip"; done) \
+  --cluster-replicas $replicas_per_master --cluster-yes
 
-echo "Redis Cluster setup complete!"
+echo "Redis Cluster setup complete!"

docs-website/router/configuration.mdx

@@ -1321,9 +1321,9 @@ These apply when `cluster_enabled: true`.
 | Parameter        | Description                                                                                | Default Value |
 | ---------------- | ------------------------------------------------------------------------------------------ | ------------- |
 | max_redirects    | Number of MOVED/ASK redirects to follow before giving up on a command.                      | 3             |
-| read_only        | Allow routing read-only commands to replicas.                                               | false         |
-| route_by_latency | Route read-only commands to the closest node by measured latency. Implies `read_only`.      | false         |
-| route_randomly   | Route read-only commands to a random node. Implies `read_only`.                             | false         |
+| read_only        | Route read-only commands to a replica of the shard that owns the key, instead of its master. Requires a cluster that has replicas, see [Reading from replicas](/router/configuration#reading-from-replicas). | false         |
+| route_by_latency | Route read-only commands to whichever node of the shard has the lowest measured latency, the master included. Implies `read_only`. | false         |
+| route_randomly   | Route read-only commands to a random node of the shard, the master included. Implies `read_only`. | false         |
 
 <Warning>
   With `cluster_enabled: true`, query parameters are only read from the **first** URL in the list. The router uses the remaining URLs as additional cluster seed addresses and takes only their host and port, so parameters placed on them are silently dropped. Put all connection options on the first URL.
@@ -1342,6 +1342,35 @@ These apply when `cluster_enabled: true`.
   Setting `max_active_conns` lower than `pool_size` therefore admits more concurrent commands than there are connections available for, and the excess fails instead of queueing. Keep `max_active_conns` at or above `pool_size`, or leave it unset.
 </Warning>
 
+### Reading from replicas
+
+`read_only=true` sends read-only commands to a replica of the shard that owns the key instead of to its master, which spreads read traffic over more nodes. It is a Redis Cluster feature, so it needs `cluster_enabled: true`.
+
+```yaml config.yaml
+storage_providers:
+  redis:
+    - id: "redis-provider"
+      urls:
+        - "redis://localhost:7001?read_only=true" # options read from the first URL only
+        - "redis://localhost:7002"
+        - "redis://localhost:7003"
+      cluster_enabled: true
+```
+
+Only commands that Redis itself flags as read-only are routed this way. Writes always go to the master, and so do Lua scripts, because `EVALSHA` carries no read-only flag. The rate limiter's counter script therefore keeps running on the master and is unaffected by this setting, while features that read with a plain `GET`, such as automatic persisted queries, start reading from replicas.
+
+The cluster has to actually have replicas for the setting to do anything. Against a cluster of masters only it changes nothing, and a shard whose replica is unreachable falls back to its master. Reads keep working in both cases, they just are not distributed, and neither case is reported as an error.
+
+<Warning>
+  Replication is asynchronous, so a replica can answer with an older value than the master holds, or miss a key entirely for a moment after it is written. A `GET` that immediately follows its `SET` may return the previous value or nothing at all. For automatic persisted queries that means an operation can look unregistered on the request right after it was registered. Only enable `read_only` where a stale read is acceptable.
+</Warning>
+
+`route_by_latency` and `route_randomly` both imply `read_only`, but they choose among all nodes of the shard, the master included. They distribute reads without guaranteeing that a replica serves them, and `route_by_latency` in particular often settles on the master when it is the closest node. Plain `read_only` always prefers a replica.
+
+<Note>
+  These three parameters are only recognised on a cluster URL. Without `cluster_enabled: true` the router fails at startup with `redis: unexpected option: read_only`.
+</Note>
+
 ### Sizing the pool
 
 For a Redis Cluster these limits apply **per node**, not to the cluster as a whole: `pool_size=20` against a six-node cluster permits up to 20 connections to each node.