feat(router): enable using redis clusters for rate limiting and apq (#…

…1499)
wundergraph · Jan 31, 2025 · 7c5b3a7 · 7c5b3a7
1 parent c645f8e
commit 7c5b3a7
Show file tree

Hide file tree

Showing 18 changed files with 533 additions and 72 deletions.
diff --git a/.github/workflows/router-ci.yaml b/.github/workflows/router-ci.yaml
@@ -156,6 +156,16 @@ jobs:
         with:
           cache-dependency-path: |
             router-tests/go.sum
+      - name: Setup Redis Cluster (for Cluster tests)
+        uses: vishnudxb/[email protected]
+        with:
+          master1-port: 7000
+          master2-port: 7001
+          master3-port: 7002
+          slave1-port: 7003
+          slave2-port: 7004
+          slave3-port: 7005
+          sleep-duration: 5
       - uses: nick-fields/retry@v3
         with:
           timeout_minutes: 30

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -243,6 +243,55 @@ services:
     profiles:
       - dev
 
+#  3 node minimum for a cluster, per redis documentation
+  redis-cluster-node-0:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      - primary
+    ports:
+      - "7000:6379"
+    volumes:
+      - ./docker/redis/redis-cluster.conf:/usr/local/etc/redis/redis.conf
+    profiles:
+      - dev
+
+  redis-cluster-node-1:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      - primary
+    ports:
+      - "7001:6379"
+    volumes:
+      - ./docker/redis/redis-cluster.conf:/usr/local/etc/redis/redis.conf
+    profiles:
+      - dev
+
+  redis-cluster-node-2:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: redis-server /usr/local/etc/redis/redis.conf
+    networks:
+      - primary
+    ports:
+      - "7002:6379"
+    volumes:
+      - ./docker/redis/redis-cluster.conf:/usr/local/etc/redis/redis.conf
+    profiles:
+      - dev
+
+  redis-cluster-configure:
+    image: redis:${DC_REDIS_VERSION:-7.2.4}-alpine
+    command: /usr/local/etc/redis/redis-cluster-create.sh
+    networks:
+      - primary
+    depends_on:
+      - redis-cluster-node-0
+      - redis-cluster-node-1
+      - redis-cluster-node-2
+    volumes:
+      - ./docker/redis/:/usr/local/etc/redis/
+
   kafka:
     image: bitnami/kafka:3.7.0
     ports:
@@ -279,3 +328,7 @@ volumes:
   minio:
   redis:
   redis-slave:
+  redis-cluster-configure:
+  redis-cluster-node-0:
+  redis-cluster-node-1:
+  redis-cluster-node-2:
diff --git a/docker/redis/redis-cluster-create.sh b/docker/redis/redis-cluster-create.sh
@@ -0,0 +1,13 @@
+# wait for the docker-compose depends_on to spin up the redis nodes usually takes this long
+sleep 10
+
+node_0_ip=$(getent hosts redis-cluster-node-0 | awk '{ print $1 }')
+node_1_ip=$(getent hosts redis-cluster-node-1 | awk '{ print $1 }')
+node_2_ip=$(getent hosts redis-cluster-node-2 | awk '{ print $1 }')
+
+
+redis-cli --cluster create \
+  $node_0_ip:6379 \
+  $node_1_ip:6379 \
+  $node_2_ip:6379 \
+  --cluster-replicas 0 --cluster-yes
diff --git a/docker/redis/redis-cluster.conf b/docker/redis/redis-cluster.conf
@@ -0,0 +1,9 @@
+cluster-enabled yes
+cluster-config-file nodes.conf
+cluster-node-timeout 5000
+bind 0.0.0.0
+protected-mode no
+maxmemory 100mb
+maxmemory-policy noeviction
+save 60 1
+appendonly no
diff --git a/router-tests/automatic_persisted_queries_test.go b/router-tests/automatic_persisted_queries_test.go
@@ -164,6 +164,7 @@ func TestAutomaticPersistedQueries(t *testing.T) {
 			redisLocalUrl = "localhost:6379"
 			redisUrl      = fmt.Sprintf("redis://%s", redisLocalUrl)
 			redisPassword = "test"
+			client        = redis.NewClient(&redis.Options{Addr: redisLocalUrl, Password: redisPassword})
 		)
 		t.Parallel()
 
@@ -172,18 +173,17 @@ func TestAutomaticPersistedQueries(t *testing.T) {
 
 			key := uuid.New().String()
 			t.Cleanup(func() {
-				client := redis.NewClient(&redis.Options{Addr: redisLocalUrl, Password: redisPassword})
 				del := client.Del(context.Background(), key)
 				require.NoError(t, del.Err())
 			})
 
 			testenv.Run(t, &testenv.Config{
 				RouterOptions: []core.Option{
 					core.WithStorageProviders(config.StorageProviders{
-						Redis: []config.BaseStorageProvider{
+						Redis: []config.RedisStorageProvider{
 							{
-								URL: redisUrl,
-								ID:  "redis",
+								URLs: []string{redisUrl},
+								ID:   "redis",
 							},
 						}})},
 				ApqConfig: config.AutomaticPersistedQueriesConfig{
@@ -206,18 +206,17 @@ func TestAutomaticPersistedQueries(t *testing.T) {
 
 			key := uuid.New().String()
 			t.Cleanup(func() {
-				client := redis.NewClient(&redis.Options{Addr: redisLocalUrl, Password: redisPassword})
 				del := client.Del(context.Background(), key)
 				require.NoError(t, del.Err())
 			})
 
 			testenv.Run(t, &testenv.Config{
 				RouterOptions: []core.Option{
 					core.WithStorageProviders(config.StorageProviders{
-						Redis: []config.BaseStorageProvider{
+						Redis: []config.RedisStorageProvider{
 							{
-								URL: redisUrl,
-								ID:  "redis",
+								URLs: []string{redisUrl},
+								ID:   "redis",
 							},
 						}})},
 				ApqConfig: config.AutomaticPersistedQueriesConfig{
@@ -264,18 +263,17 @@ func TestAutomaticPersistedQueries(t *testing.T) {
 
 			key := uuid.New().String()
 			t.Cleanup(func() {
-				client := redis.NewClient(&redis.Options{Addr: redisLocalUrl, Password: redisPassword})
 				del := client.Del(context.Background(), key)
 				require.NoError(t, del.Err())
 			})
 
 			testenv.Run(t, &testenv.Config{
 				RouterOptions: []core.Option{
 					core.WithStorageProviders(config.StorageProviders{
-						Redis: []config.BaseStorageProvider{
+						Redis: []config.RedisStorageProvider{
 							{
-								URL: redisUrl,
-								ID:  "redis",
+								URLs: []string{redisUrl},
+								ID:   "redis",
 							},
 						}})},
 				ApqConfig: config.AutomaticPersistedQueriesConfig{
@@ -314,18 +312,17 @@ func TestAutomaticPersistedQueries(t *testing.T) {
 
 			key := uuid.New().String()
 			t.Cleanup(func() {
-				client := redis.NewClient(&redis.Options{Addr: redisLocalUrl, Password: redisPassword})
 				del := client.Del(context.Background(), key)
 				require.NoError(t, del.Err())
 			})
 
 			testenv.Run(t, &testenv.Config{
 				RouterOptions: []core.Option{
 					core.WithStorageProviders(config.StorageProviders{
-						Redis: []config.BaseStorageProvider{
+						Redis: []config.RedisStorageProvider{
 							{
-								URL: redisUrl,
-								ID:  "redis",
+								URLs: []string{redisUrl},
+								ID:   "redis",
 							},
 						}})},
 				ApqConfig: config.AutomaticPersistedQueriesConfig{
@@ -366,6 +363,70 @@ func TestAutomaticPersistedQueries(t *testing.T) {
 				require.Equal(t, `{"data":{"__typename":"Query"}}`, res3.Body)
 			})
 		})
+
+		t.Run("works with cluster mode", func(t *testing.T) {
+			t.Parallel()
+
+			clusterUrls := []string{"localhost:7000", "localhost:7001"}
+			clusterClient := redis.NewClusterClient(&redis.ClusterOptions{
+				Addrs:    clusterUrls,
+				Password: redisPassword,
+			})
+
+			key := uuid.New().String()
+			t.Cleanup(func() {
+				del := clusterClient.Del(context.Background(), key)
+				require.NoError(t, del.Err())
+			})
+
+			testenv.Run(t, &testenv.Config{
+				RouterOptions: []core.Option{
+					core.WithStorageProviders(config.StorageProviders{
+						Redis: []config.RedisStorageProvider{
+							{
+								ClusterEnabled: true,
+								URLs:           clusterUrls,
+								ID:             "redis",
+							},
+						}})},
+				ApqConfig: config.AutomaticPersistedQueriesConfig{
+					Enabled: true,
+					Storage: config.AutomaticPersistedQueriesStorageConfig{
+						ProviderID:   "redis",
+						ObjectPrefix: key,
+					},
+				},
+			}, func(t *testing.T, xEnv *testenv.Environment) {
+				header := make(http.Header)
+				header.Add("graphql-client-name", "my-client")
+				res0 := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{
+					Extensions: []byte(`{"persistedQuery": {"version": 1, "sha256Hash": "ecf4edb46db40b5132295c0291d62fb65d6759a9eedfa4d5d612dd5ec54a6b38"}}`),
+					Header:     header,
+				})
+				require.Equal(t, `{"errors":[{"message":"PersistedQueryNotFound","extensions":{"code":"PERSISTED_QUERY_NOT_FOUND"}}]}`, res0.Body)
+
+				res := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{
+					Query:      `{__typename}`,
+					Extensions: []byte(`{"persistedQuery": {"version": 1, "sha256Hash": "ecf4edb46db40b5132295c0291d62fb65d6759a9eedfa4d5d612dd5ec54a6b38"}}`),
+					Header:     header,
+				})
+				require.Equal(t, `{"data":{"__typename":"Query"}}`, res.Body)
+
+				res2 := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{
+					Extensions: []byte(`{"persistedQuery": {"version": 1, "sha256Hash": "ecf4edb46db40b5132295c0291d62fb65d6759a9eedfa4d5d612dd5ec54a6b38"}}`),
+					Header:     header,
+				})
+				require.Equal(t, `{"data":{"__typename":"Query"}}`, res2.Body)
+
+				header2 := make(http.Header)
+				header2.Add("graphql-client-name", "not-my-client")
+				res3 := xEnv.MakeGraphQLRequestOK(testenv.GraphQLRequest{
+					Extensions: []byte(`{"persistedQuery": {"version": 1, "sha256Hash": "ecf4edb46db40b5132295c0291d62fb65d6759a9eedfa4d5d612dd5ec54a6b38"}}`),
+					Header:     header2,
+				})
+				require.Equal(t, `{"data":{"__typename":"Query"}}`, res3.Body)
+			})
+		})
 	})
 }