Merged in feature/redis-api-get-all-keys-endopoint (pull request #3306)

feature: Add getAllKeys endpoint

* feature: Add getAllKeys endpoint

* rename DELETE_BATCH_SIZE to SCAN_BATCH_SIZE


Approved-by: Anton Gunnarsson

apps/redis-api/src/services/redis/getAllKeys.ts

@@ -0,0 +1,98 @@
+import { env } from "@/env";
+import { redis } from ".";
+import { timeout } from "@/utils/timeout";
+
+/**
+ * Retrieves all Redis keys that match any of the provided patterns.
+ *
+ * This function performs an iterative SCAN operation on the Redis database to retrieve keys
+ * in batches, avoiding blocking the Redis server. It filters the keys against the provided
+ * patterns and returns only those that match at least one pattern.
+ *
+ * @param patterns - An array of pattern strings to match against Redis keys.
+ *                   Keys matching any of these patterns will be included in the result.
+ * @returns A promise that resolves to an array of matched key strings.
+ *
+ * @remarks
+ * - Uses Redis SCAN command with a cursor-based iteration to handle large keyspaces efficiently
+ * - Batch size is controlled by `env.SCAN_BATCH_SIZE`
+ * - Includes a 100ms timeout between batches to prevent overwhelming the Redis server
+ * - The SCAN operation uses a wildcard "*" match, with additional filtering applied via `matchKey`
+ *
+ * @example
+ * ```typescript
+ * const keys = await getAllKeys(['user:*', 'session:*']);
+ * console.log(keys); // ['user:123', 'user:456', 'session:abc']
+ * ```
+ */
+export async function getAllKeys(
+    patterns: string[],
+    options?: { caseInsensitive?: boolean },
+) {
+    let cursor = "0";
+    const SCAN_SIZE = env.SCAN_BATCH_SIZE;
+    let matchedKeys: string[] = [];
+    let totalKeys = 0;
+    do {
+        const [newCursor, keys] = await redis.scan(
+            cursor,
+            "MATCH",
+            "*",
+            "COUNT",
+            SCAN_SIZE,
+        );
+
+        cursor = newCursor;
+
+        if (!keys.length) continue;
+
+        totalKeys += keys.length;
+
+        matchedKeys = [
+            ...matchedKeys,
+            ...keys.filter((key) =>
+                patterns.some((pattern) =>
+                    matchKey(key, pattern, options?.caseInsensitive),
+                ),
+            ),
+        ];
+
+        await timeout(100);
+    } while (cursor !== "0");
+
+    return { totalKeys, matchedKeys };
+}
+
+function matchKey(
+    key: string,
+    pattern: string,
+    caseInsensitive = false,
+): boolean {
+    if (caseInsensitive) {
+        key = key.toLowerCase();
+        pattern = pattern.toLowerCase();
+    }
+
+    const startsWithWildcard = pattern.startsWith("*");
+    const endsWithWildcard = pattern.endsWith("*");
+
+    const cleanPattern = pattern.replace(/^\*|\*$/g, ""); // remove outer *
+
+    if (!startsWithWildcard && !endsWithWildcard) {
+        return key === pattern;
+    }
+
+    if (startsWithWildcard && endsWithWildcard) {
+        return key.includes(cleanPattern);
+    }
+
+    if (startsWithWildcard) {
+        return key.endsWith(cleanPattern);
+    }
+
+    if (endsWithWildcard) {
+        return key.startsWith(cleanPattern);
+    }
+
+    return false;
+}

apps/redis-api/src/services/redis/queueDelete.ts

@@ -7,6 +7,7 @@ import { loggerModule } from "@/utils/logger";
 import { timeout } from "@/utils/timeout";
 
 import { bullmqredis, redis } from ".";
+import { getAllKeys } from "./getAllKeys";
 
 const DELETE_JOB = "deleteQueueJob";
 const deleteQueueLogger = loggerModule("deleteQueue");
@@ -84,34 +85,7 @@ export async function queueDeleteMultiple({
 }
 
 async function deleteWithPatterns(patterns: string[]) {
-    let cursor = "0";
-    const SCAN_SIZE = env.DELETE_BATCH_SIZE;
-    let matchedKeys: string[] = [];
-    let totalKeys = 0;
-    do {
-        const [newCursor, keys] = await redis.scan(
-            cursor,
-            "MATCH",
-            "*",
-            "COUNT",
-            SCAN_SIZE,
-        );
-
-        cursor = newCursor;
-
-        if (!keys.length) continue;
-
-        totalKeys += keys.length;
-
-        matchedKeys = [
-            ...matchedKeys,
-            ...keys.filter((key) =>
-                patterns.some((pattern) => matchKey(key, pattern)),
-            ),
-        ];
-
-        await timeout(100);
-    } while (cursor !== "0");
+    const { totalKeys, matchedKeys } = await getAllKeys(patterns);
 
     let deleted = 0;
     if (matchedKeys.length > 0) {
@@ -125,28 +99,3 @@ async function deleteWithPatterns(patterns: string[]) {
 
     return deleted;
 }
-
-function matchKey(key: string, pattern: string): boolean {
-    const startsWithWildcard = pattern.startsWith("*");
-    const endsWithWildcard = pattern.endsWith("*");
-
-    const cleanPattern = pattern.replace(/^\*|\*$/g, ""); // remove outer *
-
-    if (!startsWithWildcard && !endsWithWildcard) {
-        return key === pattern;
-    }
-
-    if (startsWithWildcard && endsWithWildcard) {
-        return key.includes(cleanPattern);
-    }
-
-    if (startsWithWildcard) {
-        return key.endsWith(cleanPattern);
-    }
-
-    if (endsWithWildcard) {
-        return key.startsWith(cleanPattern);
-    }
-
-    return false;
-}