Merged in feat/redis-fix (pull request #3209)

Feat/redis fix

* feat(redis): delete multiple keys in one partition scan

* fix(BOOK-603): make it possible to do multiple deletes in redis at once using one partition scan

* filter out invalid keys

* fix: if no valid keys are present return early

* .

* fix: do redis deletes after scanning through all keys

* fix: do redis deletes after scanning through all keys


Approved-by: Linus Flood

apps/redis-api/src/services/redis/queueDelete.ts

@@ -86,8 +86,8 @@ export async function queueDeleteMultiple({
 async function deleteWithPatterns(patterns: string[]) {
     let cursor = "0";
     const SCAN_SIZE = env.DELETE_BATCH_SIZE;
-    let totalDeleteCount = 0;
-
+    let matchedKeys: string[] = [];
+    let totalKeys = 0;
     do {
         const [newCursor, keys] = await redis.scan(
             cursor,
@@ -101,19 +101,29 @@ async function deleteWithPatterns(patterns: string[]) {
 
         if (!keys.length) continue;
 
-        const matchedKeys = keys.filter((key) =>
-            patterns.some((pattern) => matchKey(key, pattern)),
-        );
+        totalKeys += keys.length;
 
-        if (!matchedKeys.length) continue;
-
-        const deleted = await redis.unlink(...matchedKeys);
-        totalDeleteCount += deleted;
+        matchedKeys = [
+            ...matchedKeys,
+            ...keys.filter((key) =>
+                patterns.some((pattern) => matchKey(key, pattern)),
+            ),
+        ];
 
         await timeout(100);
     } while (cursor !== "0");
 
-    return totalDeleteCount;
+    let deleted = 0;
+    if (matchedKeys.length > 0) {
+        deleted = await redis.unlink(...matchedKeys);
+    }
+
+    deleteQueueLogger.info(
+        `Scanned ${totalKeys} keys, matched ${matchedKeys.length}, deleted ${deleted} keys.`,
+        { totalKeys, matchedKeys: matchedKeys.length, deleted },
+    );
+
+    return deleted;
 }
 
 function matchKey(key: string, pattern: string): boolean {