Merged in feat/redis-batch-delete (pull request #2170)

feat: redis - batch delete in smaller chunks

* feat: redis - batch delete in smaller chunks

* fix: redis-api remove unnecessary loop when fuzzily deleting

* increase timeout between delete batches


Approved-by: Joakim Jäderberg

apps/redis-api/src/routes/api/cache.ts

@@ -67,34 +67,27 @@ export const cacheRoutes = new Elysia({ prefix: "/cache" })
     )
     .delete(
         "/",
-        async ({ query: { key, fuzzy }, set }) => {
+        async ({ query: { key, fuzzy } }) => {
             key = validateKey(key);
             cacheRouteLogger.info(
                 `DELETE /cache ${key} ${fuzzy ? "fuzzy" : ""}`
             );
+            const deletedKeys: number = fuzzy
+                ? await deleteWithPattern(`*${key}*`)
+                : await redis.del(key);
 
-            if (fuzzy) {
-                await deleteWithPattern(`*${key}*`);
-            } else {
-                const deletedKeys = await redis.del(key);
-                if (deletedKeys === 0) {
-                    cacheRouteLogger.info(
-                        `Key '${key}' not found, nothing deleted`
-                    );
-                } else {
-                    cacheRouteLogger.info(`Deleted key '${key}'`);
-                }
-            }
-
-            set.status = 204;
-            return undefined;
+            cacheRouteLogger.info(`Deleted ${deletedKeys} keys for '${key}'`);
+            return { deletedKeys };
         },
         {
             query: t.Object({
                 ...QUERY_TYPE.properties,
                 ...t.Object({ fuzzy: t.Optional(t.Boolean()) }).properties,
             }),
-            response: { 204: t.Undefined(), 400: t.String() },
+            response: {
+                200: t.Object({ deletedKeys: t.Number() }),
+                400: t.String(),
+            },
         }
     );
 
@@ -116,7 +109,9 @@ function validateKey(key: string) {
 
 async function deleteWithPattern(pattern: string) {
     let cursor = "0";
-    const keys: string[] = [];
+    const SCAN_SIZE = 500;
+
+    let totalDeleteCount = 0;
 
     do {
         const [newCursor, foundKeys] = await redis.scan(
@@ -124,24 +119,21 @@ async function deleteWithPattern(pattern: string) {
             "MATCH",
             pattern,
             "COUNT",
-            5000
+            SCAN_SIZE
         );
 
-        // Throttle calls to Redis to avoid overwhelming it
-        await timeout(50);
-
         cursor = newCursor;
-        keys.push(...foundKeys);
+        if (foundKeys.length === 0) {
+            continue;
+        }
+
+        const deleteCount = await redis.del(foundKeys);
+
+        cacheRouteLogger.info(`Deleted ${deleteCount} keys in this batch.`);
+        totalDeleteCount += deleteCount;
+
+        await timeout(10);
     } while (cursor !== "0");
 
-    if (keys.length > 0) {
-        const deleteCount = await redis.del(...keys);
-        keys.map((key, idx) => {
-            cacheRouteLogger.info(
-                `Deleted key ${idx + 1}/${deleteCount}: ${key}`
-            );
-        });
-
-        cacheRouteLogger.info(`Deleted number of keys: ${deleteCount}`);
-    }
+    return totalDeleteCount;
 }