Merged in feature/redis-queue-deletes (pull request #2397)

Feature/redis queue deletes

* feat: add queuing for deletes

* merge

* .

* .

* .


Approved-by: Linus Flood

apps/redis-api/src/routes/api/cache.ts

@@ -4,9 +4,8 @@ import { Elysia, t } from "elysia";
 import { ModelValidationError } from "@/errors/ModelValidationError";
 import { redis } from "@/services/redis";
 import { loggerModule } from "@/utils/logger";
-import { timeout } from "@/utils/timeout";
 import { truncate } from "@/utils/truncate";
-import { env } from "@/env";
+import { queueDelete } from "@/services/redis/queueDelete";
 
 const MIN_LENGTH = 1;
 
@@ -73,19 +72,21 @@ export const cacheRoutes = new Elysia({ prefix: "/cache" })
         "/",
         async ({ query: { key, fuzzy } }) => {
             key = validateKey(key);
+            cacheRouteLogger.debug(
+                `DELETE /cache ${key} ${fuzzy ? "fuzzy" : ""}`,
+            );
 
-            const keyToDelete = fuzzy ? `*${key}*` : key;
-            cacheRouteLogger.debug(`DELETE /cache ${keyToDelete}`);
-
+            if (fuzzy) {
+                await queueDelete({ pattern: `*${key}*` });
+                return { status: "queued" };
+            }
             const now = performance.now();
-            const deletedKeys: number = fuzzy
-                ? await deleteWithPattern(keyToDelete)
-                : await redis.del(keyToDelete);
+            const deletedKeys = await redis.unlink(key);
             const elapsed = performance.now() - now;
 
             cacheRouteLogger.info(
-                `Deleted ${deletedKeys} keys for '${keyToDelete}' in ${elapsed}ms`,
-                { fuzzy, deletedKeys, keyToDelete, elapsed },
+                `Deleted ${deletedKeys} keys for '${key}' in ${elapsed}ms`,
+                { fuzzy, deletedKeys, key, elapsed },
             );
 
             return { deletedKeys };
@@ -96,7 +97,10 @@ export const cacheRoutes = new Elysia({ prefix: "/cache" })
                 ...t.Object({ fuzzy: t.Optional(t.Boolean()) }).properties,
             }),
             response: {
-                200: t.Object({ deletedKeys: t.Number() }),
+                200: t.Union([
+                    t.Object({ deletedKeys: t.Number() }),
+                    t.Object({ status: t.Literal("queued") }),
+                ]),
                 400: t.String(),
             },
         },
@@ -117,32 +121,3 @@ function validateKey(key: string) {
 
     return parsedKey;
 }
-
-async function deleteWithPattern(pattern: string) {
-    let cursor = "0";
-    const SCAN_SIZE = env.DELETE_BATCH_SIZE;
-
-    let totalDeleteCount = 0;
-
-    do {
-        const [newCursor, foundKeys] = await redis.scan(
-            cursor,
-            "MATCH",
-            pattern,
-            "COUNT",
-            SCAN_SIZE,
-        );
-
-        cursor = newCursor;
-        if (foundKeys.length === 0) {
-            continue;
-        }
-
-        const deleteCount = await redis.del(foundKeys);
-
-        cacheRouteLogger.debug(`Deleted ${deleteCount} keys in this batch.`);
-        totalDeleteCount += deleteCount;
-    } while (cursor !== "0");
-
-    return totalDeleteCount;
-}