diff --git a/apps/redis-api/ci/bicep/app/main.bicep b/apps/redis-api/ci/bicep/app/main.bicep index 2c74f5856..8e6f8679f 100644 --- a/apps/redis-api/ci/bicep/app/main.bicep +++ b/apps/redis-api/ci/bicep/app/main.bicep @@ -13,7 +13,7 @@ param sentryEnvironment string param sentryDSN string param sentryEnabled string param sentryTraceSampleRate string -param deleteBatchSize string +param scanBatchSize string param timestamp string = utcNow() @@ -48,7 +48,7 @@ module containerApp 'containerApp.bicep' = { { name: 'SENTRY_ENABLED', value: sentryEnabled } { name: 'SENTRY_TRACE_SAMPLE_RATE', value: sentryTraceSampleRate } { name: 'VERSION', value: version } - { name: 'DELETE_BATCH_SIZE', value: deleteBatchSize } + { name: 'SCAN_BATCH_SIZE', value: scanBatchSize } { name: 'timestamp', value: timestamp } ] diff --git a/apps/redis-api/ci/bicep/main.bicep b/apps/redis-api/ci/bicep/main.bicep index 8fc4ed4f9..883a40c68 100644 --- a/apps/redis-api/ci/bicep/main.bicep +++ b/apps/redis-api/ci/bicep/main.bicep @@ -1,4 +1,4 @@ -import { Environment, EnvironmentVar } from 'types.bicep' +import { Environment } from 'types.bicep' targetScope = 'subscription' @@ -12,7 +12,7 @@ param secondaryApiKey string param sentryDSN string param sentryEnabled string param sentryTraceSampleRate string -param deleteBatchSize int = 2000 +param scanBatchSize int = 2000 @description('The location for the resource group') param location string = 'westeurope' @@ -54,6 +54,6 @@ module containerApp 'app/main.bicep' = { sentryEnabled: sentryEnabled sentryTraceSampleRate: sentryTraceSampleRate version: version - deleteBatchSize: string(deleteBatchSize) + scanBatchSize: string(scanBatchSize) } } diff --git a/apps/redis-api/src/env.ts b/apps/redis-api/src/env.ts index 0ca309d96..8891ce6b3 100644 --- a/apps/redis-api/src/env.ts +++ b/apps/redis-api/src/env.ts @@ -44,7 +44,7 @@ export const env = createEnv({ .refine((s) => s === "true" || s === "false") .transform((s) => s === "true"), SENTRY_TRACE_SAMPLE_RATE: z.coerce.number().default(0.001), - DELETE_BATCH_SIZE: z.coerce.number().default(2000), + SCAN_BATCH_SIZE: z.coerce.number().default(2000), }, createFinalSchema: (shape) => { return z.object(shape).transform((env, ctx) => { diff --git a/apps/redis-api/src/routes/api/cache.ts b/apps/redis-api/src/routes/api/cache.ts index 322950c1a..575896751 100644 --- a/apps/redis-api/src/routes/api/cache.ts +++ b/apps/redis-api/src/routes/api/cache.ts @@ -6,6 +6,7 @@ import { queueDelete, queueDeleteMultiple } from "@/services/redis/queueDelete"; import { loggerModule } from "@/utils/logger"; import { truncate } from "@/utils/truncate"; import { validateKey } from "@/utils/validateKey"; +import { getAllKeys } from "@/services/redis/getAllKeys"; const QUERY_TYPE = t.Object({ key: t.String({}) }); const DELETEMULTIPLE_BODY_TYPE = t.Object({ @@ -47,6 +48,27 @@ export const cacheRoutes = new Elysia({ prefix: "/cache" }) response: { 200: t.Object({ data: t.Any() }), 404: t.String() }, }, ) + .get( + "/all", + async ({ query: { key } }) => { + key = validateKey(key); + const { matchedKeys, totalKeys } = await getAllKeys([`*${key}*`], { + caseInsensitive: true, + }); + + return { data: matchedKeys.toSorted(), totalKeys }; + }, + { + query: QUERY_TYPE, + response: { + 200: t.Object({ + data: t.Array(t.String()), + totalKeys: t.Number(), + }), + 400: t.String(), + }, + }, + ) .put( "/", async ({ query: { key }, body, status, set }) => { diff --git a/apps/redis-api/src/services/redis/getAllKeys.ts b/apps/redis-api/src/services/redis/getAllKeys.ts new file mode 100644 index 000000000..74b443c83 --- /dev/null +++ b/apps/redis-api/src/services/redis/getAllKeys.ts @@ -0,0 +1,98 @@ +import { env } from "@/env"; +import { redis } from "."; +import { timeout } from "@/utils/timeout"; + +/** + * Retrieves all Redis keys that match any of the provided patterns. + * + * This function performs an iterative SCAN operation on the Redis database to retrieve keys + * in batches, avoiding blocking the Redis server. It filters the keys against the provided + * patterns and returns only those that match at least one pattern. + * + * @param patterns - An array of pattern strings to match against Redis keys. + * Keys matching any of these patterns will be included in the result. + * @returns A promise that resolves to an array of matched key strings. + * + * @remarks + * - Uses Redis SCAN command with a cursor-based iteration to handle large keyspaces efficiently + * - Batch size is controlled by `env.SCAN_BATCH_SIZE` + * - Includes a 100ms timeout between batches to prevent overwhelming the Redis server + * - The SCAN operation uses a wildcard "*" match, with additional filtering applied via `matchKey` + * + * @example + * ```typescript + * const keys = await getAllKeys(['user:*', 'session:*']); + * console.log(keys); // ['user:123', 'user:456', 'session:abc'] + * ``` + */ +export async function getAllKeys( + patterns: string[], + options?: { caseInsensitive?: boolean }, +) { + let cursor = "0"; + const SCAN_SIZE = env.SCAN_BATCH_SIZE; + let matchedKeys: string[] = []; + let totalKeys = 0; + do { + const [newCursor, keys] = await redis.scan( + cursor, + "MATCH", + "*", + "COUNT", + SCAN_SIZE, + ); + + cursor = newCursor; + + if (!keys.length) continue; + + totalKeys += keys.length; + + matchedKeys = [ + ...matchedKeys, + ...keys.filter((key) => + patterns.some((pattern) => + matchKey(key, pattern, options?.caseInsensitive), + ), + ), + ]; + + await timeout(100); + } while (cursor !== "0"); + + return { totalKeys, matchedKeys }; +} + +function matchKey( + key: string, + pattern: string, + caseInsensitive = false, +): boolean { + if (caseInsensitive) { + key = key.toLowerCase(); + pattern = pattern.toLowerCase(); + } + + const startsWithWildcard = pattern.startsWith("*"); + const endsWithWildcard = pattern.endsWith("*"); + + const cleanPattern = pattern.replace(/^\*|\*$/g, ""); // remove outer * + + if (!startsWithWildcard && !endsWithWildcard) { + return key === pattern; + } + + if (startsWithWildcard && endsWithWildcard) { + return key.includes(cleanPattern); + } + + if (startsWithWildcard) { + return key.endsWith(cleanPattern); + } + + if (endsWithWildcard) { + return key.startsWith(cleanPattern); + } + + return false; +} diff --git a/apps/redis-api/src/services/redis/queueDelete.ts b/apps/redis-api/src/services/redis/queueDelete.ts index 8214c3dbc..4724da7ff 100644 --- a/apps/redis-api/src/services/redis/queueDelete.ts +++ b/apps/redis-api/src/services/redis/queueDelete.ts @@ -7,6 +7,7 @@ import { loggerModule } from "@/utils/logger"; import { timeout } from "@/utils/timeout"; import { bullmqredis, redis } from "."; +import { getAllKeys } from "./getAllKeys"; const DELETE_JOB = "deleteQueueJob"; const deleteQueueLogger = loggerModule("deleteQueue"); @@ -84,34 +85,7 @@ export async function queueDeleteMultiple({ } async function deleteWithPatterns(patterns: string[]) { - let cursor = "0"; - const SCAN_SIZE = env.DELETE_BATCH_SIZE; - let matchedKeys: string[] = []; - let totalKeys = 0; - do { - const [newCursor, keys] = await redis.scan( - cursor, - "MATCH", - "*", - "COUNT", - SCAN_SIZE, - ); - - cursor = newCursor; - - if (!keys.length) continue; - - totalKeys += keys.length; - - matchedKeys = [ - ...matchedKeys, - ...keys.filter((key) => - patterns.some((pattern) => matchKey(key, pattern)), - ), - ]; - - await timeout(100); - } while (cursor !== "0"); + const { totalKeys, matchedKeys } = await getAllKeys(patterns); let deleted = 0; if (matchedKeys.length > 0) { @@ -125,28 +99,3 @@ async function deleteWithPatterns(patterns: string[]) { return deleted; } - -function matchKey(key: string, pattern: string): boolean { - const startsWithWildcard = pattern.startsWith("*"); - const endsWithWildcard = pattern.endsWith("*"); - - const cleanPattern = pattern.replace(/^\*|\*$/g, ""); // remove outer * - - if (!startsWithWildcard && !endsWithWildcard) { - return key === pattern; - } - - if (startsWithWildcard && endsWithWildcard) { - return key.includes(cleanPattern); - } - - if (startsWithWildcard) { - return key.endsWith(cleanPattern); - } - - if (endsWithWildcard) { - return key.startsWith(cleanPattern); - } - - return false; -}