Merged in feature/redis (pull request #1478)

Distributed cache

* cache deleteKey now uses an options object instead of a lonely argument variable fuzzy

* merge

* remove debug logs and cleanup

* cleanup

* add fault handling

* add fault handling

* add pid when logging redis client creation

* add identifier when logging redis client creation

* cleanup

* feat: add redis-api as it's own app

* feature: use http wrapper for redis

* feat: add the possibility to fallback to unstable_cache

* Add error handling if redis cache is unresponsive

* add logging for unstable_cache

* merge

* don't cache errors

* fix: metadatabase on branchdeploys

* Handle when /en/destinations throws
add ErrorBoundary

* Add sentry-logging when ErrorBoundary catches exception

* Fix error handling for distributed cache

* cleanup code

* Added Application Insights back

* Update generateApiKeys script and remove duplicate

* Merge branch 'feature/redis' of bitbucket.org:scandic-swap/web into feature/redis

* merge


Approved-by: Linus Flood

apps/redis-api/src/env.ts

@@ -0,0 +1,56 @@
+import { createEnv } from "@t3-oss/env-core";
+import { z } from "zod";
+
+const redisConnectionRegex =
+    /^((?<username>.*?):(?<password>.*?)@)?(?<host>.*?):(?<port>\d+)$/;
+export const env = createEnv({
+    server: {
+        IS_PROD: z
+            .boolean()
+            .default(false)
+            .transform(
+                () =>
+                    process.env.BUN_ENV === "production" ||
+                    process.env.NODE_ENV === "production"
+            ),
+        IS_DEV: z
+            .boolean()
+            .default(false)
+            .transform(
+                () =>
+                    process.env.BUN_ENV === "development" ||
+                    process.env.NODE_ENV === "development"
+            ),
+        VERSION: z.string().min(1).default("development"),
+        PORT: z.coerce.number().default(3001),
+        REDIS_CONNECTION: z.string().regex(redisConnectionRegex),
+
+        PRIMARY_API_KEY:
+            process.env.NODE_ENV === "development"
+                ? z.string().optional()
+                : z.string().min(10),
+
+        SECONDARY_API_KEY:
+            process.env.NODE_ENV === "development"
+                ? z.string().optional()
+                : z.string().min(10),
+    },
+    runtimeEnv: {
+        ...process.env,
+    },
+});
+
+const redisMatch = env.REDIS_CONNECTION.match(redisConnectionRegex);
+if (!redisMatch?.groups) {
+    throw new Error("Invalid REDIS_CONNECTION format");
+}
+
+export const redisConfig = {
+    host: redisMatch.groups.host,
+    port: Number(redisMatch.groups.port),
+    username: redisMatch.groups.username,
+    password: redisMatch.groups.password,
+};
+
+console.log("env", env);
+console.log("redisConfig", redisConfig);

apps/redis-api/src/errors/AuthenticationError.ts

@@ -0,0 +1,6 @@
+export class AuthenticationError extends Error {
+    constructor(public message: string) {
+        super(message);
+        this.name = "AuthenticationError";
+    }
+}

apps/redis-api/src/errors/ModelValidationError.ts

@@ -0,0 +1,6 @@
+export class ModelValidationError extends Error {
+    constructor(public message: string) {
+        super(message);
+        this.name = "ModelValidationError";
+    }
+}

apps/redis-api/src/index.ts

@@ -0,0 +1,59 @@
+import { Elysia } from "elysia";
+
+import { swagger } from "@elysiajs/swagger";
+import { apiRoutes } from "@/routes/api";
+import { healthRoutes } from "@/routes/health";
+import { baseLogger } from "@/utils/logger";
+import { env } from "@/env";
+import serverTiming from "@elysiajs/server-timing";
+import { AuthenticationError } from "@/errors/AuthenticationError";
+import { ModelValidationError } from "@/errors/ModelValidationError";
+
+const app = new Elysia()
+    .use(serverTiming())
+    .error("AUTHENTICATION_ERROR", AuthenticationError)
+    .error("MODEL_VALIDATION_ERROR", ModelValidationError)
+
+    .onError(({ code, error, set }) => {
+        switch (code) {
+            case "MODEL_VALIDATION_ERROR":
+                set.status = 400;
+                return getErrorReturn(error);
+            case "AUTHENTICATION_ERROR":
+                set.status = 401;
+                return getErrorReturn(error);
+            case "NOT_FOUND":
+                set.status = 404;
+                return getErrorReturn(error);
+            case "INTERNAL_SERVER_ERROR":
+                set.status = 500;
+                return getErrorReturn(error);
+        }
+    });
+
+if (env.IS_DEV) {
+    app.use(
+        swagger({
+            documentation: {
+                info: {
+                    title: "Redis API",
+                    version: "1.0.0",
+                },
+            },
+        })
+    );
+}
+
+app.use(apiRoutes);
+app.use(healthRoutes);
+
+app.listen(env.PORT, (server) => {
+    baseLogger.info(`🦊 REDISAPI@${env.VERSION} running on ${server.url}`);
+});
+
+function getErrorReturn(error: Error) {
+    return {
+        status: "error",
+        message: error.toString(),
+    };
+}

apps/redis-api/src/middleware/apiKeyMiddleware.ts

@@ -0,0 +1,28 @@
+import { AuthenticationError } from "@/errors/AuthenticationError";
+import type { Context } from "elysia";
+import { env } from "@/env";
+
+const API_KEY_HEADER = "x-api-key";
+
+export const apiKeyMiddleware = ({ headers }: Context) => {
+    if (!isApiKeyRequired()) {
+        return;
+    }
+
+    const apiKey = headers[API_KEY_HEADER];
+    if (!apiKey) {
+        throw new AuthenticationError("No API KEY provided");
+    }
+
+    if (!validateApiKey(apiKey)) {
+        throw new AuthenticationError("Invalid API key");
+    }
+};
+
+function isApiKeyRequired(): boolean {
+    return Boolean(env.PRIMARY_API_KEY) || Boolean(env.SECONDARY_API_KEY);
+}
+
+function validateApiKey(apiKey: string): boolean {
+    return apiKey === env.PRIMARY_API_KEY || apiKey === env.SECONDARY_API_KEY;
+}

apps/redis-api/src/routes/api/cache.ts

@@ -0,0 +1,93 @@
+import { Elysia, t, ValidationError } from "elysia";
+import { redis } from "@/services/redis";
+import { ModelValidationError } from "@/errors/ModelValidationError";
+
+const MIN_LENGTH = 1;
+
+const QUERY_TYPE = t.Object({ key: t.String({ minLength: MIN_LENGTH }) });
+
+export const cacheRoutes = new Elysia({ prefix: "/cache" })
+    .get(
+        "/",
+        async ({ query: { key }, error }) => {
+            key = validateKey(key);
+            console.log("GET /cache", key);
+
+            const value = await redis.get(key);
+            if (!value) {
+                return error("Not Found", "Not Found");
+            }
+
+            try {
+                const output = JSON.parse(value);
+                return { data: output };
+            } catch (e) {
+                redis.del(key);
+                throw e;
+            }
+        },
+        {
+            query: QUERY_TYPE,
+            response: { 200: t.Object({ data: t.Any() }), 404: t.String() },
+        }
+    )
+    .put(
+        "/",
+        async ({ query: { key }, body, error, set }) => {
+            key = validateKey(key);
+            console.log("PUT /cache", key);
+
+            if (!body.ttl || body.ttl < 0) {
+                return error("Bad Request", "ttl is required");
+            }
+
+            await redis.set(key, JSON.stringify(body.data), "EX", body.ttl);
+
+            set.status = 204;
+            return;
+        },
+        {
+            body: t.Object({ data: t.Any(), ttl: t.Number() }),
+            query: QUERY_TYPE,
+            response: { 204: t.Void(), 400: t.String() },
+        }
+    )
+    .delete(
+        "/",
+        async ({ query: { key, fuzzy }, set }) => {
+            key = validateKey(key);
+            console.log("DELETE /cache", key);
+
+            if (fuzzy) {
+                key = `*${key}*`;
+            }
+
+            await redis.del(key);
+
+            set.status = 204;
+            return;
+        },
+        {
+            query: t.Object({
+                ...QUERY_TYPE.properties,
+                ...t.Object({ fuzzy: t.Optional(t.Boolean()) }).properties,
+            }),
+            response: { 204: t.Void(), 400: t.String() },
+        }
+    );
+
+function validateKey(key: string) {
+    const parsedKey = decodeURIComponent(key);
+
+    if (parsedKey.length < MIN_LENGTH) {
+        throw new ModelValidationError(
+            "Key has to be atleast 1 character long"
+        );
+    }
+
+    if (parsedKey.includes("*")) {
+        throw new ModelValidationError("Key cannot contain wildcards");
+    }
+
+    return parsedKey;
+}

apps/redis-api/src/routes/api/index.ts

@@ -0,0 +1,7 @@
+import { Elysia } from "elysia";
+import { cacheRoutes } from "./cache";
+import { apiKeyMiddleware } from "@/middleware/apiKeyMiddleware";
+
+export const apiRoutes = new Elysia({ prefix: "/api" })
+    .guard({ beforeHandle: apiKeyMiddleware })
+    .use(cacheRoutes);

apps/redis-api/src/routes/health.ts

@@ -0,0 +1,34 @@
+import Elysia, { t } from "elysia";
+
+import { redis } from "@/services/redis";
+import { baseLogger } from "@/utils/logger";
+
+export const healthRoutes = new Elysia().get(
+    "/health",
+    async ({ set, error }) => {
+        const perf = performance.now();
+        try {
+            await redis.ping();
+        } catch (e) {
+            baseLogger.error("Redis connection error:", e);
+            console.log("Redis connection error:", e);
+
+            return error(503, { healthy: false });
+        }
+
+        const duration = performance.now() - perf;
+        baseLogger.info(`Service healthy: ${duration.toFixed(2)} ms`);
+
+        return { healthy: true };
+    },
+    {
+        response: {
+            200: t.Object({
+                healthy: t.Boolean(),
+            }),
+            503: t.Object({
+                healthy: t.Boolean(),
+            }),
+        },
+    }
+);

apps/redis-api/src/services/redis.ts

@@ -0,0 +1,19 @@
+import { redisConfig, env } from "@/env";
+import ioredis from "ioredis";
+
+const redis = new ioredis({
+    host: redisConfig.host,
+    port: redisConfig.port,
+    username: redisConfig.username,
+    password: redisConfig.password,
+    maxRetriesPerRequest: 1, // Avoid excessive retries,
+    tls: !env.IS_DEV
+        ? {
+              rejectUnauthorized: true,
+          }
+        : undefined,
+    lazyConnect: true,
+    connectTimeout: 10_000,
+});
+
+export { redis };

apps/redis-api/src/utils/logger.ts

@@ -0,0 +1,34 @@
+import pino from "pino";
+import { mask } from "./mask";
+import { env } from "@/env";
+
+const serializers: { [key: string]: pino.SerializerFn } = {
+	password: (payload) => {
+		if (payload) {
+			return env.IS_DEV
+				? mask(payload)
+				: mask(payload, {
+						visibleStart: 0,
+						visibleEnd: 0,
+					});
+		}
+
+		return payload;
+	},
+	email: (payload) => {
+		if (payload) {
+			return env.IS_DEV ? payload : mask(payload);
+		}
+		return payload;
+	},
+};
+
+export const baseLogger = pino({
+	level: process.env.LOG_LEVEL || "info",
+	timestamp: pino.stdTimeFunctions.isoTime,
+	serializers,
+});
+
+export const loggerModule = (loggerName: string) => {
+	return baseLogger.child({ module: loggerName });
+};

apps/redis-api/src/utils/mask.test.ts

@@ -0,0 +1,42 @@
+import { describe, it, expect } from "bun:test";
+import { mask } from "./mask";
+
+describe("mask", () => {
+	it("should return empty string for empty input", () => {
+		expect(mask("")).toBe("");
+	});
+
+	it("should mask string with default parameters", () => {
+		expect(mask("1234567890")).toBe("12******90");
+	});
+
+	it("should show custom number of characters at start", () => {
+		expect(mask("1234567890", { visibleStart: 3 })).toBe("123*****90");
+	});
+
+	it("should show custom number of characters at end", () => {
+		expect(mask("1234567890", { visibleStart: 2, visibleEnd: 3 })).toBe(
+			"12*****890",
+		);
+	});
+
+	it("should mask entire string when visible parts exceed length", () => {
+		expect(mask("123", { visibleStart: 2, visibleEnd: 2 })).toBe("***");
+	});
+
+	it("should handle undefined end part", () => {
+		expect(mask("1234567890", { visibleStart: 2, visibleEnd: 0 })).toBe(
+			"12********",
+		);
+	});
+
+	it("should handle long strings", () => {
+		expect(mask("12345678901234567890")).toBe("12**********90");
+	});
+
+	it("should handle emails", () => {
+		expect(mask("test.testsson@scandichotels.com")).toBe(
+			"te*********on@sc*********ls.com",
+		);
+	});
+});

apps/redis-api/src/utils/mask.ts

@@ -0,0 +1,42 @@
+/**
+ * Masks a string by replacing characters with a mask character
+ * @param value - The string to mask
+ * @param visibleStart - Number of characters to show at start (default: 0)
+ * @param visibleEnd - Number of characters to show at end (default: 4)
+ * @param maskChar - Character to use for masking (default: '*')
+ * @returns The masked string
+ */
+const maskChar = "*";
+export function mask(
+	value: string,
+	options?: { visibleStart?: number; visibleEnd?: number; maxLength?: number },
+): string {
+	if (!value) return "";
+
+	const { visibleStart = 2, visibleEnd = 2, maxLength = 10 } = options ?? {};
+
+	if (isEmail(value)) {
+		return maskEmail(value);
+	}
+
+	const totalVisible = visibleStart + visibleEnd;
+	if (value.length <= totalVisible) {
+		return maskChar.repeat(value.length);
+	}
+
+	const start = value.slice(0, visibleStart);
+	const middle = value.slice(visibleStart, -visibleEnd || undefined);
+	const end = visibleEnd ? value.slice(-visibleEnd) : "";
+
+	const maskedLength = Math.min(middle.length, maxLength);
+	return start + maskChar.repeat(maskedLength) + end;
+}
+
+function maskEmail(email: string): string {
+	const [local, domain] = email.split("@");
+	if (!domain || !local) return mask(email);
+	const [subDomain, tld] = domain.split(/\.(?=[^.]+$)/);
+	return `${mask(local)}@${mask(subDomain ?? "")}.${tld}`;
+}
+
+const isEmail = (value: string) => /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(value);