diff --git a/apps/scandic-web/lib/graphql/batchEdgeRequest.ts b/apps/scandic-web/lib/graphql/batchEdgeRequest.ts deleted file mode 100644 index 54f6a09df..000000000 --- a/apps/scandic-web/lib/graphql/batchEdgeRequest.ts +++ /dev/null @@ -1,41 +0,0 @@ -import deepmerge from "deepmerge" - -import { arrayMerge } from "@/utils/merge" - -import { edgeRequest } from "./edgeRequest" - -import type { BatchRequestDocument } from "graphql-request" - -import type { Data } from "@/types/request" - -export async function batchEdgeRequest( - queries: BatchRequestDocument[] -): Promise> { - try { - const response = await Promise.allSettled( - queries.map((query) => edgeRequest(query.document, query.variables)) - ) - - let data = {} as T - const reasons: PromiseRejectedResult["reason"][] = [] - response.forEach((res) => { - if (res.status === "fulfilled") { - data = deepmerge(data, res.value.data, { arrayMerge }) - } else { - reasons.push(res.reason) - } - }) - - if (reasons.length) { - reasons.forEach((reason) => { - console.error(`Batch request failed`, reason) - }) - } - - return { data } - } catch (error) { - console.error("Error in batched graphql request") - console.error(error) - throw new Error("Something went wrong") - } -} diff --git a/apps/scandic-web/lib/graphql/edgeRequest.ts b/apps/scandic-web/lib/graphql/edgeRequest.ts deleted file mode 100644 index 12aef10d4..000000000 --- a/apps/scandic-web/lib/graphql/edgeRequest.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { GraphQLClient } from "graphql-request" - -import { env } from "@/env/server" - -import { request as _request } from "./_request" - -import type { DocumentNode } from "graphql" - -import type { Data } from "@/types/request" - -export async function edgeRequest( - query: string | DocumentNode, - variables?: {}, - params?: RequestInit -): Promise> { - // Creating a new client for each request to avoid conflicting parameters - const client = new GraphQLClient(env.CMS_URL, { - fetch: fetch, - }) - - return _request(client, query, variables, params) -} diff --git a/apps/scandic-web/lib/graphql/getOperationName.ts b/apps/scandic-web/lib/graphql/getOperationName.ts new file mode 100644 index 000000000..5b88cdd77 --- /dev/null +++ b/apps/scandic-web/lib/graphql/getOperationName.ts @@ -0,0 +1,22 @@ +import type { DocumentNode } from "graphql" + +export function getOperationName(query: string | DocumentNode): string { + let operationName = "" + + if (typeof query === "string") { + const operationRegex = /(query|mutation|subscription)\s+(\w+)/ + const match = query.match(operationRegex) + if (match && match[2]) { + operationName = match[2] + } + } else { + const opDefinition = query.definitions.find( + (def) => def.kind === "OperationDefinition" && def.name + ) + if (opDefinition && "name" in opDefinition && opDefinition.name) { + operationName = opDefinition.name.value + } + } + + return operationName ?? "AnonymousOperation" +} diff --git a/apps/scandic-web/lib/graphql/request.ts b/apps/scandic-web/lib/graphql/request.ts index eb2c40a12..db2bf5800 100644 --- a/apps/scandic-web/lib/graphql/request.ts +++ b/apps/scandic-web/lib/graphql/request.ts @@ -1,5 +1,6 @@ import fetchRetry from "fetch-retry" import { GraphQLClient } from "graphql-request" +import stringify from "json-stable-stringify-without-jsonify" import { cache as reactCache } from "react" import { env } from "@/env/server" @@ -8,6 +9,7 @@ import { getPreviewHash, isPreviewByUid } from "@/lib/previewContext" import { type CacheTime, getCacheClient } from "@/services/dataCache" import { request as _request } from "./_request" +import { getOperationName } from "./getOperationName" import type { DocumentNode } from "graphql" @@ -37,12 +39,23 @@ export async function request( return doCall() } + const queryString = typeof query === "string" ? query : stringify(query) + const variablesString = stringify(variables) + + const fullQuery = `${queryString}${variablesString}` + const queryHash = await sha256(fullQuery) + const operationName = getOperationName(query) + const cacheKey: string = Array.isArray(cacheOptions.key) ? cacheOptions.key.join("_") : cacheOptions.key + const extendedCacheKey = `${operationName}:${queryHash}:${cacheKey}` + const _dataCache = await getCacheClient() - return _dataCache.cacheOrGet(cacheKey, doCall, cacheOptions.ttl) + return _dataCache.cacheOrGet(extendedCacheKey, doCall, cacheOptions.ttl, { + includeGitHashInKey: false, + }) } function internalRequest( @@ -81,3 +94,13 @@ function internalRequest( return _request(client, query, variables, mergedParams) } + +async function sha256(input: string) { + const encoder = new TextEncoder() + const data = encoder.encode(input) + const hashBuffer = await crypto.subtle.digest("SHA-256", data) + const hashArray = Array.from(new Uint8Array(hashBuffer)) + const hashHex = hashArray.map((b) => b.toString(16).padStart(2, "0")).join("") + + return hashHex +} diff --git a/apps/scandic-web/middlewares/myPages.ts b/apps/scandic-web/middlewares/myPages.ts index 21da23fc8..9d290d0b3 100644 --- a/apps/scandic-web/middlewares/myPages.ts +++ b/apps/scandic-web/middlewares/myPages.ts @@ -9,7 +9,7 @@ import { import { notFound } from "@/server/errors/next" import { getPublicNextURL } from "@/server/utils" -import { fetchAndCacheEntry } from "@/services/cms/fetchAndCacheEntry" +import { resolve as resolveEntry } from "@/utils/entry" import { findLang } from "@/utils/languages" import { getDefaultRequestHeaders } from "./utils" @@ -32,10 +32,7 @@ export const middleware: NextMiddleware = async (request) => { } const pathNameWithoutLang = nextUrl.pathname.replace(`/${lang}`, "") - const { uid, contentType } = await fetchAndCacheEntry( - pathNameWithoutLang, - lang - ) + const { uid, contentType } = await resolveEntry(pathNameWithoutLang, lang) if (!uid || !contentType) { throw notFound( `Unable to resolve CMS entry for locale "${lang}": ${pathNameWithoutLang}` diff --git a/apps/scandic-web/middlewares/webView.ts b/apps/scandic-web/middlewares/webView.ts index a87f3474c..f0efac22e 100644 --- a/apps/scandic-web/middlewares/webView.ts +++ b/apps/scandic-web/middlewares/webView.ts @@ -10,8 +10,8 @@ import { import { env } from "@/env/server" import { badRequest, notFound } from "@/server/errors/next" -import { fetchAndCacheEntry } from "@/services/cms/fetchAndCacheEntry" import { decryptData } from "@/utils/aes" +import { resolve as resolveEntry } from "@/utils/entry" import { findLang } from "@/utils/languages" import { getDefaultRequestHeaders } from "./utils" @@ -147,7 +147,7 @@ async function handleWebviewRewrite({ const pathNameWithoutLang = path.replace(`/${lang}/webview`, "") - const { uid } = await fetchAndCacheEntry(pathNameWithoutLang, lang) + const { uid } = await resolveEntry(pathNameWithoutLang, lang) if (uid) { headers.set("x-uid", uid) } diff --git a/apps/scandic-web/server/routers/contentstack/collectionPage/utils.ts b/apps/scandic-web/server/routers/contentstack/collectionPage/utils.ts index ae6a2728f..aeddbfad4 100644 --- a/apps/scandic-web/server/routers/contentstack/collectionPage/utils.ts +++ b/apps/scandic-web/server/routers/contentstack/collectionPage/utils.ts @@ -32,6 +32,7 @@ export async function fetchCollectionPageRefs(lang: Lang, uid: string) { metricsGetCollectionPageRefs.start() const cacheKey = generateRefsResponseTag(lang, uid) + const refsResponse = await request( GetCollectionPageRefs, { diff --git a/apps/scandic-web/services/cms/fetchAndCacheEntry.ts b/apps/scandic-web/services/cms/fetchAndCacheEntry.ts deleted file mode 100644 index 5215c6219..000000000 --- a/apps/scandic-web/services/cms/fetchAndCacheEntry.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { getCacheClient } from "@/services/dataCache" -import { resolve as resolveEntry } from "@/utils/entry" - -import type { Lang } from "@/constants/languages" - -export const fetchAndCacheEntry = async (path: string, lang: Lang) => { - path = path || "/" - const cacheKey = `${lang}:resolveentry:${path}` - const cache = await getCacheClient() - - return cache.cacheOrGet( - cacheKey, - async () => { - const { contentType, uid } = await resolveEntry(path, lang) - - return { - contentType, - uid, - } - }, - "max" - ) -} diff --git a/apps/scandic-web/services/cms/getUidAndContentTypeByPath.ts b/apps/scandic-web/services/cms/getUidAndContentTypeByPath.ts index 712d01418..add3e8e84 100644 --- a/apps/scandic-web/services/cms/getUidAndContentTypeByPath.ts +++ b/apps/scandic-web/services/cms/getUidAndContentTypeByPath.ts @@ -1,10 +1,9 @@ import { Lang } from "@/constants/languages" +import { resolve as resolveEntry } from "@/utils/entry" import { findLang } from "@/utils/languages" import { removeTrailingSlash } from "@/utils/url" -import { fetchAndCacheEntry } from "./fetchAndCacheEntry" - export const getUidAndContentTypeByPath = async (pathname: string) => { const lang = findLang(pathname) @@ -12,7 +11,7 @@ export const getUidAndContentTypeByPath = async (pathname: string) => { const contentTypePathName = pathWithoutTrailingSlash.replace(`/${lang}`, "") - const { contentType, uid } = await fetchAndCacheEntry( + const { contentType, uid } = await resolveEntry( contentTypePathName, lang ?? Lang.en ) diff --git a/apps/scandic-web/services/dataCache/DistributedCache/cacheOrGet.ts b/apps/scandic-web/services/dataCache/DistributedCache/cacheOrGet.ts index e4850b109..69c3181e3 100644 --- a/apps/scandic-web/services/dataCache/DistributedCache/cacheOrGet.ts +++ b/apps/scandic-web/services/dataCache/DistributedCache/cacheOrGet.ts @@ -15,7 +15,9 @@ export const cacheOrGet: DataCache["cacheOrGet"] = async ( ttl: CacheTime, opts?: CacheOrGetOptions ) => { - const cacheKey = generateCacheKey(key) + const cacheKey = generateCacheKey(key, { + includeGitHashInKey: opts?.includeGitHashInKey ?? true, + }) let cachedValue: Awaited | undefined = undefined if (shouldGetFromCache(opts)) { @@ -32,8 +34,12 @@ export const cacheOrGet: DataCache["cacheOrGet"] = async ( const perf = performance.now() const data = await callback(overrideTTL) + const size = JSON.stringify(data).length / (1024 * 1024) + if (size >= 5) { + cacheLogger.warn(`'${key}' is larger than 5MB!`) + } cacheLogger.debug( - `Getting data '${cacheKey}' took ${(performance.now() - perf).toFixed(2)}ms` + `Fetching data took ${(performance.now() - perf).toFixed(2)}ms ${size.toFixed(4)}MB for '${key}'` ) await set(cacheKey, data, realTTL) diff --git a/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/getPrefix.ts b/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/getPrefix.ts index 42dd39ccd..a2dfa2ffd 100644 --- a/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/getPrefix.ts +++ b/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/getPrefix.ts @@ -2,28 +2,41 @@ import { env } from "@/env/server" import { getBranchPrefix } from "./getBranchPrefix" -export function getPrefix(): string { +export function getPrefix(options: { + includeGitHashInKey: boolean + includeBranchPrefix: boolean +}): string { + const prefixTokens = [] + + const includeGitHashInKey = options.includeGitHashInKey + const includeBranchPrefix = options.includeBranchPrefix + if (process.env.NODE_ENV === "development") { const devPrefix = process.env.USER || process.env.USERNAME || "dev" return `${devPrefix}` } - const branch = env.BRANCH.trim() - const gitSha = env.GIT_SHA?.trim().substring(0, 7) + if (includeGitHashInKey) { + const gitSha = env.GIT_SHA?.trim().substring(0, 7) - if (!branch && !gitSha) { - throw new Error("Unable to getPrefix, BRANCH and GIT_SHA must be set") + if (!gitSha) { + throw new Error("Unable to getPrefix, GIT_SHA must be set") + } + + prefixTokens.push(gitSha) } - if (!branch) { - throw new Error("Unable to getPrefix, BRANCH must be set") - } + if (includeBranchPrefix) { + const branch = env.BRANCH?.trim() - if (!gitSha) { - throw new Error("Unable to getPrefix, GIT_SHA must be set") + if (!branch) { + throw new Error("Unable to getPrefix, BRANCH must be set") + } + const branchPrefix = getBranchPrefix(branch) + if (branchPrefix) { + prefixTokens.push(branchPrefix) + } } - const prefixTokens = [getBranchPrefix(branch), gitSha].filter(Boolean) - return prefixTokens.join(":") } diff --git a/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/index.ts b/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/index.ts index 5b0dc8cbb..7b8b6b97e 100644 --- a/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/index.ts +++ b/apps/scandic-web/services/dataCache/DistributedCache/generateCacheKey/index.ts @@ -1,13 +1,20 @@ import { getPrefix } from "./getPrefix" -export function generateCacheKey(key: string | string[]): string { +export function generateCacheKey( + key: string | string[], + options?: { includeGitHashInKey?: boolean } +): string { + const includeGitHashInKey = options?.includeGitHashInKey ?? true const keyArray = (Array.isArray(key) ? key : [key]).filter(Boolean) if (keyArray.length === 0) { throw new Error("No keys provided") } - const prefix = getPrefix() + const prefix = getPrefix({ + includeGitHashInKey, + includeBranchPrefix: true, + }) const keyTokens = [prefix, keyArray.join("_")].filter(Boolean).join(":") diff --git a/apps/scandic-web/services/dataCache/cacheOrGetOptions.ts b/apps/scandic-web/services/dataCache/cacheOrGetOptions.ts index afb66de45..0ce4cdc51 100644 --- a/apps/scandic-web/services/dataCache/cacheOrGetOptions.ts +++ b/apps/scandic-web/services/dataCache/cacheOrGetOptions.ts @@ -7,6 +7,7 @@ export type CacheStrategy = "cache-first" | "fetch-then-cache" export type CacheOrGetOptions = { cacheStrategy?: CacheStrategy + includeGitHashInKey?: boolean } export function defaultCacheOrGetOptions( diff --git a/apps/scandic-web/utils/entry.ts b/apps/scandic-web/utils/entry.ts index 452b90d67..a1b8b1bf0 100644 --- a/apps/scandic-web/utils/entry.ts +++ b/apps/scandic-web/utils/entry.ts @@ -1,5 +1,5 @@ import { Lang } from "@/constants/languages" -import { batchEdgeRequest } from "@/lib/graphql/batchEdgeRequest" +import { batchRequest } from "@/lib/graphql/batchRequest" import { EntryByUrlBatch1, EntryByUrlBatch2, @@ -13,14 +13,23 @@ export async function resolve(url: string, lang = Lang.en) { // The maximum amount of content types you can query is 6, therefor more // than that is being batched - const response = await batchEdgeRequest([ + const cacheKey = `${lang}:${url}:resolveentry` + const response = await batchRequest([ { document: EntryByUrlBatch1, variables, + cacheOptions: { + ttl: "max", + key: cacheKey, + }, }, { document: EntryByUrlBatch2, variables, + cacheOptions: { + ttl: "max", + key: cacheKey, + }, }, ])