diff --git a/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageCount.graphql b/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageCount.graphql new file mode 100644 index 000000000..6c83af553 --- /dev/null +++ b/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageCount.graphql @@ -0,0 +1,7 @@ +#import "../../Fragments/System.graphql" + +query GetHotelPageCount($locale: String!) { + all_hotel_page(locale: $locale) { + total + } +} diff --git a/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageUrl.graphql b/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageUrl.graphql index 26c28cfd9..6d2856641 100644 --- a/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageUrl.graphql +++ b/apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageUrl.graphql @@ -1,7 +1,7 @@ #import "../../Fragments/System.graphql" -query GetHotelPageUrls($locale: String!) { - all_hotel_page(locale: $locale) { +query GetHotelPageUrls($locale: String!, $skip: Int) { + all_hotel_page(locale: $locale, limit: 100, skip: $skip) { items { url hotel_page_id diff --git a/apps/scandic-web/server/routers/contentstack/hotelPage/output.ts b/apps/scandic-web/server/routers/contentstack/hotelPage/output.ts index 3e4b0bbe5..a9e4d447e 100644 --- a/apps/scandic-web/server/routers/contentstack/hotelPage/output.ts +++ b/apps/scandic-web/server/routers/contentstack/hotelPage/output.ts @@ -136,3 +136,21 @@ export const hotelPageUrlsSchema = z }), }) .transform(({ all_hotel_page }) => all_hotel_page.items) + +export const batchedHotelPageUrlsSchema = z + .array( + z.object({ + data: hotelPageUrlsSchema, + }) + ) + .transform((allItems) => { + return allItems.flatMap((item) => item.data) + }) + +export const hotelPageCountSchema = z + .object({ + all_hotel_page: z.object({ + total: z.number(), + }), + }) + .transform(({ all_hotel_page }) => all_hotel_page.total) diff --git a/apps/scandic-web/server/routers/contentstack/hotelPage/telemetry.ts b/apps/scandic-web/server/routers/contentstack/hotelPage/telemetry.ts index 83117d350..2c094c055 100644 --- a/apps/scandic-web/server/routers/contentstack/hotelPage/telemetry.ts +++ b/apps/scandic-web/server/routers/contentstack/hotelPage/telemetry.ts @@ -31,3 +31,13 @@ export const getHotelPageUrlsSuccessCounter = meter.createCounter( export const getHotelPageUrlsFailCounter = meter.createCounter( "trpc.contentstack.hotelPageUrls.get-fail" ) + +export const getHotelPageCountCounter = meter.createCounter( + "trpc.contentstack.hotelPageCount.get" +) +export const getHotelPageCountSuccessCounter = meter.createCounter( + "trpc.contentstack.hotelPageCount.get-success" +) +export const getHotelPageCountFailCounter = meter.createCounter( + "trpc.contentstack.hotelPageCount.get-fail" +) diff --git a/apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts b/apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts index 605eade32..4127b7734 100644 --- a/apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts +++ b/apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts @@ -1,12 +1,20 @@ import { GetHotelPageRefs } from "@/lib/graphql/Query/HotelPage/HotelPage.graphql" +import { GetHotelPageCount } from "@/lib/graphql/Query/HotelPage/HotelPageCount.graphql" import { GetHotelPageUrls } from "@/lib/graphql/Query/HotelPage/HotelPageUrl.graphql" import { request } from "@/lib/graphql/request" import { notFound } from "@/server/errors/trpc" import { generateTag, generateTagsFromSystem } from "@/utils/generateTag" -import { hotelPageRefsSchema, hotelPageUrlsSchema } from "./output" import { + batchedHotelPageUrlsSchema, + hotelPageCountSchema, + hotelPageRefsSchema, +} from "./output" +import { + getHotelPageCountCounter, + getHotelPageCountFailCounter, + getHotelPageCountSuccessCounter, getHotelPageRefsCounter, getHotelPageRefsFailCounter, getHotelPageRefsSuccessCounter, @@ -15,9 +23,12 @@ import { getHotelPageUrlsSuccessCounter, } from "./telemetry" +import type { BatchRequestDocument } from "graphql-request" + import { HotelPageEnum } from "@/types/enums/hotelPage" import type { System } from "@/types/requests/system" import type { + GetHotelPageCountData, GetHotelPageRefsSchema, GetHotelPageUrlsData, HotelPageRefs, @@ -132,15 +143,15 @@ export function getConnections({ hotel_page }: HotelPageRefs) { return connections } -export async function getHotelPageUrls(lang: Lang) { - getHotelPageUrlsCounter.add(1, { lang }) +export async function getHotelPageCount(lang: Lang) { + getHotelPageCountCounter.add(1, { lang }) console.info( - "contentstack.hotelPageUrls start", + "contentstack.hotelPageCount start", JSON.stringify({ query: { lang } }) ) - const tags = [`${lang}:hotel_page_urls`] - const response = await request( - GetHotelPageUrls, + const tags = [`${lang}:hotel_page_count`] + const response = await request( + GetHotelPageCount, { locale: lang, }, @@ -153,19 +164,82 @@ export async function getHotelPageUrls(lang: Lang) { ) if (!response.data) { - getHotelPageUrlsFailCounter.add(1, { + getHotelPageCountFailCounter.add(1, { lang, error_type: "not_found", - error: `Hotel pages not found for lang: ${lang}`, + error: `Hotel pages count not found for lang: ${lang}`, }) console.error( - "contentstack.hotelPageUrls not found error", + "contentstack.hotelPageCount not found error", JSON.stringify({ query: { lang } }) ) + return 0 + } + + const validatedResponse = hotelPageCountSchema.safeParse(response.data) + + if (!validatedResponse.success) { + getHotelPageCountFailCounter.add(1, { + lang, + error_type: "validation_error", + error: JSON.stringify(validatedResponse.error), + }) + console.error( + "contentstack.hotelPageCount validation error", + JSON.stringify({ + query: { lang }, + error: validatedResponse.error, + }) + ) + return 0 + } + getHotelPageCountSuccessCounter.add(1, { lang }) + console.info( + "contentstack.hotelPageCount success", + JSON.stringify({ query: { lang } }) + ) + + return validatedResponse.data +} + +export async function getHotelPageUrls(lang: Lang) { + getHotelPageUrlsCounter.add(1, { lang }) + console.info( + "contentstack.hotelPageUrls start", + JSON.stringify({ query: { lang } }) + ) + const hotelPageCount = await getHotelPageCount(lang) + + if (hotelPageCount === 0) { return [] } - const validatedHotelPageUrls = hotelPageUrlsSchema.safeParse(response.data) + // Calculating the amount of requests needed to fetch all hotel pages. + // Contentstack has a limit of 100 items per request. + // So we need to make multiple requests to fetch urls to all hotel pages. + // The `batchRequest` function is not working here, because the arrayMerge is + // used for other purposes. + const amountOfRequests = Math.ceil(hotelPageCount / 100) + const requests: (BatchRequestDocument & { options?: RequestInit })[] = + Array.from({ length: amountOfRequests }).map((_, i) => ({ + document: GetHotelPageUrls, + variables: { locale: lang, skip: i * 100 }, + options: { + cache: "force-cache", + next: { + tags: [`${lang}:hotel_page_urls_batch_${i}`], + }, + }, + })) + + const batchedResponse = await Promise.all( + requests.map((req) => + request(req.document, req.variables, req.options) + ) + ) + + const validatedHotelPageUrls = + batchedHotelPageUrlsSchema.safeParse(batchedResponse) if (!validatedHotelPageUrls.success) { getHotelPageUrlsFailCounter.add(1, { diff --git a/apps/scandic-web/types/trpc/routers/contentstack/hotelPage.ts b/apps/scandic-web/types/trpc/routers/contentstack/hotelPage.ts index fede08e2d..641d0cf0f 100644 --- a/apps/scandic-web/types/trpc/routers/contentstack/hotelPage.ts +++ b/apps/scandic-web/types/trpc/routers/contentstack/hotelPage.ts @@ -2,6 +2,7 @@ import type { z } from "zod" import type { contentBlock, + hotelPageCountSchema, hotelPageRefsSchema, hotelPageSchema, hotelPageUrlsSchema, @@ -24,4 +25,6 @@ export interface GetHotelPageRefsSchema export interface HotelPageRefs extends z.output {} export interface GetHotelPageUrlsData extends z.input {} +export interface GetHotelPageCountData + extends z.input {} export type HotelPageUrls = z.output