Merged in fix/SW-1784-hotel-url-fetching (pull request #1450)

fix(SW-1784): Fixed issue fetching all hotel urls at once.

* fix(SW-1784): Fixed issue fetching all hotel urls at once.


Approved-by: Christian Andolf

apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageCount.graphql

@@ -0,0 +1,7 @@
+#import "../../Fragments/System.graphql"
+
+query GetHotelPageCount($locale: String!) {
+  all_hotel_page(locale: $locale) {
+    total
+  }
+}

apps/scandic-web/lib/graphql/Query/HotelPage/HotelPageUrl.graphql

@@ -1,7 +1,7 @@
 #import "../../Fragments/System.graphql"
 
-query GetHotelPageUrls($locale: String!) {
-  all_hotel_page(locale: $locale) {
+query GetHotelPageUrls($locale: String!, $skip: Int) {
+  all_hotel_page(locale: $locale, limit: 100, skip: $skip) {
     items {
       url
       hotel_page_id

apps/scandic-web/server/routers/contentstack/hotelPage/output.ts

@@ -136,3 +136,21 @@ export const hotelPageUrlsSchema = z
     }),
   })
   .transform(({ all_hotel_page }) => all_hotel_page.items)
+
+export const batchedHotelPageUrlsSchema = z
+  .array(
+    z.object({
+      data: hotelPageUrlsSchema,
+    })
+  )
+  .transform((allItems) => {
+    return allItems.flatMap((item) => item.data)
+  })
+
+export const hotelPageCountSchema = z
+  .object({
+    all_hotel_page: z.object({
+      total: z.number(),
+    }),
+  })
+  .transform(({ all_hotel_page }) => all_hotel_page.total)

apps/scandic-web/server/routers/contentstack/hotelPage/telemetry.ts

@@ -31,3 +31,13 @@ export const getHotelPageUrlsSuccessCounter = meter.createCounter(
 export const getHotelPageUrlsFailCounter = meter.createCounter(
   "trpc.contentstack.hotelPageUrls.get-fail"
 )
+
+export const getHotelPageCountCounter = meter.createCounter(
+  "trpc.contentstack.hotelPageCount.get"
+)
+export const getHotelPageCountSuccessCounter = meter.createCounter(
+  "trpc.contentstack.hotelPageCount.get-success"
+)
+export const getHotelPageCountFailCounter = meter.createCounter(
+  "trpc.contentstack.hotelPageCount.get-fail"
+)

apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts

@@ -1,12 +1,20 @@
 import { GetHotelPageRefs } from "@/lib/graphql/Query/HotelPage/HotelPage.graphql"
+import { GetHotelPageCount } from "@/lib/graphql/Query/HotelPage/HotelPageCount.graphql"
 import { GetHotelPageUrls } from "@/lib/graphql/Query/HotelPage/HotelPageUrl.graphql"
 import { request } from "@/lib/graphql/request"
 import { notFound } from "@/server/errors/trpc"
 
 import { generateTag, generateTagsFromSystem } from "@/utils/generateTag"
 
-import { hotelPageRefsSchema, hotelPageUrlsSchema } from "./output"
 import {
+  batchedHotelPageUrlsSchema,
+  hotelPageCountSchema,
+  hotelPageRefsSchema,
+} from "./output"
+import {
+  getHotelPageCountCounter,
+  getHotelPageCountFailCounter,
+  getHotelPageCountSuccessCounter,
   getHotelPageRefsCounter,
   getHotelPageRefsFailCounter,
   getHotelPageRefsSuccessCounter,
@@ -15,9 +23,12 @@ import {
   getHotelPageUrlsSuccessCounter,
 } from "./telemetry"
 
+import type { BatchRequestDocument } from "graphql-request"
+
 import { HotelPageEnum } from "@/types/enums/hotelPage"
 import type { System } from "@/types/requests/system"
 import type {
+  GetHotelPageCountData,
   GetHotelPageRefsSchema,
   GetHotelPageUrlsData,
   HotelPageRefs,
@@ -132,15 +143,15 @@ export function getConnections({ hotel_page }: HotelPageRefs) {
   return connections
 }
 
-export async function getHotelPageUrls(lang: Lang) {
-  getHotelPageUrlsCounter.add(1, { lang })
+export async function getHotelPageCount(lang: Lang) {
+  getHotelPageCountCounter.add(1, { lang })
   console.info(
-    "contentstack.hotelPageUrls start",
+    "contentstack.hotelPageCount start",
     JSON.stringify({ query: { lang } })
   )
-  const tags = [`${lang}:hotel_page_urls`]
-  const response = await request<GetHotelPageUrlsData>(
-    GetHotelPageUrls,
+  const tags = [`${lang}:hotel_page_count`]
+  const response = await request<GetHotelPageCountData>(
+    GetHotelPageCount,
     {
       locale: lang,
     },
@@ -153,19 +164,82 @@ export async function getHotelPageUrls(lang: Lang) {
   )
 
   if (!response.data) {
-    getHotelPageUrlsFailCounter.add(1, {
+    getHotelPageCountFailCounter.add(1, {
       lang,
       error_type: "not_found",
-      error: `Hotel pages not found for lang: ${lang}`,
+      error: `Hotel pages count not found for lang: ${lang}`,
     })
     console.error(
-      "contentstack.hotelPageUrls not found error",
+      "contentstack.hotelPageCount not found error",
       JSON.stringify({ query: { lang } })
     )
+    return 0
+  }
+
+  const validatedResponse = hotelPageCountSchema.safeParse(response.data)
+
+  if (!validatedResponse.success) {
+    getHotelPageCountFailCounter.add(1, {
+      lang,
+      error_type: "validation_error",
+      error: JSON.stringify(validatedResponse.error),
+    })
+    console.error(
+      "contentstack.hotelPageCount validation error",
+      JSON.stringify({
+        query: { lang },
+        error: validatedResponse.error,
+      })
+    )
+    return 0
+  }
+  getHotelPageCountSuccessCounter.add(1, { lang })
+  console.info(
+    "contentstack.hotelPageCount success",
+    JSON.stringify({ query: { lang } })
+  )
+
+  return validatedResponse.data
+}
+
+export async function getHotelPageUrls(lang: Lang) {
+  getHotelPageUrlsCounter.add(1, { lang })
+  console.info(
+    "contentstack.hotelPageUrls start",
+    JSON.stringify({ query: { lang } })
+  )
+  const hotelPageCount = await getHotelPageCount(lang)
+
+  if (hotelPageCount === 0) {
     return []
   }
 
-  const validatedHotelPageUrls = hotelPageUrlsSchema.safeParse(response.data)
+  // Calculating the amount of requests needed to fetch all hotel pages.
+  // Contentstack has a limit of 100 items per request.
+  // So we need to make multiple requests to fetch urls to all hotel pages.
+  // The `batchRequest` function is not working here, because the arrayMerge is
+  // used for other purposes.
+  const amountOfRequests = Math.ceil(hotelPageCount / 100)
+  const requests: (BatchRequestDocument & { options?: RequestInit })[] =
+    Array.from({ length: amountOfRequests }).map((_, i) => ({
+      document: GetHotelPageUrls,
+      variables: { locale: lang, skip: i * 100 },
+      options: {
+        cache: "force-cache",
+        next: {
+          tags: [`${lang}:hotel_page_urls_batch_${i}`],
+        },
+      },
+    }))
+
+  const batchedResponse = await Promise.all(
+    requests.map((req) =>
+      request<GetHotelPageUrlsData>(req.document, req.variables, req.options)
+    )
+  )
+
+  const validatedHotelPageUrls =
+    batchedHotelPageUrlsSchema.safeParse(batchedResponse)
 
   if (!validatedHotelPageUrls.success) {
     getHotelPageUrlsFailCounter.add(1, {

apps/scandic-web/types/trpc/routers/contentstack/hotelPage.ts

@@ -2,6 +2,7 @@ import type { z } from "zod"
 
 import type {
   contentBlock,
+  hotelPageCountSchema,
   hotelPageRefsSchema,
   hotelPageSchema,
   hotelPageUrlsSchema,
@@ -24,4 +25,6 @@ export interface GetHotelPageRefsSchema
 export interface HotelPageRefs extends z.output<typeof hotelPageRefsSchema> {}
 export interface GetHotelPageUrlsData
   extends z.input<typeof hotelPageUrlsSchema> {}
+export interface GetHotelPageCountData
+  extends z.input<typeof hotelPageCountSchema> {}
 export type HotelPageUrls = z.output<typeof hotelPageUrlsSchema>