Merged in fix/SW-1819-batching-city-urls (pull request #1477)

fix(SW-1819): Batching fetch for city page urls

* fix(SW-1819): Batching fetch for city page urls


Approved-by: Fredrik Thorsson
Approved-by: Matilda Landström

apps/scandic-web/server/routers/contentstack/destinationCityPage/utils.ts

@@ -1,19 +1,26 @@
+import { GetCityPageCount } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageCount.graphql"
 import { GetCityPageUrls } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql"
 import { request } from "@/lib/graphql/request"
 
 import { generateTag, generateTagsFromSystem } from "@/utils/generateTag"
 
-import { cityPageUrlsSchema } from "./output"
+import { batchedCityPageUrlsSchema, cityPageCountSchema } from "./output"
 import {
+  getCityPageCountCounter,
+  getCityPageCountFailCounter,
+  getCityPageCountSuccessCounter,
   getCityPageUrlsCounter,
   getCityPageUrlsFailCounter,
   getCityPageUrlsSuccessCounter,
 } from "./telemetry"
 
+import type { BatchRequestDocument } from "graphql-request"
+
 import { DestinationCityPageEnum } from "@/types/enums/destinationCityPage"
 import type { System } from "@/types/requests/system"
 import type {
   DestinationCityPageRefs,
+  GetCityPageCountData,
   GetCityPageUrlsData,
 } from "@/types/trpc/routers/contentstack/destinationCityPage"
 import type { Lang } from "@/constants/languages"
@@ -65,40 +72,100 @@ export function getConnections({
   return connections
 }
 
-export async function getCityPageUrls(lang: Lang) {
-  getCityPageUrlsCounter.add(1, { lang })
+export async function getCityPageCount(lang: Lang) {
+  getCityPageCountCounter.add(1, { lang })
   console.info(
-    "contentstack.cityPageUrls start",
+    "contentstack.cityPageCount start",
     JSON.stringify({ query: { lang } })
   )
-  const tag = `${lang}:city_page_urls`
-  const response = await request<GetCityPageUrlsData>(
-    GetCityPageUrls,
+  const tags = [`${lang}:city_page_count`]
+  const response = await request<GetCityPageCountData>(
+    GetCityPageCount,
     {
       locale: lang,
     },
     {
       cache: "force-cache",
       next: {
-        tags: [tag],
+        tags,
       },
     }
   )
-
   if (!response.data) {
-    getCityPageUrlsFailCounter.add(1, {
+    getCityPageCountFailCounter.add(1, {
       lang,
       error_type: "not_found",
-      error: `Destination city pages not found for lang: ${lang}`,
+      error: `City pages count not found for lang: ${lang}`,
     })
     console.error(
-      "contentstack.cityPageUrls not found error",
+      "contentstack.cityPageCount not found error",
       JSON.stringify({ query: { lang } })
     )
+    return 0
+  }
+
+  const validatedResponse = cityPageCountSchema.safeParse(response.data)
+
+  if (!validatedResponse.success) {
+    getCityPageCountFailCounter.add(1, {
+      lang,
+      error_type: "validation_error",
+      error: JSON.stringify(validatedResponse.error),
+    })
+    console.error(
+      "contentstack.hotelPageCount validation error",
+      JSON.stringify({
+        query: { lang },
+        error: validatedResponse.error,
+      })
+    )
+    return 0
+  }
+  getCityPageCountSuccessCounter.add(1, { lang })
+  console.info(
+    "contentstack.cityPageCount success",
+    JSON.stringify({ query: { lang } })
+  )
+
+  return validatedResponse.data
+}
+
+export async function getCityPageUrls(lang: Lang) {
+  getCityPageUrlsCounter.add(1, { lang })
+  console.info(
+    "contentstack.cityPageUrls start",
+    JSON.stringify({ query: { lang } })
+  )
+  const count = await getCityPageCount(lang)
+
+  if (count === 0) {
     return []
   }
 
-  const validatedResponse = cityPageUrlsSchema.safeParse(response.data)
+  // Calculating the amount of requests needed to fetch all pages.
+  // Contentstack has a limit of 100 items per request.
+  // So we need to make multiple requests to fetch urls to all pages.
+  // The `batchRequest` function is not working here, because the arrayMerge is
+  // used for other purposes.
+  const amountOfRequests = Math.ceil(count / 100)
+  const requests: (BatchRequestDocument & { options?: RequestInit })[] =
+    Array.from({ length: amountOfRequests }).map((_, i) => ({
+      document: GetCityPageUrls,
+      variables: { locale: lang, skip: i * 100 },
+      options: {
+        cache: "force-cache",
+        next: {
+          tags: [`${lang}:city_page_urls_batch_${i}`],
+        },
+      },
+    }))
+
+  const batchedResponse = await Promise.all(
+    requests.map((req) =>
+      request<GetCityPageUrlsData>(req.document, req.variables, req.options)
+    )
+  )
+  const validatedResponse = batchedCityPageUrlsSchema.safeParse(batchedResponse)
 
   if (!validatedResponse.success) {
     getCityPageUrlsFailCounter.add(1, {