Merged in fix/SW-1819-batching-city-urls (pull request #1477)

fix(SW-1819): Batching fetch for city page urls

* fix(SW-1819): Batching fetch for city page urls


Approved-by: Fredrik Thorsson
Approved-by: Matilda Landström

apps/scandic-web/server/routers/contentstack/destinationCityPage/output.ts

@@ -74,14 +74,16 @@ export const destinationCityListDataSchema = z
               })
               .transform(
                 ({ destination_experiences }) => destination_experiences
-              ),
+              )
+              .nullish(),
             images: z
               .array(z.object({ image: tempImageVaultAssetSchema }))
               .transform((images) =>
                 images
                   .map((image) => image.image)
                   .filter((image): image is ImageVaultAsset => !!image)
-              ),
+              )
+              .nullish(),
             url: z.string(),
             system: systemSchema,
           })
@@ -192,6 +194,14 @@ export const destinationCityPageSchema = z
     }
   })
 
+export const cityPageCountSchema = z
+  .object({
+    all_destination_city_page: z.object({
+      total: z.number(),
+    }),
+  })
+  .transform(({ all_destination_city_page }) => all_destination_city_page.total)
+
 export const cityPageUrlsSchema = z
   .object({
     all_destination_city_page: z.object({
@@ -213,6 +223,16 @@ export const cityPageUrlsSchema = z
   })
   .transform(({ all_destination_city_page }) => all_destination_city_page.items)
 
+export const batchedCityPageUrlsSchema = z
+  .array(
+    z.object({
+      data: cityPageUrlsSchema,
+    })
+  )
+  .transform((allItems) => {
+    return allItems.flatMap((item) => item.data)
+  })
+
 /** REFS */
 const destinationCityPageContentRefs = z
   .object({

apps/scandic-web/server/routers/contentstack/destinationCityPage/telemetry.ts

@@ -22,6 +22,16 @@ export const getDestinationCityPageFailCounter = meter.createCounter(
   "trpc.contentstack.destinationCityPage.get-fail"
 )
 
+export const getCityPageCountCounter = meter.createCounter(
+  "trpc.contentstack.cityPageCount.get"
+)
+export const getCityPageCountSuccessCounter = meter.createCounter(
+  "trpc.contentstack.cityPageCount.get-success"
+)
+export const getCityPageCountFailCounter = meter.createCounter(
+  "trpc.contentstack.cityPageCount.get-fail"
+)
+
 export const getCityPageUrlsCounter = meter.createCounter(
   "trpc.contentstack.cityPageUrls.get"
 )

apps/scandic-web/server/routers/contentstack/destinationCityPage/utils.ts

@@ -1,19 +1,26 @@
+import { GetCityPageCount } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageCount.graphql"
 import { GetCityPageUrls } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql"
 import { request } from "@/lib/graphql/request"
 
 import { generateTag, generateTagsFromSystem } from "@/utils/generateTag"
 
-import { cityPageUrlsSchema } from "./output"
+import { batchedCityPageUrlsSchema, cityPageCountSchema } from "./output"
 import {
+  getCityPageCountCounter,
+  getCityPageCountFailCounter,
+  getCityPageCountSuccessCounter,
   getCityPageUrlsCounter,
   getCityPageUrlsFailCounter,
   getCityPageUrlsSuccessCounter,
 } from "./telemetry"
 
+import type { BatchRequestDocument } from "graphql-request"
+
 import { DestinationCityPageEnum } from "@/types/enums/destinationCityPage"
 import type { System } from "@/types/requests/system"
 import type {
   DestinationCityPageRefs,
+  GetCityPageCountData,
   GetCityPageUrlsData,
 } from "@/types/trpc/routers/contentstack/destinationCityPage"
 import type { Lang } from "@/constants/languages"
@@ -65,40 +72,100 @@ export function getConnections({
   return connections
 }
 
-export async function getCityPageUrls(lang: Lang) {
-  getCityPageUrlsCounter.add(1, { lang })
+export async function getCityPageCount(lang: Lang) {
+  getCityPageCountCounter.add(1, { lang })
   console.info(
-    "contentstack.cityPageUrls start",
+    "contentstack.cityPageCount start",
     JSON.stringify({ query: { lang } })
   )
-  const tag = `${lang}:city_page_urls`
-  const response = await request<GetCityPageUrlsData>(
-    GetCityPageUrls,
+  const tags = [`${lang}:city_page_count`]
+  const response = await request<GetCityPageCountData>(
+    GetCityPageCount,
     {
       locale: lang,
     },
     {
       cache: "force-cache",
       next: {
-        tags: [tag],
+        tags,
       },
     }
   )
-
   if (!response.data) {
-    getCityPageUrlsFailCounter.add(1, {
+    getCityPageCountFailCounter.add(1, {
       lang,
       error_type: "not_found",
-      error: `Destination city pages not found for lang: ${lang}`,
+      error: `City pages count not found for lang: ${lang}`,
     })
     console.error(
-      "contentstack.cityPageUrls not found error",
+      "contentstack.cityPageCount not found error",
       JSON.stringify({ query: { lang } })
     )
+    return 0
+  }
+
+  const validatedResponse = cityPageCountSchema.safeParse(response.data)
+
+  if (!validatedResponse.success) {
+    getCityPageCountFailCounter.add(1, {
+      lang,
+      error_type: "validation_error",
+      error: JSON.stringify(validatedResponse.error),
+    })
+    console.error(
+      "contentstack.hotelPageCount validation error",
+      JSON.stringify({
+        query: { lang },
+        error: validatedResponse.error,
+      })
+    )
+    return 0
+  }
+  getCityPageCountSuccessCounter.add(1, { lang })
+  console.info(
+    "contentstack.cityPageCount success",
+    JSON.stringify({ query: { lang } })
+  )
+
+  return validatedResponse.data
+}
+
+export async function getCityPageUrls(lang: Lang) {
+  getCityPageUrlsCounter.add(1, { lang })
+  console.info(
+    "contentstack.cityPageUrls start",
+    JSON.stringify({ query: { lang } })
+  )
+  const count = await getCityPageCount(lang)
+
+  if (count === 0) {
     return []
   }
 
-  const validatedResponse = cityPageUrlsSchema.safeParse(response.data)
+  // Calculating the amount of requests needed to fetch all pages.
+  // Contentstack has a limit of 100 items per request.
+  // So we need to make multiple requests to fetch urls to all pages.
+  // The `batchRequest` function is not working here, because the arrayMerge is
+  // used for other purposes.
+  const amountOfRequests = Math.ceil(count / 100)
+  const requests: (BatchRequestDocument & { options?: RequestInit })[] =
+    Array.from({ length: amountOfRequests }).map((_, i) => ({
+      document: GetCityPageUrls,
+      variables: { locale: lang, skip: i * 100 },
+      options: {
+        cache: "force-cache",
+        next: {
+          tags: [`${lang}:city_page_urls_batch_${i}`],
+        },
+      },
+    }))
+
+  const batchedResponse = await Promise.all(
+    requests.map((req) =>
+      request<GetCityPageUrlsData>(req.document, req.variables, req.options)
+    )
+  )
+  const validatedResponse = batchedCityPageUrlsSchema.safeParse(batchedResponse)
 
   if (!validatedResponse.success) {
     getCityPageUrlsFailCounter.add(1, {

apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts

@@ -208,9 +208,9 @@ export async function getHotelPageUrls(lang: Lang) {
     "contentstack.hotelPageUrls start",
     JSON.stringify({ query: { lang } })
   )
-  const hotelPageCount = await getHotelPageCount(lang)
+  const count = await getHotelPageCount(lang)
 
-  if (hotelPageCount === 0) {
+  if (count === 0) {
     return []
   }
 
@@ -219,7 +219,7 @@ export async function getHotelPageUrls(lang: Lang) {
   // So we need to make multiple requests to fetch urls to all hotel pages.
   // The `batchRequest` function is not working here, because the arrayMerge is
   // used for other purposes.
-  const amountOfRequests = Math.ceil(hotelPageCount / 100)
+  const amountOfRequests = Math.ceil(count / 100)
   const requests: (BatchRequestDocument & { options?: RequestInit })[] =
     Array.from({ length: amountOfRequests }).map((_, i) => ({
       document: GetHotelPageUrls,
@@ -238,20 +238,20 @@ export async function getHotelPageUrls(lang: Lang) {
     )
   )
 
-  const validatedHotelPageUrls =
+  const validatedResponse =
     batchedHotelPageUrlsSchema.safeParse(batchedResponse)
 
-  if (!validatedHotelPageUrls.success) {
+  if (!validatedResponse.success) {
     getHotelPageUrlsFailCounter.add(1, {
       lang,
       error_type: "validation_error",
-      error: JSON.stringify(validatedHotelPageUrls.error),
+      error: JSON.stringify(validatedResponse.error),
     })
     console.error(
       "contentstack.hotelPageUrls validation error",
       JSON.stringify({
         query: { lang },
-        error: validatedHotelPageUrls.error,
+        error: validatedResponse.error,
       })
     )
     return []
@@ -262,5 +262,5 @@ export async function getHotelPageUrls(lang: Lang) {
     JSON.stringify({ query: { lang } })
   )
 
-  return validatedHotelPageUrls.data
+  return validatedResponse.data
 }