Merged in fix/warmup-autocomplete-data (pull request #2212)

warmup autocomplete data

* warmup autocomplete data


Approved-by: Anton Gunnarsson

apps/scandic-web/server/routers/autocomplete/destinations.ts

@@ -40,105 +40,13 @@ type DestinationsAutoCompleteOutput = {
 export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure
   .input(destinationsAutoCompleteInputSchema)
   .query(async ({ ctx, input }): Promise<DestinationsAutoCompleteOutput> => {
-    const cacheClient = await getCacheClient()
-
     const lang = input.lang || ctx.lang
 
-    const locations: AutoCompleteLocation[] = await cacheClient.cacheOrGet(
+    const locations: AutoCompleteLocation[] =
-      `autocomplete:destinations:locations:${lang}`,
+      await getAutoCompleteDestinationsData({
-      async () => {
+        lang,
-        const hotelUrlsPromise = safeTry(getHotelPageUrls(lang))
+        serviceToken: ctx.serviceToken,
-        const cityUrlsPromise = safeTry(getCityPageUrls(lang))
+      })
-        const countryUrlsPromise = safeTry(getCountryPageUrls(lang))
-        const countries = await getCountries({
-          lang: lang,
-          serviceToken: ctx.serviceToken,
-        })
-
-        if (!countries) {
-          throw new Error("Unable to fetch countries")
-        }
-
-        const countryNames = countries.data.map((country) => country.name)
-        const citiesByCountry = await getCitiesByCountry({
-          countries: countryNames,
-          serviceToken: ctx.serviceToken,
-          lang,
-        })
-
-        const locations = await getLocations({
-          lang: lang,
-          serviceToken: ctx.serviceToken,
-          citiesByCountry: citiesByCountry,
-        })
-
-        const activeLocations = locations.filter((location) => {
-          return (
-            location.type === "cities" ||
-            (location.type === "hotels" && location.isActive)
-          )
-        })
-
-        const [hotelUrls, hotelUrlsError] = await hotelUrlsPromise
-        const [cityUrls, cityUrlsError] = await cityUrlsPromise
-        const [countryUrls, countryUrlsError] = await countryUrlsPromise
-
-        if (
-          hotelUrlsError ||
-          cityUrlsError ||
-          countryUrlsError ||
-          !hotelUrls ||
-          !cityUrls ||
-          !countryUrls
-        ) {
-          throw new Error("Unable to fetch location URLs")
-        }
-
-        const hotelsAndCities = activeLocations
-          .map((location) => {
-            let url: string | undefined
-
-            if (location.type === "cities") {
-              url = cityUrls.find(
-                (c) =>
-                  c.city &&
-                  location.cityIdentifier &&
-                  c.city === location.cityIdentifier
-              )?.url
-            }
-
-            if (location.type === "hotels") {
-              url = hotelUrls.find(
-                (h) => h.hotelId && location.id && h.hotelId === location.id
-              )?.url
-            }
-
-            return { ...location, url }
-          })
-          .map(mapLocationToAutoCompleteLocation)
-          .filter(isDefined)
-
-        const countryAutoCompleteLocations = countries.data.map((country) => {
-          const url = countryUrls.find(
-            (c) =>
-              c.country &&
-              ApiCountry[lang][c.country as Country] === country.name
-          )?.url
-
-          return {
-            id: country.id,
-            name: country.name,
-            type: "countries",
-            searchTokens: [country.name],
-            destination: "",
-            url,
-          } satisfies AutoCompleteLocation
-        })
-
-        return [...hotelsAndCities, ...countryAutoCompleteLocations]
-      },
-      "1d"
-    )
 
     const hits = filterAndCategorizeAutoComplete({
       locations: locations,
@@ -176,3 +84,110 @@ function isCity(
 ): location is AutoCompleteLocation & { type: "cities" } {
   return !!location && location.type === "cities"
 }
+
+export async function getAutoCompleteDestinationsData({
+  lang,
+  serviceToken,
+  warmup = false,
+}: {
+  lang: Lang
+  serviceToken: string
+  warmup?: boolean
+}) {
+  const cacheClient = await getCacheClient()
+  return await cacheClient.cacheOrGet(
+    `autocomplete:destinations:locations:${lang}`,
+    async () => {
+      const hotelUrlsPromise = safeTry(getHotelPageUrls(lang))
+      const cityUrlsPromise = safeTry(getCityPageUrls(lang))
+      const countryUrlsPromise = safeTry(getCountryPageUrls(lang))
+      const countries = await getCountries({
+        lang: lang,
+        serviceToken,
+      })
+
+      if (!countries) {
+        throw new Error("Unable to fetch countries")
+      }
+
+      const countryNames = countries.data.map((country) => country.name)
+      const citiesByCountry = await getCitiesByCountry({
+        countries: countryNames,
+        serviceToken: serviceToken,
+        lang,
+      })
+
+      const locations = await getLocations({
+        lang: lang,
+        serviceToken: serviceToken,
+        citiesByCountry: citiesByCountry,
+      })
+
+      const activeLocations = locations.filter((location) => {
+        return (
+          location.type === "cities" ||
+          (location.type === "hotels" && location.isActive)
+        )
+      })
+
+      const [hotelUrls, hotelUrlsError] = await hotelUrlsPromise
+      const [cityUrls, cityUrlsError] = await cityUrlsPromise
+      const [countryUrls, countryUrlsError] = await countryUrlsPromise
+
+      if (
+        hotelUrlsError ||
+        cityUrlsError ||
+        countryUrlsError ||
+        !hotelUrls ||
+        !cityUrls ||
+        !countryUrls
+      ) {
+        throw new Error("Unable to fetch location URLs")
+      }
+
+      const hotelsAndCities = activeLocations
+        .map((location) => {
+          let url: string | undefined
+
+          if (location.type === "cities") {
+            url = cityUrls.find(
+              (c) =>
+                c.city &&
+                location.cityIdentifier &&
+                c.city === location.cityIdentifier
+            )?.url
+          }
+
+          if (location.type === "hotels") {
+            url = hotelUrls.find(
+              (h) => h.hotelId && location.id && h.hotelId === location.id
+            )?.url
+          }
+
+          return { ...location, url }
+        })
+        .map(mapLocationToAutoCompleteLocation)
+        .filter(isDefined)
+
+      const countryAutoCompleteLocations = countries.data.map((country) => {
+        const url = countryUrls.find(
+          (c) =>
+            c.country && ApiCountry[lang][c.country as Country] === country.name
+        )?.url
+
+        return {
+          id: country.id,
+          name: country.name,
+          type: "countries",
+          searchTokens: [country.name],
+          destination: "",
+          url,
+        } satisfies AutoCompleteLocation
+      })
+
+      return [...hotelsAndCities, ...countryAutoCompleteLocations]
+    },
+    "1d",
+    { cacheStrategy: warmup ? "fetch-then-cache" : "cache-first" }
+  )
+}

apps/scandic-web/server/routers/hotels/utils.ts

@@ -61,6 +61,8 @@ import type {
   RoomConfiguration,
 } from "@/types/trpc/routers/hotel/roomAvailability"
 import type { Endpoint } from "@/lib/api/endpoints"
+import { chunk } from "@/utils/chunk"
+import { z } from "zod"
 
 export function getPoiGroupByCategoryName(category: string | undefined) {
   if (!category) return PointOfInterestGroupEnum.LOCATION
@@ -275,6 +277,7 @@ export async function getLocations({
         }
         throw new Error("downstream error")
       }
+
       const apiJson = await apiResponse.json()
       const verifiedLocations = locationsSchema.safeParse(apiJson)
       if (!verifiedLocations.success) {
@@ -283,46 +286,55 @@ export async function getLocations({
         throw new Error("Unable to parse locations")
       }
 
-      return await Promise.all(
+      const chunkedLocations = chunk(verifiedLocations.data.data, 10)
-        verifiedLocations.data.data.map(async (location) => {
+      let locations: z.infer<typeof locationsSchema>["data"] = []
-          if (location.type === "cities") {
-            if (citiesByCountry) {
-              const country = Object.keys(citiesByCountry).find((country) =>
-                citiesByCountry[country].find(
-                  (loc) => loc.name === location.name
-                )
-              )
-              if (country) {
-                return {
-                  ...location,
-                  country,
-                }
-              } else {
-                console.info(
-                  `Location cannot be found in any of the countries cities`
-                )
-                console.info(location)
-              }
-            }
-          } else if (location.type === "hotels") {
-            if (location.relationships.city?.url) {
-              const city = await getCity({
-                cityUrl: location.relationships.city.url,
-                serviceToken,
-              })
-              if (city) {
-                return deepmerge(location, {
-                  relationships: {
-                    city,
-                  },
-                })
-              }
-            }
-          }
 
-          return location
+      for (const chunk of chunkedLocations) {
-        })
+        locations = [
-      )
+          ...(await Promise.all(
+            chunk.map(async (location) => {
+              if (location.type === "cities") {
+                if (citiesByCountry) {
+                  const country = Object.keys(citiesByCountry).find((country) =>
+                    citiesByCountry[country].find(
+                      (loc) => loc.name === location.name
+                    )
+                  )
+                  if (country) {
+                    return {
+                      ...location,
+                      country,
+                    }
+                  } else {
+                    console.info(
+                      `Location cannot be found in any of the countries cities`
+                    )
+                    console.info(location)
+                  }
+                }
+              } else if (location.type === "hotels") {
+                if (location.relationships.city?.url) {
+                  const city = await getCity({
+                    cityUrl: location.relationships.city.url,
+                    serviceToken,
+                  })
+                  if (city) {
+                    return deepmerge(location, {
+                      relationships: {
+                        city,
+                      },
+                    })
+                  }
+                }
+              }
+
+              return location
+            })
+          )),
+        ]
+      }
+
+      return locations
     },
     "1d"
   )
@@ -506,40 +518,49 @@ export async function getHotelsByHotelIds({
     cacheKey,
     async () => {
       const hotelPages = await getHotelPageUrls(lang)
-      const hotels = await Promise.all(
+      const chunkedHotelIds = chunk(hotelIds, 10)
-        hotelIds.map(async (hotelId) => {
-          const hotelResponse = await getHotel(
-            { hotelId, language: lang, isCardOnlyPayment: false },
-            serviceToken
-          )
 
-          if (!hotelResponse) {
+      const hotels: DestinationPagesHotelData[] = []
-            throw new Error(`Hotel not found: ${hotelId}`)
+      for (const hotelIdChunk of chunkedHotelIds) {
-          }
+        hotels.push(
+          ...(await Promise.all(
+            hotelIdChunk.map(async (hotelId) => {
+              const hotelResponse = await getHotel(
+                { hotelId, language: lang, isCardOnlyPayment: false },
+                serviceToken
+              )
 
-          const hotelPage = hotelPages.find((page) => page.hotelId === hotelId)
+              if (!hotelResponse) {
-          const { hotel, cities } = hotelResponse
+                throw new Error(`Hotel not found: ${hotelId}`)
-          const data: DestinationPagesHotelData = {
+              }
-            hotel: {
-              id: hotel.id,
-              galleryImages: hotel.galleryImages,
-              name: hotel.name,
-              tripadvisor: hotel.ratings?.tripAdvisor?.rating,
-              detailedFacilities: hotel.detailedFacilities || [],
-              location: hotel.location,
-              hotelType: hotel.hotelType,
-              type: hotel.type,
-              address: hotel.address,
-              cityIdentifier: cities?.[0]?.cityIdentifier,
-              hotelDescription: hotel.hotelContent?.texts.descriptions?.short,
-            },
-            url: hotelPage?.url ?? "",
-          }
 
-          return { ...data, url: hotelPage?.url ?? null }
+              const hotelPage = hotelPages.find(
-        })
+                (page) => page.hotelId === hotelId
-      )
+              )
+              const { hotel, cities } = hotelResponse
+              const data: DestinationPagesHotelData = {
+                hotel: {
+                  id: hotel.id,
+                  galleryImages: hotel.galleryImages,
+                  name: hotel.name,
+                  tripadvisor: hotel.ratings?.tripAdvisor?.rating,
+                  detailedFacilities: hotel.detailedFacilities || [],
+                  location: hotel.location,
+                  hotelType: hotel.hotelType,
+                  type: hotel.type,
+                  address: hotel.address,
+                  cityIdentifier: cities?.[0]?.cityIdentifier,
+                  hotelDescription:
+                    hotel.hotelContent?.texts.descriptions?.short,
+                },
+                url: hotelPage?.url ?? "",
+              }
 
+              return data
+            })
+          ))
+        )
+      }
       return hotels.filter(
         (hotel): hotel is DestinationPagesHotelData => !!hotel
       )

apps/scandic-web/services/dataCache/MemoryCache/UnstableCache/cacheOrGet.ts

@@ -31,6 +31,12 @@ export const cacheOrGet: DataCache["cacheOrGet"] = async <T>(
     revalidate: getCacheTimeInSeconds(ttl),
     tags: key,
   })()
+
+  const size = JSON.stringify(res).length / (1024 * 1024)
+  cacheLogger.debug(`'${key}': ${size}mb`)
+  if (size > 5) {
+    cacheLogger.warn(`'${key}' is larger than 5mb!`)
+  }
   cacheLogger.debug(`'${key}' took ${(performance.now() - perf).toFixed(2)}ms`)
 
   return res

apps/scandic-web/services/warmup/index.ts

@@ -1,5 +1,6 @@
 import { Lang } from "@/constants/languages"
 
+import { warmupAutoComplete } from "./wamupAutoCompleteLocations"
 import { warmupCountry } from "./warmupCountries"
 import { warmupHotelData } from "./warmupHotelData"
 import { warmupHotelIdsByCountry } from "./warmupHotelIdsByCountry"
@@ -35,6 +36,13 @@ export const warmupFunctions: Record<WarmupFunctionsKey, WarmupFunction> = {
   hotelData_fi: warmupHotelData(Lang.fi),
   hotelData_sv: warmupHotelData(Lang.sv),
   hotelData_no: warmupHotelData(Lang.no),
+
+  autoComplete_en: warmupAutoComplete(Lang.en),
+  autoComplete_da: warmupAutoComplete(Lang.da),
+  autoComplete_de: warmupAutoComplete(Lang.de),
+  autoComplete_fi: warmupAutoComplete(Lang.fi),
+  autoComplete_sv: warmupAutoComplete(Lang.sv),
+  autoComplete_no: warmupAutoComplete(Lang.no),
 }
 
 export async function warmup(key: WarmupFunctionsKey): Promise<WarmupResult> {

apps/scandic-web/services/warmup/wamupAutoCompleteLocations.ts

@@ -0,0 +1,27 @@
+import { getAutoCompleteDestinationsData } from "@/server/routers/autocomplete/destinations"
+import { getServiceToken } from "@/server/tokenManager"
+
+import type { Lang } from "@/constants/languages"
+import type { WarmupFunction, WarmupResult } from "."
+
+export const warmupAutoComplete =
+  (lang: Lang): WarmupFunction =>
+  async (): Promise<WarmupResult> => {
+    try {
+      const serviceToken = await getServiceToken()
+      await getAutoCompleteDestinationsData({
+        lang,
+        serviceToken: serviceToken.access_token,
+        warmup: true,
+      })
+    } catch (error) {
+      return {
+        status: "error",
+        error: error as Error,
+      }
+    }
+
+    return {
+      status: "completed",
+    }
+  }

apps/scandic-web/services/warmup/warmupKeys.ts

@@ -9,6 +9,7 @@ export const warmupKeys = [
   ...langs.map((lang) => `countries_${lang}` as const),
   "hotelsByCountry",
   ...langs.map((lang) => `hotelData_${lang}` as const),
+  ...langs.map((lang) => `autoComplete_${lang}` as const),
 ] as const
 
 export type WarmupFunctionsKey = (typeof warmupKeys)[number]

apps/scandic-web/types/hotel.ts

@@ -83,4 +83,4 @@ export type HotelDataWithUrl = HotelData & { url: string }
 
 export type DestinationPagesHotelData = z.output<
   typeof destinationPagesHotelDataSchema
-> & { url: string }
+>

apps/scandic-web/utils/chunk.ts

@@ -0,0 +1,11 @@
+/**
+ * Splits an array into chunks of a specified size
+ */
+export function chunk<T>(array: T[], size: number): T[][] {
+  const result: T[][] = []
+  for (let i = 0; i < array.length; i += size) {
+    result.push(array.slice(i, i + size))
+  }
+
+  return result
+}