Merged in fix/warmup-autocomplete-data (pull request #2212)

warmup autocomplete data

* warmup autocomplete data


Approved-by: Anton Gunnarsson

apps/scandic-web/server/routers/autocomplete/destinations.ts

@@ -40,11 +40,62 @@ type DestinationsAutoCompleteOutput = {
 export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure
   .input(destinationsAutoCompleteInputSchema)
   .query(async ({ ctx, input }): Promise<DestinationsAutoCompleteOutput> => {
-    const cacheClient = await getCacheClient()
-
     const lang = input.lang || ctx.lang
 
-    const locations: AutoCompleteLocation[] = await cacheClient.cacheOrGet(
+    const locations: AutoCompleteLocation[] =
+      await getAutoCompleteDestinationsData({
+        lang,
+        serviceToken: ctx.serviceToken,
+      })
+
+    const hits = filterAndCategorizeAutoComplete({
+      locations: locations,
+      query: input.query,
+      includeTypes: input.includeTypes,
+    })
+
+    const selectedHotel = locations.find(
+      (location) =>
+        location.type === "hotels" && location.id === input.selectedHotelId
+    )
+
+    const selectedCity = locations.find(
+      (location) =>
+        location.type === "cities" && location.name === input.selectedCity
+    )
+
+    return {
+      hits: hits,
+      currentSelection: {
+        city: isCity(selectedCity) ? selectedCity : null,
+        hotel: isHotel(selectedHotel) ? selectedHotel : null,
+      },
+    }
+  })
+
+function isHotel(
+  location: AutoCompleteLocation | null | undefined
+): location is AutoCompleteLocation & { type: "hotels" } {
+  return !!location && location.type === "hotels"
+}
+
+function isCity(
+  location: AutoCompleteLocation | null | undefined
+): location is AutoCompleteLocation & { type: "cities" } {
+  return !!location && location.type === "cities"
+}
+
+export async function getAutoCompleteDestinationsData({
+  lang,
+  serviceToken,
+  warmup = false,
+}: {
+  lang: Lang
+  serviceToken: string
+  warmup?: boolean
+}) {
+  const cacheClient = await getCacheClient()
+  return await cacheClient.cacheOrGet(
     `autocomplete:destinations:locations:${lang}`,
     async () => {
       const hotelUrlsPromise = safeTry(getHotelPageUrls(lang))
@@ -52,7 +103,7 @@ export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure
       const countryUrlsPromise = safeTry(getCountryPageUrls(lang))
       const countries = await getCountries({
         lang: lang,
-          serviceToken: ctx.serviceToken,
+        serviceToken,
       })
 
       if (!countries) {
@@ -62,13 +113,13 @@ export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure
       const countryNames = countries.data.map((country) => country.name)
       const citiesByCountry = await getCitiesByCountry({
         countries: countryNames,
-          serviceToken: ctx.serviceToken,
+        serviceToken: serviceToken,
         lang,
       })
 
       const locations = await getLocations({
         lang: lang,
-          serviceToken: ctx.serviceToken,
+        serviceToken: serviceToken,
         citiesByCountry: citiesByCountry,
       })
 
@@ -121,8 +172,7 @@ export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure
       const countryAutoCompleteLocations = countries.data.map((country) => {
         const url = countryUrls.find(
           (c) =>
-              c.country &&
+            c.country && ApiCountry[lang][c.country as Country] === country.name
-              ApiCountry[lang][c.country as Country] === country.name
         )?.url
 
         return {
@@ -137,42 +187,7 @@ export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure
 
       return [...hotelsAndCities, ...countryAutoCompleteLocations]
     },
-      "1d"
+    "1d",
+    { cacheStrategy: warmup ? "fetch-then-cache" : "cache-first" }
   )
-
-    const hits = filterAndCategorizeAutoComplete({
-      locations: locations,
-      query: input.query,
-      includeTypes: input.includeTypes,
-    })
-
-    const selectedHotel = locations.find(
-      (location) =>
-        location.type === "hotels" && location.id === input.selectedHotelId
-    )
-
-    const selectedCity = locations.find(
-      (location) =>
-        location.type === "cities" && location.name === input.selectedCity
-    )
-
-    return {
-      hits: hits,
-      currentSelection: {
-        city: isCity(selectedCity) ? selectedCity : null,
-        hotel: isHotel(selectedHotel) ? selectedHotel : null,
-      },
-    }
-  })
-
-function isHotel(
-  location: AutoCompleteLocation | null | undefined
-): location is AutoCompleteLocation & { type: "hotels" } {
-  return !!location && location.type === "hotels"
-}
-
-function isCity(
-  location: AutoCompleteLocation | null | undefined
-): location is AutoCompleteLocation & { type: "cities" } {
-  return !!location && location.type === "cities"
 }

apps/scandic-web/server/routers/hotels/utils.ts

@@ -61,6 +61,8 @@ import type {
   RoomConfiguration,
 } from "@/types/trpc/routers/hotel/roomAvailability"
 import type { Endpoint } from "@/lib/api/endpoints"
+import { chunk } from "@/utils/chunk"
+import { z } from "zod"
 
 export function getPoiGroupByCategoryName(category: string | undefined) {
   if (!category) return PointOfInterestGroupEnum.LOCATION
@@ -275,6 +277,7 @@ export async function getLocations({
         }
         throw new Error("downstream error")
       }
+
       const apiJson = await apiResponse.json()
       const verifiedLocations = locationsSchema.safeParse(apiJson)
       if (!verifiedLocations.success) {
@@ -283,8 +286,13 @@ export async function getLocations({
         throw new Error("Unable to parse locations")
       }
 
-      return await Promise.all(
+      const chunkedLocations = chunk(verifiedLocations.data.data, 10)
-        verifiedLocations.data.data.map(async (location) => {
+      let locations: z.infer<typeof locationsSchema>["data"] = []
+
+      for (const chunk of chunkedLocations) {
+        locations = [
+          ...(await Promise.all(
+            chunk.map(async (location) => {
               if (location.type === "cities") {
                 if (citiesByCountry) {
                   const country = Object.keys(citiesByCountry).find((country) =>
@@ -322,7 +330,11 @@ export async function getLocations({
 
               return location
             })
-      )
+          )),
+        ]
+      }
+
+      return locations
     },
     "1d"
   )
@@ -506,8 +518,13 @@ export async function getHotelsByHotelIds({
     cacheKey,
     async () => {
       const hotelPages = await getHotelPageUrls(lang)
-      const hotels = await Promise.all(
+      const chunkedHotelIds = chunk(hotelIds, 10)
-        hotelIds.map(async (hotelId) => {
+
+      const hotels: DestinationPagesHotelData[] = []
+      for (const hotelIdChunk of chunkedHotelIds) {
+        hotels.push(
+          ...(await Promise.all(
+            hotelIdChunk.map(async (hotelId) => {
               const hotelResponse = await getHotel(
                 { hotelId, language: lang, isCardOnlyPayment: false },
                 serviceToken
@@ -517,7 +534,9 @@ export async function getHotelsByHotelIds({
                 throw new Error(`Hotel not found: ${hotelId}`)
               }
 
-          const hotelPage = hotelPages.find((page) => page.hotelId === hotelId)
+              const hotelPage = hotelPages.find(
+                (page) => page.hotelId === hotelId
+              )
               const { hotel, cities } = hotelResponse
               const data: DestinationPagesHotelData = {
                 hotel: {
@@ -531,15 +550,17 @@ export async function getHotelsByHotelIds({
                   type: hotel.type,
                   address: hotel.address,
                   cityIdentifier: cities?.[0]?.cityIdentifier,
-              hotelDescription: hotel.hotelContent?.texts.descriptions?.short,
+                  hotelDescription:
+                    hotel.hotelContent?.texts.descriptions?.short,
                 },
                 url: hotelPage?.url ?? "",
               }
 
-          return { ...data, url: hotelPage?.url ?? null }
+              return data
             })
+          ))
         )
-
+      }
       return hotels.filter(
         (hotel): hotel is DestinationPagesHotelData => !!hotel
       )

apps/scandic-web/services/dataCache/MemoryCache/UnstableCache/cacheOrGet.ts

@@ -31,6 +31,12 @@ export const cacheOrGet: DataCache["cacheOrGet"] = async <T>(
     revalidate: getCacheTimeInSeconds(ttl),
     tags: key,
   })()
+
+  const size = JSON.stringify(res).length / (1024 * 1024)
+  cacheLogger.debug(`'${key}': ${size}mb`)
+  if (size > 5) {
+    cacheLogger.warn(`'${key}' is larger than 5mb!`)
+  }
   cacheLogger.debug(`'${key}' took ${(performance.now() - perf).toFixed(2)}ms`)
 
   return res

apps/scandic-web/services/warmup/index.ts

@@ -1,5 +1,6 @@
 import { Lang } from "@/constants/languages"
 
+import { warmupAutoComplete } from "./wamupAutoCompleteLocations"
 import { warmupCountry } from "./warmupCountries"
 import { warmupHotelData } from "./warmupHotelData"
 import { warmupHotelIdsByCountry } from "./warmupHotelIdsByCountry"
@@ -35,6 +36,13 @@ export const warmupFunctions: Record<WarmupFunctionsKey, WarmupFunction> = {
   hotelData_fi: warmupHotelData(Lang.fi),
   hotelData_sv: warmupHotelData(Lang.sv),
   hotelData_no: warmupHotelData(Lang.no),
+
+  autoComplete_en: warmupAutoComplete(Lang.en),
+  autoComplete_da: warmupAutoComplete(Lang.da),
+  autoComplete_de: warmupAutoComplete(Lang.de),
+  autoComplete_fi: warmupAutoComplete(Lang.fi),
+  autoComplete_sv: warmupAutoComplete(Lang.sv),
+  autoComplete_no: warmupAutoComplete(Lang.no),
 }
 
 export async function warmup(key: WarmupFunctionsKey): Promise<WarmupResult> {

apps/scandic-web/services/warmup/wamupAutoCompleteLocations.ts

@@ -0,0 +1,27 @@
+import { getAutoCompleteDestinationsData } from "@/server/routers/autocomplete/destinations"
+import { getServiceToken } from "@/server/tokenManager"
+
+import type { Lang } from "@/constants/languages"
+import type { WarmupFunction, WarmupResult } from "."
+
+export const warmupAutoComplete =
+  (lang: Lang): WarmupFunction =>
+  async (): Promise<WarmupResult> => {
+    try {
+      const serviceToken = await getServiceToken()
+      await getAutoCompleteDestinationsData({
+        lang,
+        serviceToken: serviceToken.access_token,
+        warmup: true,
+      })
+    } catch (error) {
+      return {
+        status: "error",
+        error: error as Error,
+      }
+    }
+
+    return {
+      status: "completed",
+    }
+  }

apps/scandic-web/services/warmup/warmupKeys.ts

@@ -9,6 +9,7 @@ export const warmupKeys = [
   ...langs.map((lang) => `countries_${lang}` as const),
   "hotelsByCountry",
   ...langs.map((lang) => `hotelData_${lang}` as const),
+  ...langs.map((lang) => `autoComplete_${lang}` as const),
 ] as const
 
 export type WarmupFunctionsKey = (typeof warmupKeys)[number]

apps/scandic-web/types/hotel.ts

@@ -83,4 +83,4 @@ export type HotelDataWithUrl = HotelData & { url: string }
 
 export type DestinationPagesHotelData = z.output<
   typeof destinationPagesHotelDataSchema
-> & { url: string }
+>

apps/scandic-web/utils/chunk.ts

@@ -0,0 +1,11 @@
+/**
+ * Splits an array into chunks of a specified size
+ */
+export function chunk<T>(array: T[], size: number): T[][] {
+  const result: T[][] = []
+  for (let i = 0; i < array.length; i += size) {
+    result.push(array.slice(i, i + size))
+  }
+
+  return result
+}