From 196ea2994fbf70109e612f1c8d6ee7711b135f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joakim=20J=C3=A4derberg?= Date: Mon, 26 May 2025 08:23:20 +0000 Subject: [PATCH] Merged in fix/warmup-autocomplete-data (pull request #2212) warmup autocomplete data * warmup autocomplete data Approved-by: Anton Gunnarsson --- .../routers/autocomplete/destinations.ts | 209 ++++++++++-------- .../server/routers/hotels/utils.ts | 159 +++++++------ .../MemoryCache/UnstableCache/cacheOrGet.ts | 6 + apps/scandic-web/services/warmup/index.ts | 8 + .../warmup/wamupAutoCompleteLocations.ts | 27 +++ .../scandic-web/services/warmup/warmupKeys.ts | 1 + apps/scandic-web/types/hotel.ts | 2 +- apps/scandic-web/utils/chunk.ts | 11 + 8 files changed, 256 insertions(+), 167 deletions(-) create mode 100644 apps/scandic-web/services/warmup/wamupAutoCompleteLocations.ts create mode 100644 apps/scandic-web/utils/chunk.ts diff --git a/apps/scandic-web/server/routers/autocomplete/destinations.ts b/apps/scandic-web/server/routers/autocomplete/destinations.ts index 064d6553b..7dacea2ae 100644 --- a/apps/scandic-web/server/routers/autocomplete/destinations.ts +++ b/apps/scandic-web/server/routers/autocomplete/destinations.ts @@ -40,105 +40,13 @@ type DestinationsAutoCompleteOutput = { export const getDestinationsAutoCompleteRoute = safeProtectedServiceProcedure .input(destinationsAutoCompleteInputSchema) .query(async ({ ctx, input }): Promise => { - const cacheClient = await getCacheClient() - const lang = input.lang || ctx.lang - const locations: AutoCompleteLocation[] = await cacheClient.cacheOrGet( - `autocomplete:destinations:locations:${lang}`, - async () => { - const hotelUrlsPromise = safeTry(getHotelPageUrls(lang)) - const cityUrlsPromise = safeTry(getCityPageUrls(lang)) - const countryUrlsPromise = safeTry(getCountryPageUrls(lang)) - const countries = await getCountries({ - lang: lang, - serviceToken: ctx.serviceToken, - }) - - if (!countries) { - throw new Error("Unable to fetch countries") - } - - const countryNames = countries.data.map((country) => country.name) - const citiesByCountry = await getCitiesByCountry({ - countries: countryNames, - serviceToken: ctx.serviceToken, - lang, - }) - - const locations = await getLocations({ - lang: lang, - serviceToken: ctx.serviceToken, - citiesByCountry: citiesByCountry, - }) - - const activeLocations = locations.filter((location) => { - return ( - location.type === "cities" || - (location.type === "hotels" && location.isActive) - ) - }) - - const [hotelUrls, hotelUrlsError] = await hotelUrlsPromise - const [cityUrls, cityUrlsError] = await cityUrlsPromise - const [countryUrls, countryUrlsError] = await countryUrlsPromise - - if ( - hotelUrlsError || - cityUrlsError || - countryUrlsError || - !hotelUrls || - !cityUrls || - !countryUrls - ) { - throw new Error("Unable to fetch location URLs") - } - - const hotelsAndCities = activeLocations - .map((location) => { - let url: string | undefined - - if (location.type === "cities") { - url = cityUrls.find( - (c) => - c.city && - location.cityIdentifier && - c.city === location.cityIdentifier - )?.url - } - - if (location.type === "hotels") { - url = hotelUrls.find( - (h) => h.hotelId && location.id && h.hotelId === location.id - )?.url - } - - return { ...location, url } - }) - .map(mapLocationToAutoCompleteLocation) - .filter(isDefined) - - const countryAutoCompleteLocations = countries.data.map((country) => { - const url = countryUrls.find( - (c) => - c.country && - ApiCountry[lang][c.country as Country] === country.name - )?.url - - return { - id: country.id, - name: country.name, - type: "countries", - searchTokens: [country.name], - destination: "", - url, - } satisfies AutoCompleteLocation - }) - - return [...hotelsAndCities, ...countryAutoCompleteLocations] - }, - "1d" - ) + const locations: AutoCompleteLocation[] = + await getAutoCompleteDestinationsData({ + lang, + serviceToken: ctx.serviceToken, + }) const hits = filterAndCategorizeAutoComplete({ locations: locations, @@ -176,3 +84,110 @@ function isCity( ): location is AutoCompleteLocation & { type: "cities" } { return !!location && location.type === "cities" } + +export async function getAutoCompleteDestinationsData({ + lang, + serviceToken, + warmup = false, +}: { + lang: Lang + serviceToken: string + warmup?: boolean +}) { + const cacheClient = await getCacheClient() + return await cacheClient.cacheOrGet( + `autocomplete:destinations:locations:${lang}`, + async () => { + const hotelUrlsPromise = safeTry(getHotelPageUrls(lang)) + const cityUrlsPromise = safeTry(getCityPageUrls(lang)) + const countryUrlsPromise = safeTry(getCountryPageUrls(lang)) + const countries = await getCountries({ + lang: lang, + serviceToken, + }) + + if (!countries) { + throw new Error("Unable to fetch countries") + } + + const countryNames = countries.data.map((country) => country.name) + const citiesByCountry = await getCitiesByCountry({ + countries: countryNames, + serviceToken: serviceToken, + lang, + }) + + const locations = await getLocations({ + lang: lang, + serviceToken: serviceToken, + citiesByCountry: citiesByCountry, + }) + + const activeLocations = locations.filter((location) => { + return ( + location.type === "cities" || + (location.type === "hotels" && location.isActive) + ) + }) + + const [hotelUrls, hotelUrlsError] = await hotelUrlsPromise + const [cityUrls, cityUrlsError] = await cityUrlsPromise + const [countryUrls, countryUrlsError] = await countryUrlsPromise + + if ( + hotelUrlsError || + cityUrlsError || + countryUrlsError || + !hotelUrls || + !cityUrls || + !countryUrls + ) { + throw new Error("Unable to fetch location URLs") + } + + const hotelsAndCities = activeLocations + .map((location) => { + let url: string | undefined + + if (location.type === "cities") { + url = cityUrls.find( + (c) => + c.city && + location.cityIdentifier && + c.city === location.cityIdentifier + )?.url + } + + if (location.type === "hotels") { + url = hotelUrls.find( + (h) => h.hotelId && location.id && h.hotelId === location.id + )?.url + } + + return { ...location, url } + }) + .map(mapLocationToAutoCompleteLocation) + .filter(isDefined) + + const countryAutoCompleteLocations = countries.data.map((country) => { + const url = countryUrls.find( + (c) => + c.country && ApiCountry[lang][c.country as Country] === country.name + )?.url + + return { + id: country.id, + name: country.name, + type: "countries", + searchTokens: [country.name], + destination: "", + url, + } satisfies AutoCompleteLocation + }) + + return [...hotelsAndCities, ...countryAutoCompleteLocations] + }, + "1d", + { cacheStrategy: warmup ? "fetch-then-cache" : "cache-first" } + ) +} diff --git a/apps/scandic-web/server/routers/hotels/utils.ts b/apps/scandic-web/server/routers/hotels/utils.ts index a7e405816..8d3ddf0e3 100644 --- a/apps/scandic-web/server/routers/hotels/utils.ts +++ b/apps/scandic-web/server/routers/hotels/utils.ts @@ -61,6 +61,8 @@ import type { RoomConfiguration, } from "@/types/trpc/routers/hotel/roomAvailability" import type { Endpoint } from "@/lib/api/endpoints" +import { chunk } from "@/utils/chunk" +import { z } from "zod" export function getPoiGroupByCategoryName(category: string | undefined) { if (!category) return PointOfInterestGroupEnum.LOCATION @@ -275,6 +277,7 @@ export async function getLocations({ } throw new Error("downstream error") } + const apiJson = await apiResponse.json() const verifiedLocations = locationsSchema.safeParse(apiJson) if (!verifiedLocations.success) { @@ -283,46 +286,55 @@ export async function getLocations({ throw new Error("Unable to parse locations") } - return await Promise.all( - verifiedLocations.data.data.map(async (location) => { - if (location.type === "cities") { - if (citiesByCountry) { - const country = Object.keys(citiesByCountry).find((country) => - citiesByCountry[country].find( - (loc) => loc.name === location.name - ) - ) - if (country) { - return { - ...location, - country, - } - } else { - console.info( - `Location cannot be found in any of the countries cities` - ) - console.info(location) - } - } - } else if (location.type === "hotels") { - if (location.relationships.city?.url) { - const city = await getCity({ - cityUrl: location.relationships.city.url, - serviceToken, - }) - if (city) { - return deepmerge(location, { - relationships: { - city, - }, - }) - } - } - } + const chunkedLocations = chunk(verifiedLocations.data.data, 10) + let locations: z.infer["data"] = [] - return location - }) - ) + for (const chunk of chunkedLocations) { + locations = [ + ...(await Promise.all( + chunk.map(async (location) => { + if (location.type === "cities") { + if (citiesByCountry) { + const country = Object.keys(citiesByCountry).find((country) => + citiesByCountry[country].find( + (loc) => loc.name === location.name + ) + ) + if (country) { + return { + ...location, + country, + } + } else { + console.info( + `Location cannot be found in any of the countries cities` + ) + console.info(location) + } + } + } else if (location.type === "hotels") { + if (location.relationships.city?.url) { + const city = await getCity({ + cityUrl: location.relationships.city.url, + serviceToken, + }) + if (city) { + return deepmerge(location, { + relationships: { + city, + }, + }) + } + } + } + + return location + }) + )), + ] + } + + return locations }, "1d" ) @@ -506,40 +518,49 @@ export async function getHotelsByHotelIds({ cacheKey, async () => { const hotelPages = await getHotelPageUrls(lang) - const hotels = await Promise.all( - hotelIds.map(async (hotelId) => { - const hotelResponse = await getHotel( - { hotelId, language: lang, isCardOnlyPayment: false }, - serviceToken - ) + const chunkedHotelIds = chunk(hotelIds, 10) - if (!hotelResponse) { - throw new Error(`Hotel not found: ${hotelId}`) - } + const hotels: DestinationPagesHotelData[] = [] + for (const hotelIdChunk of chunkedHotelIds) { + hotels.push( + ...(await Promise.all( + hotelIdChunk.map(async (hotelId) => { + const hotelResponse = await getHotel( + { hotelId, language: lang, isCardOnlyPayment: false }, + serviceToken + ) - const hotelPage = hotelPages.find((page) => page.hotelId === hotelId) - const { hotel, cities } = hotelResponse - const data: DestinationPagesHotelData = { - hotel: { - id: hotel.id, - galleryImages: hotel.galleryImages, - name: hotel.name, - tripadvisor: hotel.ratings?.tripAdvisor?.rating, - detailedFacilities: hotel.detailedFacilities || [], - location: hotel.location, - hotelType: hotel.hotelType, - type: hotel.type, - address: hotel.address, - cityIdentifier: cities?.[0]?.cityIdentifier, - hotelDescription: hotel.hotelContent?.texts.descriptions?.short, - }, - url: hotelPage?.url ?? "", - } + if (!hotelResponse) { + throw new Error(`Hotel not found: ${hotelId}`) + } - return { ...data, url: hotelPage?.url ?? null } - }) - ) + const hotelPage = hotelPages.find( + (page) => page.hotelId === hotelId + ) + const { hotel, cities } = hotelResponse + const data: DestinationPagesHotelData = { + hotel: { + id: hotel.id, + galleryImages: hotel.galleryImages, + name: hotel.name, + tripadvisor: hotel.ratings?.tripAdvisor?.rating, + detailedFacilities: hotel.detailedFacilities || [], + location: hotel.location, + hotelType: hotel.hotelType, + type: hotel.type, + address: hotel.address, + cityIdentifier: cities?.[0]?.cityIdentifier, + hotelDescription: + hotel.hotelContent?.texts.descriptions?.short, + }, + url: hotelPage?.url ?? "", + } + return data + }) + )) + ) + } return hotels.filter( (hotel): hotel is DestinationPagesHotelData => !!hotel ) diff --git a/apps/scandic-web/services/dataCache/MemoryCache/UnstableCache/cacheOrGet.ts b/apps/scandic-web/services/dataCache/MemoryCache/UnstableCache/cacheOrGet.ts index b9539f1b4..cf5085289 100644 --- a/apps/scandic-web/services/dataCache/MemoryCache/UnstableCache/cacheOrGet.ts +++ b/apps/scandic-web/services/dataCache/MemoryCache/UnstableCache/cacheOrGet.ts @@ -31,6 +31,12 @@ export const cacheOrGet: DataCache["cacheOrGet"] = async ( revalidate: getCacheTimeInSeconds(ttl), tags: key, })() + + const size = JSON.stringify(res).length / (1024 * 1024) + cacheLogger.debug(`'${key}': ${size}mb`) + if (size > 5) { + cacheLogger.warn(`'${key}' is larger than 5mb!`) + } cacheLogger.debug(`'${key}' took ${(performance.now() - perf).toFixed(2)}ms`) return res diff --git a/apps/scandic-web/services/warmup/index.ts b/apps/scandic-web/services/warmup/index.ts index 013574eb6..a6c74e3f4 100644 --- a/apps/scandic-web/services/warmup/index.ts +++ b/apps/scandic-web/services/warmup/index.ts @@ -1,5 +1,6 @@ import { Lang } from "@/constants/languages" +import { warmupAutoComplete } from "./wamupAutoCompleteLocations" import { warmupCountry } from "./warmupCountries" import { warmupHotelData } from "./warmupHotelData" import { warmupHotelIdsByCountry } from "./warmupHotelIdsByCountry" @@ -35,6 +36,13 @@ export const warmupFunctions: Record = { hotelData_fi: warmupHotelData(Lang.fi), hotelData_sv: warmupHotelData(Lang.sv), hotelData_no: warmupHotelData(Lang.no), + + autoComplete_en: warmupAutoComplete(Lang.en), + autoComplete_da: warmupAutoComplete(Lang.da), + autoComplete_de: warmupAutoComplete(Lang.de), + autoComplete_fi: warmupAutoComplete(Lang.fi), + autoComplete_sv: warmupAutoComplete(Lang.sv), + autoComplete_no: warmupAutoComplete(Lang.no), } export async function warmup(key: WarmupFunctionsKey): Promise { diff --git a/apps/scandic-web/services/warmup/wamupAutoCompleteLocations.ts b/apps/scandic-web/services/warmup/wamupAutoCompleteLocations.ts new file mode 100644 index 000000000..f16332e77 --- /dev/null +++ b/apps/scandic-web/services/warmup/wamupAutoCompleteLocations.ts @@ -0,0 +1,27 @@ +import { getAutoCompleteDestinationsData } from "@/server/routers/autocomplete/destinations" +import { getServiceToken } from "@/server/tokenManager" + +import type { Lang } from "@/constants/languages" +import type { WarmupFunction, WarmupResult } from "." + +export const warmupAutoComplete = + (lang: Lang): WarmupFunction => + async (): Promise => { + try { + const serviceToken = await getServiceToken() + await getAutoCompleteDestinationsData({ + lang, + serviceToken: serviceToken.access_token, + warmup: true, + }) + } catch (error) { + return { + status: "error", + error: error as Error, + } + } + + return { + status: "completed", + } + } diff --git a/apps/scandic-web/services/warmup/warmupKeys.ts b/apps/scandic-web/services/warmup/warmupKeys.ts index ee8a06def..77dbab106 100644 --- a/apps/scandic-web/services/warmup/warmupKeys.ts +++ b/apps/scandic-web/services/warmup/warmupKeys.ts @@ -9,6 +9,7 @@ export const warmupKeys = [ ...langs.map((lang) => `countries_${lang}` as const), "hotelsByCountry", ...langs.map((lang) => `hotelData_${lang}` as const), + ...langs.map((lang) => `autoComplete_${lang}` as const), ] as const export type WarmupFunctionsKey = (typeof warmupKeys)[number] diff --git a/apps/scandic-web/types/hotel.ts b/apps/scandic-web/types/hotel.ts index b499f6efa..79b3e6643 100644 --- a/apps/scandic-web/types/hotel.ts +++ b/apps/scandic-web/types/hotel.ts @@ -83,4 +83,4 @@ export type HotelDataWithUrl = HotelData & { url: string } export type DestinationPagesHotelData = z.output< typeof destinationPagesHotelDataSchema -> & { url: string } +> diff --git a/apps/scandic-web/utils/chunk.ts b/apps/scandic-web/utils/chunk.ts new file mode 100644 index 000000000..d75a7884b --- /dev/null +++ b/apps/scandic-web/utils/chunk.ts @@ -0,0 +1,11 @@ +/** + * Splits an array into chunks of a specified size + */ +export function chunk(array: T[], size: number): T[][] { + const result: T[][] = [] + for (let i = 0; i < array.length; i += size) { + result.push(array.slice(i, i + size)) + } + + return result +}