scandic/web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged in fix/SW-1819-batching-city-urls (pull request #1477)

Browse Source 
fix(SW-1819): Batching fetch for city page urls

* fix(SW-1819): Batching fetch for city page urls


Approved-by: Fredrik Thorsson
Approved-by: Matilda Landström
This commit is contained in:
Erik Tiekstra
2025-03-05 10:15:52 +00:00
parent 6897e3bb0d
commit 76c20df8e8
10 changed files with 143 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
apps/scandic-web/components/ContentType/DestinationPage/CityListing/CityListingItem/index.tsx
View File

@@ -22,7 +22,7 @@ interface CityListingItemProps {
export default function CityListingItem({ city }: CityListingItemProps) { export default function CityListingItem({ city }: CityListingItemProps) {
const intl = useIntl() const intl = useIntl()
const galleryImages = mapImageVaultImagesToGalleryImages(city.images) const galleryImages = mapImageVaultImagesToGalleryImages(city.images || [])
return ( return (
<article className={styles.container}> <article className={styles.container}>
@@ -40,7 +40,9 @@ export default function CityListingItem({ city }: CityListingItemProps) {
<Subtitle asChild> <Subtitle asChild>
<h3>{city.heading}</h3> <h3>{city.heading}</h3>
</Subtitle> </Subtitle>
{city.experiences?.length && (
<ExperienceList experiences={city.experiences} /> <ExperienceList experiences={city.experiences} />
)}
<Body>{city.preamble}</Body> <Body>{city.preamble}</Body>
<Divider variant="horizontal" color="primaryLightSubtle" /> <Divider variant="horizontal" color="primaryLightSubtle" />

4
apps/scandic-web/components/ContentType/DestinationPage/DestinationCountryPage/CountryMap/CityListItem/index.tsx
View File

@@ -20,7 +20,7 @@ interface CityListItemProps {
export default function CityListItem({ city }: CityListItemProps) { export default function CityListItem({ city }: CityListItemProps) {
const intl = useIntl() const intl = useIntl()
const galleryImages = mapImageVaultImagesToGalleryImages(city.images) const galleryImages = mapImageVaultImagesToGalleryImages(city.images || [])
return ( return (
<article className={styles.cityListItem}> <article className={styles.cityListItem}>
@@ -38,9 +38,11 @@ export default function CityListItem({ city }: CityListItemProps) {
<Subtitle asChild> <Subtitle asChild>
<h3>{city.heading}</h3> <h3>{city.heading}</h3>
</Subtitle> </Subtitle>
{city.experiences?.length && (
<div className={styles.experienceList}> <div className={styles.experienceList}>
<ExperienceList experiences={city.experiences} /> <ExperienceList experiences={city.experiences} />
</div> </div>
)}
<div className={styles.ctaWrapper}> <div className={styles.ctaWrapper}>
<Button intent="tertiary" theme="base" size="small" asChild> <Button intent="tertiary" theme="base" size="small" asChild>
<Link href={city.url}> <Link href={city.url}>

2
apps/scandic-web/components/ContentType/DestinationPage/TopImages/index.tsx
View File

@@ -23,7 +23,7 @@ export default function TopImages({ images, destinationName }: TopImageProps) {
open: false, open: false,
activeIndex: 0, activeIndex: 0,
}) })
const lightboxImages = mapImageVaultImagesToGalleryImages(images) const lightboxImages = mapImageVaultImagesToGalleryImages(images || [])
const maxWidth = 1366 // 1366px is the max width of the image container const maxWidth = 1366 // 1366px is the max width of the image container
const visibleImages = images.slice(0, 3) const visibleImages = images.slice(0, 3)

7
apps/scandic-web/lib/graphql/Query/DestinationCityPage/DestinationCityPageCount.graphql Normal file
View File

@@ -0,0 +1,7 @@
#import "../../Fragments/System.graphql"
query GetCityPageCount($locale: String!) {
all_destination_city_page(locale: $locale) {
total
}
}

4
apps/scandic-web/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql
View File

@@ -1,7 +1,7 @@
#import "../../Fragments/System.graphql" #import "../../Fragments/System.graphql"
query GetCityPageUrls($locale: String!) { query GetCityPageUrls($locale: String!, $skip: Int) {
all_destination_city_page(locale: $locale) { all_destination_city_page(locale: $locale, limit: 100, skip: $skip) {
items { items {
url url
destination_settings { destination_settings {

24
apps/scandic-web/server/routers/contentstack/destinationCityPage/output.ts
View File

@@ -74,14 +74,16 @@ export const destinationCityListDataSchema = z
}) })
.transform( .transform(
({ destination_experiences }) => destination_experiences ({ destination_experiences }) => destination_experiences
), )
.nullish(),
images: z images: z
.array(z.object({ image: tempImageVaultAssetSchema })) .array(z.object({ image: tempImageVaultAssetSchema }))
.transform((images) => .transform((images) =>
images images
.map((image) => image.image) .map((image) => image.image)
.filter((image): image is ImageVaultAsset => !!image) .filter((image): image is ImageVaultAsset => !!image)
), )
.nullish(),
url: z.string(), url: z.string(),
system: systemSchema, system: systemSchema,
}) })
@@ -192,6 +194,14 @@ export const destinationCityPageSchema = z
} }
}) })
export const cityPageCountSchema = z
.object({
all_destination_city_page: z.object({
total: z.number(),
}),
})
.transform(({ all_destination_city_page }) => all_destination_city_page.total)
export const cityPageUrlsSchema = z export const cityPageUrlsSchema = z
.object({ .object({
all_destination_city_page: z.object({ all_destination_city_page: z.object({
@@ -213,6 +223,16 @@ export const cityPageUrlsSchema = z
}) })
.transform(({ all_destination_city_page }) => all_destination_city_page.items) .transform(({ all_destination_city_page }) => all_destination_city_page.items)
export const batchedCityPageUrlsSchema = z
.array(
z.object({
data: cityPageUrlsSchema,
})
)
.transform((allItems) => {
return allItems.flatMap((item) => item.data)
})
/** REFS */ /** REFS */
const destinationCityPageContentRefs = z const destinationCityPageContentRefs = z
.object({ .object({

10
apps/scandic-web/server/routers/contentstack/destinationCityPage/telemetry.ts
View File

@@ -22,6 +22,16 @@ export const getDestinationCityPageFailCounter = meter.createCounter(
"trpc.contentstack.destinationCityPage.get-fail" "trpc.contentstack.destinationCityPage.get-fail"
) )
export const getCityPageCountCounter = meter.createCounter(
"trpc.contentstack.cityPageCount.get"
)
export const getCityPageCountSuccessCounter = meter.createCounter(
"trpc.contentstack.cityPageCount.get-success"
)
export const getCityPageCountFailCounter = meter.createCounter(
"trpc.contentstack.cityPageCount.get-fail"
)
export const getCityPageUrlsCounter = meter.createCounter( export const getCityPageUrlsCounter = meter.createCounter(
"trpc.contentstack.cityPageUrls.get" "trpc.contentstack.cityPageUrls.get"
) )

93
apps/scandic-web/server/routers/contentstack/destinationCityPage/utils.ts
View File

@@ -1,19 +1,26 @@
import { GetCityPageCount } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageCount.graphql"
import { GetCityPageUrls } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql" import { GetCityPageUrls } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql"
import { request } from "@/lib/graphql/request" import { request } from "@/lib/graphql/request"
import { generateTag, generateTagsFromSystem } from "@/utils/generateTag" import { generateTag, generateTagsFromSystem } from "@/utils/generateTag"
import { cityPageUrlsSchema } from "./output" import { batchedCityPageUrlsSchema, cityPageCountSchema } from "./output"
import { import {
getCityPageCountCounter,
getCityPageCountFailCounter,
getCityPageCountSuccessCounter,
getCityPageUrlsCounter, getCityPageUrlsCounter,
getCityPageUrlsFailCounter, getCityPageUrlsFailCounter,
getCityPageUrlsSuccessCounter, getCityPageUrlsSuccessCounter,
} from "./telemetry" } from "./telemetry"
import type { BatchRequestDocument } from "graphql-request"
import { DestinationCityPageEnum } from "@/types/enums/destinationCityPage" import { DestinationCityPageEnum } from "@/types/enums/destinationCityPage"
import type { System } from "@/types/requests/system" import type { System } from "@/types/requests/system"
import type { import type {
DestinationCityPageRefs, DestinationCityPageRefs,
GetCityPageCountData,
GetCityPageUrlsData, GetCityPageUrlsData,
} from "@/types/trpc/routers/contentstack/destinationCityPage" } from "@/types/trpc/routers/contentstack/destinationCityPage"
import type { Lang } from "@/constants/languages" import type { Lang } from "@/constants/languages"
@@ -65,40 +72,100 @@ export function getConnections({
return connections return connections
} }
export async function getCityPageUrls(lang: Lang) { export async function getCityPageCount(lang: Lang) {
getCityPageUrlsCounter.add(1, { lang }) getCityPageCountCounter.add(1, { lang })
console.info( console.info(
"contentstack.cityPageUrls start", "contentstack.cityPageCount start",
JSON.stringify({ query: { lang } }) JSON.stringify({ query: { lang } })
) )
const tag = `${lang}:city_page_urls` const tags = [`${lang}:city_page_count`]
const response = await request<GetCityPageUrlsData>( const response = await request<GetCityPageCountData>(
GetCityPageUrls, GetCityPageCount,
{ {
locale: lang, locale: lang,
}, },
{ {
cache: "force-cache", cache: "force-cache",
next: { next: {
tags: [tag], tags,
}, },
} }
) )
if (!response.data) { if (!response.data) {
getCityPageUrlsFailCounter.add(1, { getCityPageCountFailCounter.add(1, {
lang, lang,
error_type: "not_found", error_type: "not_found",
error: `Destination city pages not found for lang: ${lang}`, error: `City pages count not found for lang: ${lang}`,
}) })
console.error( console.error(
"contentstack.cityPageUrls not found error", "contentstack.cityPageCount not found error",
JSON.stringify({ query: { lang } }) JSON.stringify({ query: { lang } })
) )
return 0
}
const validatedResponse = cityPageCountSchema.safeParse(response.data)
if (!validatedResponse.success) {
getCityPageCountFailCounter.add(1, {
lang,
error_type: "validation_error",
error: JSON.stringify(validatedResponse.error),
})
console.error(
"contentstack.hotelPageCount validation error",
JSON.stringify({
query: { lang },
error: validatedResponse.error,
})
)
return 0
}
getCityPageCountSuccessCounter.add(1, { lang })
console.info(
"contentstack.cityPageCount success",
JSON.stringify({ query: { lang } })
)
return validatedResponse.data
}
export async function getCityPageUrls(lang: Lang) {
getCityPageUrlsCounter.add(1, { lang })
console.info(
"contentstack.cityPageUrls start",
JSON.stringify({ query: { lang } })
)
const count = await getCityPageCount(lang)
if (count === 0) {
return [] return []
} }
const validatedResponse = cityPageUrlsSchema.safeParse(response.data) // Calculating the amount of requests needed to fetch all pages.
// Contentstack has a limit of 100 items per request.
// So we need to make multiple requests to fetch urls to all pages.
// The `batchRequest` function is not working here, because the arrayMerge is
// used for other purposes.
const amountOfRequests = Math.ceil(count / 100)
const requests: (BatchRequestDocument & { options?: RequestInit })[] =
Array.from({ length: amountOfRequests }).map((_, i) => ({
document: GetCityPageUrls,
variables: { locale: lang, skip: i * 100 },
options: {
cache: "force-cache",
next: {
tags: [`${lang}:city_page_urls_batch_${i}`],
},
},
}))
const batchedResponse = await Promise.all(
requests.map((req) =>
request<GetCityPageUrlsData>(req.document, req.variables, req.options)
)
)
const validatedResponse = batchedCityPageUrlsSchema.safeParse(batchedResponse)
if (!validatedResponse.success) { if (!validatedResponse.success) {
getCityPageUrlsFailCounter.add(1, { getCityPageUrlsFailCounter.add(1, {

16
apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts
View File

@@ -208,9 +208,9 @@ export async function getHotelPageUrls(lang: Lang) {
"contentstack.hotelPageUrls start", "contentstack.hotelPageUrls start",
JSON.stringify({ query: { lang } }) JSON.stringify({ query: { lang } })
) )
const hotelPageCount = await getHotelPageCount(lang) const count = await getHotelPageCount(lang)
if (hotelPageCount === 0) { if (count === 0) {
return [] return []
} }
@@ -219,7 +219,7 @@ export async function getHotelPageUrls(lang: Lang) {
// So we need to make multiple requests to fetch urls to all hotel pages. // So we need to make multiple requests to fetch urls to all hotel pages.
// The `batchRequest` function is not working here, because the arrayMerge is // The `batchRequest` function is not working here, because the arrayMerge is
// used for other purposes. // used for other purposes.
const amountOfRequests = Math.ceil(hotelPageCount / 100) const amountOfRequests = Math.ceil(count / 100)
const requests: (BatchRequestDocument & { options?: RequestInit })[] = const requests: (BatchRequestDocument & { options?: RequestInit })[] =
Array.from({ length: amountOfRequests }).map((_, i) => ({ Array.from({ length: amountOfRequests }).map((_, i) => ({
document: GetHotelPageUrls, document: GetHotelPageUrls,
@@ -238,20 +238,20 @@ export async function getHotelPageUrls(lang: Lang) {
) )
) )
const validatedHotelPageUrls = const validatedResponse =
batchedHotelPageUrlsSchema.safeParse(batchedResponse) batchedHotelPageUrlsSchema.safeParse(batchedResponse)
if (!validatedHotelPageUrls.success) { if (!validatedResponse.success) {
getHotelPageUrlsFailCounter.add(1, { getHotelPageUrlsFailCounter.add(1, {
lang, lang,
error_type: "validation_error", error_type: "validation_error",
error: JSON.stringify(validatedHotelPageUrls.error), error: JSON.stringify(validatedResponse.error),
}) })
console.error( console.error(
"contentstack.hotelPageUrls validation error", "contentstack.hotelPageUrls validation error",
JSON.stringify({ JSON.stringify({
query: { lang }, query: { lang },
error: validatedHotelPageUrls.error, error: validatedResponse.error,
}) })
) )
return [] return []
@@ -262,5 +262,5 @@ export async function getHotelPageUrls(lang: Lang) {
JSON.stringify({ query: { lang } }) JSON.stringify({ query: { lang } })
) )
return validatedHotelPageUrls.data return validatedResponse.data
} }

3
apps/scandic-web/types/trpc/routers/contentstack/destinationCityPage.ts
View File

@@ -2,6 +2,7 @@ import type { z } from "zod"
import type { import type {
blocksSchema, blocksSchema,
cityPageCountSchema,
cityPageUrlsSchema, cityPageUrlsSchema,
destinationCityListDataSchema, destinationCityListDataSchema,
destinationCityPageRefsSchema, destinationCityPageRefsSchema,
@@ -19,6 +20,8 @@ export interface GetDestinationCityListDataResponse
export interface DestinationCityListData export interface DestinationCityListData
extends z.output<typeof destinationCityListDataSchema> {} extends z.output<typeof destinationCityListDataSchema> {}
export interface GetCityPageCountData
extends z.input<typeof cityPageCountSchema> {}
export interface GetCityPageUrlsData export interface GetCityPageUrlsData
extends z.input<typeof cityPageUrlsSchema> {} extends z.input<typeof cityPageUrlsSchema> {}