scandic/web
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged in fix/SW-1819-batching-city-urls (pull request #1477)

Browse Source 
fix(SW-1819): Batching fetch for city page urls

* fix(SW-1819): Batching fetch for city page urls


Approved-by: Fredrik Thorsson
Approved-by: Matilda Landström
This commit is contained in:
Erik Tiekstra
2025-03-05 10:15:52 +00:00
parent 6897e3bb0d
commit 76c20df8e8
10 changed files with 143 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
apps/scandic-web/components/ContentType/DestinationPage/CityListing/CityListingItem/index.tsx
View File

@@ -22,7 +22,7 @@ interface CityListingItemProps {
export default function CityListingItem({ city }: CityListingItemProps) {
const intl = useIntl()
const galleryImages = mapImageVaultImagesToGalleryImages(city.images)
const galleryImages = mapImageVaultImagesToGalleryImages(city.images || [])
return (
<article className={styles.container}>
@@ -40,7 +40,9 @@ export default function CityListingItem({ city }: CityListingItemProps) {
<Subtitle asChild>
<h3>{city.heading}</h3>
</Subtitle>
<ExperienceList experiences={city.experiences} />
{city.experiences?.length && (
<ExperienceList experiences={city.experiences} />
)}
<Body>{city.preamble}</Body>
<Divider variant="horizontal" color="primaryLightSubtle" />

10
apps/scandic-web/components/ContentType/DestinationPage/DestinationCountryPage/CountryMap/CityListItem/index.tsx
View File

@@ -20,7 +20,7 @@ interface CityListItemProps {
export default function CityListItem({ city }: CityListItemProps) {
const intl = useIntl()
const galleryImages = mapImageVaultImagesToGalleryImages(city.images)
const galleryImages = mapImageVaultImagesToGalleryImages(city.images || [])
return (
<article className={styles.cityListItem}>
@@ -38,9 +38,11 @@ export default function CityListItem({ city }: CityListItemProps) {
<Subtitle asChild>
<h3>{city.heading}</h3>
</Subtitle>
<div className={styles.experienceList}>
<ExperienceList experiences={city.experiences} />
</div>
{city.experiences?.length && (
<div className={styles.experienceList}>
<ExperienceList experiences={city.experiences} />
</div>
)}
<div className={styles.ctaWrapper}>
<Button intent="tertiary" theme="base" size="small" asChild>
<Link href={city.url}>

2
apps/scandic-web/components/ContentType/DestinationPage/TopImages/index.tsx
View File

@@ -23,7 +23,7 @@ export default function TopImages({ images, destinationName }: TopImageProps) {
open: false,
activeIndex: 0,
})
const lightboxImages = mapImageVaultImagesToGalleryImages(images)
const lightboxImages = mapImageVaultImagesToGalleryImages(images || [])
const maxWidth = 1366 // 1366px is the max width of the image container
const visibleImages = images.slice(0, 3)

7
apps/scandic-web/lib/graphql/Query/DestinationCityPage/DestinationCityPageCount.graphql Normal file
View File

@@ -0,0 +1,7 @@
#import "../../Fragments/System.graphql"
query GetCityPageCount($locale: String!) {
all_destination_city_page(locale: $locale) {
total
}
}

4
apps/scandic-web/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql
View File

@@ -1,7 +1,7 @@
#import "../../Fragments/System.graphql"
query GetCityPageUrls($locale: String!) {
all_destination_city_page(locale: $locale) {
query GetCityPageUrls($locale: String!, $skip: Int) {
all_destination_city_page(locale: $locale, limit: 100, skip: $skip) {
items {
url
destination_settings {

24
apps/scandic-web/server/routers/contentstack/destinationCityPage/output.ts
View File

@@ -74,14 +74,16 @@ export const destinationCityListDataSchema = z
})
.transform(
({ destination_experiences }) => destination_experiences
),
)
.nullish(),
images: z
.array(z.object({ image: tempImageVaultAssetSchema }))
.transform((images) =>
images
.map((image) => image.image)
.filter((image): image is ImageVaultAsset => !!image)
),
)
.nullish(),
url: z.string(),
system: systemSchema,
})
@@ -192,6 +194,14 @@ export const destinationCityPageSchema = z
}
})
export const cityPageCountSchema = z
.object({
all_destination_city_page: z.object({
total: z.number(),
}),
})
.transform(({ all_destination_city_page }) => all_destination_city_page.total)
export const cityPageUrlsSchema = z
.object({
all_destination_city_page: z.object({
@@ -213,6 +223,16 @@ export const cityPageUrlsSchema = z
})
.transform(({ all_destination_city_page }) => all_destination_city_page.items)
export const batchedCityPageUrlsSchema = z
.array(
z.object({
data: cityPageUrlsSchema,
})
)
.transform((allItems) => {
return allItems.flatMap((item) => item.data)
})
/** REFS */
const destinationCityPageContentRefs = z
.object({

10
apps/scandic-web/server/routers/contentstack/destinationCityPage/telemetry.ts
View File

@@ -22,6 +22,16 @@ export const getDestinationCityPageFailCounter = meter.createCounter(
"trpc.contentstack.destinationCityPage.get-fail"
)
export const getCityPageCountCounter = meter.createCounter(
"trpc.contentstack.cityPageCount.get"
)
export const getCityPageCountSuccessCounter = meter.createCounter(
"trpc.contentstack.cityPageCount.get-success"
)
export const getCityPageCountFailCounter = meter.createCounter(
"trpc.contentstack.cityPageCount.get-fail"
)
export const getCityPageUrlsCounter = meter.createCounter(
"trpc.contentstack.cityPageUrls.get"
)

93
apps/scandic-web/server/routers/contentstack/destinationCityPage/utils.ts
View File

@@ -1,19 +1,26 @@
import { GetCityPageCount } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageCount.graphql"
import { GetCityPageUrls } from "@/lib/graphql/Query/DestinationCityPage/DestinationCityPageUrl.graphql"
import { request } from "@/lib/graphql/request"
import { generateTag, generateTagsFromSystem } from "@/utils/generateTag"
import { cityPageUrlsSchema } from "./output"
import { batchedCityPageUrlsSchema, cityPageCountSchema } from "./output"
import {
getCityPageCountCounter,
getCityPageCountFailCounter,
getCityPageCountSuccessCounter,
getCityPageUrlsCounter,
getCityPageUrlsFailCounter,
getCityPageUrlsSuccessCounter,
} from "./telemetry"
import type { BatchRequestDocument } from "graphql-request"
import { DestinationCityPageEnum } from "@/types/enums/destinationCityPage"
import type { System } from "@/types/requests/system"
import type {
DestinationCityPageRefs,
GetCityPageCountData,
GetCityPageUrlsData,
} from "@/types/trpc/routers/contentstack/destinationCityPage"
import type { Lang } from "@/constants/languages"
@@ -65,40 +72,100 @@ export function getConnections({
return connections
}
export async function getCityPageUrls(lang: Lang) {
getCityPageUrlsCounter.add(1, { lang })
export async function getCityPageCount(lang: Lang) {
getCityPageCountCounter.add(1, { lang })
console.info(
"contentstack.cityPageUrls start",
"contentstack.cityPageCount start",
JSON.stringify({ query: { lang } })
)
const tag = `${lang}:city_page_urls`
const response = await request<GetCityPageUrlsData>(
GetCityPageUrls,
const tags = [`${lang}:city_page_count`]
const response = await request<GetCityPageCountData>(
GetCityPageCount,
{
locale: lang,
},
{
cache: "force-cache",
next: {
tags: [tag],
tags,
},
}
)
if (!response.data) {
getCityPageUrlsFailCounter.add(1, {
getCityPageCountFailCounter.add(1, {
lang,
error_type: "not_found",
error: `Destination city pages not found for lang: ${lang}`,
error: `City pages count not found for lang: ${lang}`,
})
console.error(
"contentstack.cityPageUrls not found error",
"contentstack.cityPageCount not found error",
JSON.stringify({ query: { lang } })
)
return 0
}
const validatedResponse = cityPageCountSchema.safeParse(response.data)
if (!validatedResponse.success) {
getCityPageCountFailCounter.add(1, {
lang,
error_type: "validation_error",
error: JSON.stringify(validatedResponse.error),
})
console.error(
"contentstack.hotelPageCount validation error",
JSON.stringify({
query: { lang },
error: validatedResponse.error,
})
)
return 0
}
getCityPageCountSuccessCounter.add(1, { lang })
console.info(
"contentstack.cityPageCount success",
JSON.stringify({ query: { lang } })
)
return validatedResponse.data
}
export async function getCityPageUrls(lang: Lang) {
getCityPageUrlsCounter.add(1, { lang })
console.info(
"contentstack.cityPageUrls start",
JSON.stringify({ query: { lang } })
)
const count = await getCityPageCount(lang)
if (count === 0) {
return []
}
const validatedResponse = cityPageUrlsSchema.safeParse(response.data)
// Calculating the amount of requests needed to fetch all pages.
// Contentstack has a limit of 100 items per request.
// So we need to make multiple requests to fetch urls to all pages.
// The `batchRequest` function is not working here, because the arrayMerge is
// used for other purposes.
const amountOfRequests = Math.ceil(count / 100)
const requests: (BatchRequestDocument & { options?: RequestInit })[] =
Array.from({ length: amountOfRequests }).map((_, i) => ({
document: GetCityPageUrls,
variables: { locale: lang, skip: i * 100 },
options: {
cache: "force-cache",
next: {
tags: [`${lang}:city_page_urls_batch_${i}`],
},
},
}))
const batchedResponse = await Promise.all(
requests.map((req) =>
request<GetCityPageUrlsData>(req.document, req.variables, req.options)
)
)
const validatedResponse = batchedCityPageUrlsSchema.safeParse(batchedResponse)
if (!validatedResponse.success) {
getCityPageUrlsFailCounter.add(1, {

16
apps/scandic-web/server/routers/contentstack/hotelPage/utils.ts
View File

@@ -208,9 +208,9 @@ export async function getHotelPageUrls(lang: Lang) {
"contentstack.hotelPageUrls start",
JSON.stringify({ query: { lang } })
)
const hotelPageCount = await getHotelPageCount(lang)
const count = await getHotelPageCount(lang)
if (hotelPageCount === 0) {
if (count === 0) {
return []
}
@@ -219,7 +219,7 @@ export async function getHotelPageUrls(lang: Lang) {
// So we need to make multiple requests to fetch urls to all hotel pages.
// The `batchRequest` function is not working here, because the arrayMerge is
// used for other purposes.
const amountOfRequests = Math.ceil(hotelPageCount / 100)
const amountOfRequests = Math.ceil(count / 100)
const requests: (BatchRequestDocument & { options?: RequestInit })[] =
Array.from({ length: amountOfRequests }).map((_, i) => ({
document: GetHotelPageUrls,
@@ -238,20 +238,20 @@ export async function getHotelPageUrls(lang: Lang) {
)
)
const validatedHotelPageUrls =
const validatedResponse =
batchedHotelPageUrlsSchema.safeParse(batchedResponse)
if (!validatedHotelPageUrls.success) {
if (!validatedResponse.success) {
getHotelPageUrlsFailCounter.add(1, {
lang,
error_type: "validation_error",
error: JSON.stringify(validatedHotelPageUrls.error),
error: JSON.stringify(validatedResponse.error),
})
console.error(
"contentstack.hotelPageUrls validation error",
JSON.stringify({
query: { lang },
error: validatedHotelPageUrls.error,
error: validatedResponse.error,
})
)
return []
@@ -262,5 +262,5 @@ export async function getHotelPageUrls(lang: Lang) {
JSON.stringify({ query: { lang } })
)
return validatedHotelPageUrls.data
return validatedResponse.data
}

3
apps/scandic-web/types/trpc/routers/contentstack/destinationCityPage.ts
View File

@@ -2,6 +2,7 @@ import type { z } from "zod"
import type {
blocksSchema,
cityPageCountSchema,
cityPageUrlsSchema,
destinationCityListDataSchema,
destinationCityPageRefsSchema,
@@ -19,6 +20,8 @@ export interface GetDestinationCityListDataResponse
export interface DestinationCityListData
extends z.output<typeof destinationCityListDataSchema> {}
export interface GetCityPageCountData
extends z.input<typeof cityPageCountSchema> {}
export interface GetCityPageUrlsData
extends z.input<typeof cityPageUrlsSchema> {}