feat(SW-706): add Lokalise tooling and codemod

apps/scandic-web/i18n/tooling/download.ts

@@ -0,0 +1,11 @@
+import path from "node:path"
+
+import { download } from "./lokalise"
+
+const extractPath = path.resolve(__dirname, "translations")
+
+async function main() {
+  await download(extractPath)
+}
+
+main()

apps/scandic-web/i18n/tooling/formatter.mjs

@@ -0,0 +1,10 @@
+// Run the formatter.ts through Jiti
+
+import { fileURLToPath } from "node:url"
+
+import createJiti from "jiti"
+
+const formatter = createJiti(fileURLToPath(import.meta.url))("./formatter.ts")
+
+export const format = formatter.format
+export const compile = formatter.compile

apps/scandic-web/i18n/tooling/formatter.ts

@@ -0,0 +1,99 @@
+// https://docs.lokalise.com/en/articles/3229161-structured-json
+
+import type { LokaliseMessageDescriptor } from "@/types/intl"
+
+type TranslationEntry = {
+  translation: string
+  notes?: string
+  context?: string
+  limit?: number
+  tags?: string[]
+}
+
+type CompiledEntries = Record<string, string>
+
+type LokaliseStructuredJson = Record<string, TranslationEntry>
+
+export function format(
+  msgs: LokaliseMessageDescriptor[]
+): LokaliseStructuredJson {
+  const results: LokaliseStructuredJson = {}
+  for (const [id, msg] of Object.entries(msgs)) {
+    const { defaultMessage, description } = msg
+
+    if (typeof defaultMessage === "string") {
+      const entry: TranslationEntry = {
+        translation: defaultMessage,
+      }
+
+      if (description) {
+        if (typeof description === "string") {
+          console.warn(
+            `Unsupported type for description, expected 'object', got ${typeof context}. Skipping!`,
+            msg
+          )
+        } else {
+          const { context, limit, tags } = description
+
+          if (context) {
+            if (typeof context === "string") {
+              entry.context = context
+            } else {
+              console.warn(
+                `Unsupported type for context, expected 'string', got ${typeof context}`,
+                msg
+              )
+            }
+          }
+
+          if (limit) {
+            if (limit && typeof limit === "number") {
+              entry.limit = limit
+            } else {
+              console.warn(
+                `Unsupported type for limit, expected 'number', got ${typeof limit}`,
+                msg
+              )
+            }
+          }
+
+          if (tags) {
+            if (tags && typeof tags === "string") {
+              const tagArray = tags.split(",").map((s) => s.trim())
+              if (tagArray.length) {
+                entry.tags = tagArray
+              }
+            } else {
+              console.warn(
+                `Unsupported type for tags, expected Array, got ${typeof tags}`,
+                msg
+              )
+            }
+          }
+        }
+      }
+
+      results[id] = entry
+    } else {
+      console.warn(
+        `Skipping message, unsupported type for defaultMessage, expected string, got ${typeof defaultMessage}`,
+        {
+          id,
+          msg,
+        }
+      )
+    }
+  }
+
+  return results
+}
+
+export function compile(msgs: LokaliseStructuredJson): CompiledEntries {
+  const results: CompiledEntries = {}
+
+  for (const [id, msg] of Object.entries(msgs)) {
+    results[id] = msg.translation
+  }
+
+  return results
+}

apps/scandic-web/i18n/tooling/lokalise.ts

@@ -0,0 +1,210 @@
+import fs from "node:fs/promises"
+import { performance, PerformanceObserver } from "node:perf_hooks"
+
+import { LokaliseApi } from "@lokalise/node-api"
+import AdmZip from "adm-zip"
+
+const projectId = "4194150766ff28c418f010.39532200"
+const lokaliseApi = new LokaliseApi({ apiKey: process.env.LOKALISE_API_KEY })
+
+function log(msg: string, ...args: any[]) {
+  console.log(`[lokalise] ${msg}`, ...args)
+}
+
+function error(msg: string, ...args: any[]) {
+  console.error(`[lokalise] ${msg}`, ...args)
+}
+
+let resolvePerf: (value?: unknown) => void
+const performanceMetrics = new Promise((resolve) => {
+  resolvePerf = resolve
+})
+
+const perf = new PerformanceObserver((items) => {
+  const entries = items.getEntries()
+  for (const entry of entries) {
+    if (entry.name === "done") {
+      // This is the last measure meant for clean up
+      performance.clearMarks()
+      perf.disconnect()
+      if (typeof resolvePerf === "function") {
+        resolvePerf()
+      }
+    } else {
+      log(`[metrics] ${entry.name} completed in ${entry.duration} ms`)
+    }
+  }
+  performance.clearMeasures()
+})
+perf.observe({ type: "measure" })
+
+async function waitUntilUploadDone(processId: string) {
+  return new Promise<void>((resolve, reject) => {
+    const interval = setInterval(async () => {
+      try {
+        performance.mark("waitUntilUploadDoneStart")
+
+        log("Checking upload status...")
+
+        performance.mark("getProcessStart")
+        const process = await lokaliseApi.queuedProcesses().get(processId, {
+          project_id: projectId,
+        })
+        performance.mark("getProcessEnd")
+        performance.measure(
+          "Get Queued Process",
+          "getProcessStart",
+          "getProcessEnd"
+        )
+
+        log(`Status: ${process.status}`)
+
+        if (process.status === "finished") {
+          clearInterval(interval)
+          performance.mark("waitUntilUploadDoneEnd", { detail: "success" })
+          performance.measure(
+            "Wait on upload",
+            "waitUntilUploadDoneStart",
+            "waitUntilUploadDoneEnd"
+          )
+          resolve()
+        } else if (process.status === "failed") {
+          throw process
+        }
+      } catch (e) {
+        clearInterval(interval)
+        error("An error occurred:", e)
+        performance.mark("waitUntilUploadDoneEnd", { detail: error })
+        performance.measure(
+          "Wait on upload",
+          "waitUntilUploadDoneStart",
+          "waitUntilUploadDoneEnd"
+        )
+        reject()
+      }
+    }, 1000)
+  })
+}
+
+export async function upload(filepath: string) {
+  try {
+    log(`Uploading ${filepath}...`)
+
+    performance.mark("uploadStart")
+
+    performance.mark("sourceFileReadStart")
+    const data = await fs.readFile(filepath, "utf8")
+    const buff = Buffer.from(data, "utf8")
+    const base64 = buff.toString("base64")
+    performance.mark("sourceFileReadEnd")
+    performance.measure(
+      "Read source file",
+      "sourceFileReadStart",
+      "sourceFileReadEnd"
+    )
+
+    performance.mark("lokaliseUploadInitStart")
+    const bgProcess = await lokaliseApi.files().upload(projectId, {
+      data: base64,
+      filename: "en.json",
+      lang_iso: "en",
+      detect_icu_plurals: true,
+      format: "json",
+      convert_placeholders: true,
+      replace_modified: true,
+    })
+    performance.mark("lokaliseUploadInitEnd")
+    performance.measure(
+      "Upload init",
+      "lokaliseUploadInitStart",
+      "lokaliseUploadInitEnd"
+    )
+
+    performance.mark("lokaliseUploadStart")
+    await waitUntilUploadDone(bgProcess.process_id)
+    performance.mark("lokaliseUploadEnd")
+    performance.measure(
+      "Upload transfer",
+      "lokaliseUploadStart",
+      "lokaliseUploadEnd"
+    )
+
+    log("Upload successful")
+  } catch (e) {
+    error("Upload failed", e)
+  } finally {
+    performance.mark("uploadEnd")
+
+    performance.measure("Upload operation", "uploadStart", "uploadEnd")
+  }
+
+  performance.measure("done")
+
+  await performanceMetrics
+}
+
+export async function download(extractPath: string) {
+  try {
+    log("Downloading translations...")
+
+    performance.mark("downloadStart")
+
+    performance.mark("lokaliseDownloadInitStart")
+    const downloadResponse = await lokaliseApi.files().download(projectId, {
+      format: "json_structured",
+      indentation: "2sp",
+      placeholder_format: "icu",
+      plural_format: "icu",
+      icu_numeric: true,
+      bundle_structure: "%LANG_ISO%.%FORMAT%",
+      directory_prefix: "",
+      filter_data: ["last_reviewed_only"],
+      export_empty_as: "skip",
+    })
+    performance.mark("lokaliseDownloadInitEnd")
+    performance.measure(
+      "Download init",
+      "lokaliseDownloadInitStart",
+      "lokaliseDownloadInitEnd"
+    )
+
+    const { bundle_url } = downloadResponse
+
+    performance.mark("lokaliseDownloadStart")
+    const bundleResponse = await fetch(bundle_url)
+    performance.mark("lokaliseDownloadEnd")
+    performance.measure(
+      "Download transfer",
+      "lokaliseDownloadStart",
+      "lokaliseDownloadEnd"
+    )
+
+    if (bundleResponse.ok) {
+      performance.mark("unpackTranslationsStart")
+      const arrayBuffer = await bundleResponse.arrayBuffer()
+      const buffer = Buffer.from(new Uint8Array(arrayBuffer))
+      const zip = new AdmZip(buffer)
+      zip.extractAllTo(extractPath, true)
+      performance.mark("unpackTranslationsEnd")
+      performance.measure(
+        "Unpacking translations",
+        "unpackTranslationsStart",
+        "unpackTranslationsEnd"
+      )
+
+      log("Download successful")
+    } else {
+      throw bundleResponse
+    }
+  } catch (e) {
+    error("Download failed", e)
+  } finally {
+    performance.mark("downloadEnd")
+
+    performance.measure("Download operation", "downloadStart", "downloadEnd")
+  }
+
+  performance.measure("done")
+
+  await performanceMetrics
+}

apps/scandic-web/i18n/tooling/upload.ts

@@ -0,0 +1,11 @@
+import path from "node:path"
+
+import { upload } from "./lokalise"
+
+const filepath = path.resolve(__dirname, "./extracted.json")
+
+async function main() {
+  await upload(filepath)
+}
+
+main()