From 5ff86fca32240d8a8de7c32cc66860b4eaee1814 Mon Sep 17 00:00:00 2001 From: Michael Zetterberg Date: Tue, 5 Nov 2024 16:24:10 +0100 Subject: [PATCH] feat: local netlify plugin to sync branches A custom local Netlify plugin that handles syncing a source branch to multiple destination branches. --- .../netlify-plugin-branch-sync/README.md | 82 ++++++++ .../netlify-plugin-branch-sync/index.js | 179 ++++++++++++++++++ .../netlify-plugin-branch-sync/manifest.yml | 1 + .../netlify-plugin-branch-sync/package.json | 11 ++ apps/scandic-web/netlify.toml | 3 + 5 files changed, 276 insertions(+) create mode 100644 apps/scandic-web/netlify-plugin-branch-sync/README.md create mode 100644 apps/scandic-web/netlify-plugin-branch-sync/index.js create mode 100644 apps/scandic-web/netlify-plugin-branch-sync/manifest.yml create mode 100644 apps/scandic-web/netlify-plugin-branch-sync/package.json diff --git a/apps/scandic-web/netlify-plugin-branch-sync/README.md b/apps/scandic-web/netlify-plugin-branch-sync/README.md new file mode 100644 index 000000000..d506d6d24 --- /dev/null +++ b/apps/scandic-web/netlify-plugin-branch-sync/README.md @@ -0,0 +1,82 @@ +# netlify-plugin-branch-sync + +This is a Netlify build plugin. + +Its purpose is to sync a configured source branch to other configured destination branches. + +The configuration is done inside of index.js instead of using environment variables so that the branch configurations are version controlled. + +`SYNC_SOURCE` is the source branch that will be pushed to the destination branches. + +`SYNC_DEST` is an array of destination branches that will receive the push from `SYNC_SOURCE`. + +This plugin only executes its sync operations when the branch being built in Netlify is the `SYNC_SOURCE` branch and the build context is 'Branch build'. Any other branch or build context is ignored and this plugin does nothing. + +## How to setup + +In Bitbucket, create an access token for the repository this plugin will act on, e.g. https://bitbucket.org/scandic-swap/web/admin/access-tokens. + +It needs to have the following permissions: + +- `repository` +- `repository:write` + +Add the following environment variables in the Netlify dashboard. + +> **IMPORTANT!** +> +> Be sure to set them to only be available in the "Builds" scope and only set a value for the "Branch deploys" configuration. + +- **BITBUCKET_USER_EMAIL**: The email for the access token (Bitbucket generates this and provides it when creating an access token) +- **BITBUCKET_ACCESS_TOKEN**: The access token + +## How it works + +It uses two build plugin events: + +- `onPreBuild` +- `onPostBuild` + +It fails the whole build if the plugin does not complete the sync to all the branches. + +Since it uses `git` directly and we want to sync to latest always, the plugin can be rerun in case of failures and `git` will resume where needed. + +### Event: onPreBuild + +- Checks that the current branch being built is `SYNC_SOURCE` and the build context is a branch build, otherwise does nothing. +- Checks for the presence of the required environment variables. +- Uses `git` to configure the `user.email` and set it to environment variable `BITBUCKET_USER_EMAIL` +- Attempts to restore the clone from the build cache and if that fails proceeds to clone it. +- Uses `git` to fetches the configured branches, `SYNC_SOURCE` and `SYNC_DEST` from origin, skipping all tags. +- Loops over the branches and uses `git` to sequentially push the latest ref on `SYNC_SOURCE` to each `SYNC_DEST` branch respectively. Since a single push completes fast (under 200 ms), we opt to not push in parallel to try and avoid any rate limits. +- Reports to Netlify the completion and provides some info to the "Deploy summary" for the build. +- If anything throws an error during execution the **whole build is failed** + +### Event: onPostBuild + +- Checks that the current branch being built is `SYNC_SOURCE` and the build context is a branch build, otherwise does nothing. +- Attempts to save the clone to the build cache. If it fails by throwing an error the **whole build is failed**. If it fails without throwing an error the build will succeed. + +## Under the hood + +### Bare repository + +Since this plugin only does git operations that do not need a working copy, it clones the repository it uses for the operations as a bare repository. + +The layout of a bare repository differs from a working copy. The way git identifies a working copy with through the presence of a folder named `.git`. Bare repositories do not have this folder. Instead git uses different heuristics to determine if the directory is a clone, one of which is the presence of a folder named `refs`. + +The folder named `refs` only contains two folders after a bare clone operation, `heads` and `tags`. These two folder are both empty themselves. + +This fact poses a challenge for the build cache because it only caches files and not empty directories. + +#### Build cache usage + +This plugin uses the [utilities for caching files in Netlify Build](https://github.com/netlify/build/blob/main/packages/cache-utils/README.md). + +This utility uses [cpy](https://www.npmjs.com/package/cpy) to **copy** files to and from the cache and uses [move-file](https://www.npmjs.com/package/move-file) to **move** files to and from the cache. + +The use of `cpy` is what causes the build cache to not include empty directories because `cpy` only copies files and skips empty directories. + +So to work around the problem this plugin uses the `move mode` of the build cache API which instead uses `move-file` instead of `cpy`, keeping the full directory layout as git requires it. + +The `move mode` is required for both the `restore()` and `save()` build cache operations for the workaround to work. It is done by passing `move: true` as options to the calls. diff --git a/apps/scandic-web/netlify-plugin-branch-sync/index.js b/apps/scandic-web/netlify-plugin-branch-sync/index.js new file mode 100644 index 000000000..1132f81f1 --- /dev/null +++ b/apps/scandic-web/netlify-plugin-branch-sync/index.js @@ -0,0 +1,179 @@ +// @ts-check + +import { existsSync } from "node:fs" + +const SYNC_SOURCE = "master" + +// SYNC_DEST is defined in code and not in an environment variable to +// have this config version controlled. +const SYNC_DEST = [ + "test", + // "stage", + // "prod" +] + +const CLONE_DIR = `${process.env.HOME}/branch-sync-clone` + +function error(msg) { + throw new Error(`[branch-sync] ${msg}`) +} + +function createLogger() { + return { + debug(msg) { + console.log(`[branch-sync] ${msg}`) + }, + info(msg) { + // info logs are grey + console.log("\x1b[90m%s\x1b[0m", `[branch-sync] ${msg}`) + }, + } +} + +export const onPreBuild = async function ({ utils }) { + // Only run for branch builds of source branch. + if ( + process.env.BRANCH === SYNC_SOURCE && + process.env.CONTEXT === "branch-deploy" + ) { + try { + const logger = createLogger() + + if (!process.env.BITBUCKET_USER_EMAIL) { + error( + `Missing commit user email, set env var 'BITBUCKET_USER_EMAIL'. See README.` + ) + } + + if (!process.env.BITBUCKET_ACCESS_TOKEN) { + error( + `Missing access token for Bitbucket, set env var 'BITBUCKET_ACCESS_TOKEN'. See README.` + ) + } + + const { run } = utils + + await run("git", [ + "config", + "user.email", + process.env.BITBUCKET_USER_EMAIL, + ]) + + logger.debug( + `Git user configured with email ${process.env.BITBUCKET_USER_EMAIL}` + ) + + await run("git", ["config", "user.name", "Netlify Branch Sync Bot"]) + + logger.debug(`Git user configured with name 'Netlify Branch Sync Bot'`) + + logger.debug(`Clone directory: ${CLONE_DIR}`) + + if ( + await utils.cache.restore(CLONE_DIR, { + move: true, + }) + ) { + logger.debug(`Restored cached for ${CLONE_DIR}`) + } else { + logger.debug(`Nothing to restore from cache for ${CLONE_DIR}`) + } + + if (existsSync(CLONE_DIR)) { + try { + await run("git", [ + "-C", + CLONE_DIR, + "rev-parse", + "--is-bare-repository", + ]) + logger.debug( + `Verified cached clone directory is a valid bare repository.` + ) + } catch (e) { + logger.info( + `Cached clone directory is corrupted or invalid. Removing and recloning. Error: ${e.message}` + ) + await run("rm", ["-rf", CLONE_DIR]) // Remove corrupted cache + } + } + + if (existsSync(CLONE_DIR)) { + // Fetch if already cloned. + logger.debug(`Fetching from origin`) + + await run("git", [ + "-C", + CLONE_DIR, + "fetch", + "--no-tags", + "origin", + SYNC_SOURCE, + ]) + } else { + // Clone if there is no clone. + const token = process.env.BITBUCKET_ACCESS_TOKEN ?? "" + const cloneURL = `https://x-token-auth:${token}@bitbucket.org/scandic-swap/web.git` + + logger.debug(`Cloning from ${cloneURL.replace(token, "****")}`) + logger.debug(`Cloning to ${CLONE_DIR}`) + + await run("git", [ + "clone", + "--bare", + "--branch", + SYNC_SOURCE, + cloneURL, + CLONE_DIR, + ]) + } + + logger.debug(`Attempting to sync: ${SYNC_DEST.join(", ")}`) + + for (let i = 0; i < SYNC_DEST.length; ++i) { + const branch = SYNC_DEST[i] + await run("git", ["-C", CLONE_DIR, "push", "origin", `HEAD:${branch}`]) + console.log(`Successfully synced '${branch}' with '${SYNC_SOURCE}'`) + } + + utils.status.show({ + title: "Branch sync", + summary: "All branches are synced! ✅", + text: [ + `The following branches have been synced with '${SYNC_SOURCE}'@${process.env.COMMIT_REF}:`, + "", + ] + .concat( + SYNC_DEST.map((branch) => { + return `- ${branch} - [Deployment list](https://app.netlify.com/sites/web-scandic-hotels/deploys?filter=${branch})` + }) + ) + .join("\n"), + }) + } catch (error) { + utils.build.failBuild("Failed to sync branches.", { error }) + } + } +} + +export const onPostBuild = async function ({ utils }) { + // Only run for branch builds of source branch. + if ( + process.env.BRANCH === SYNC_SOURCE && + process.env.CONTEXT === "branch-deploy" + ) { + try { + const logger = createLogger() + + if ( + await utils.cache.save(CLONE_DIR, { + move: true, + }) + ) { + logger.debug(`Saved cached for ${CLONE_DIR}`) + } + } catch (error) { + utils.build.failBuild("Failed to sync branches.", { error }) + } + } +} diff --git a/apps/scandic-web/netlify-plugin-branch-sync/manifest.yml b/apps/scandic-web/netlify-plugin-branch-sync/manifest.yml new file mode 100644 index 000000000..54e10a0b5 --- /dev/null +++ b/apps/scandic-web/netlify-plugin-branch-sync/manifest.yml @@ -0,0 +1 @@ +name: netlify-plugin-branch-sync diff --git a/apps/scandic-web/netlify-plugin-branch-sync/package.json b/apps/scandic-web/netlify-plugin-branch-sync/package.json new file mode 100644 index 000000000..c94c3bbc3 --- /dev/null +++ b/apps/scandic-web/netlify-plugin-branch-sync/package.json @@ -0,0 +1,11 @@ +{ + "name": "netlify-plugin-branch-sync", + "version": "1.0.0", + "type": "module", + "description": "Sync branches", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "license": "MIT" +} diff --git a/apps/scandic-web/netlify.toml b/apps/scandic-web/netlify.toml index 7bd58da97..727bbfa00 100644 --- a/apps/scandic-web/netlify.toml +++ b/apps/scandic-web/netlify.toml @@ -28,6 +28,9 @@ TERM = "xterm" [[plugins]] package = "@netlify/plugin-nextjs" +[[plugins]] +package = "./apps/scandic-web/netlify-plugin-branch-sync" + [images] remote_images = [ "https://imagevault-stage.scandichotels.com.*",