From 9ec1da1ad185a226280212a62a42319965a62ffc Mon Sep 17 00:00:00 2001 From: Lucas Duailibe Date: Mon, 18 Jun 2018 15:16:40 -0300 Subject: [PATCH] [internal] Cache build results (#4693) --- .circleci/config.yml | 14 +++ .gitignore | 1 + package.json | 1 + scripts/build/build.js | 71 +++++++++--- scripts/build/bundler.js | 21 +++- scripts/build/cache.js | 129 ++++++++++++++++++++++ scripts/build/util.js | 2 - scripts/release/steps/generate-bundles.js | 2 +- scripts/release/utils.js | 5 +- yarn.lock | 30 ++++- 10 files changed, 254 insertions(+), 22 deletions(-) create mode 100644 scripts/build/cache.js diff --git a/.circleci/config.yml b/.circleci/config.yml index f3b49092..74d29f76 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -22,6 +22,18 @@ aliases: - node_modules key: v1-yarn-deps-{{ checksum "yarn.lock" }} + - &restore_build_cache + restore_cache: + keys: + - v1-build-cache-{{ .Branch }} + - v1-build-cache-master + + - &save_build_cache + save_cache: + paths: + - .cache + key: v1-build-cache-{{ .Branch }} + # Default - &defaults working_directory: ~/prettier @@ -54,7 +66,9 @@ jobs: steps: - attach_workspace: at: ~/prettier + - *restore_build_cache - run: yarn build + - *save_build_cache - persist_to_workspace: root: . paths: diff --git a/.gitignore b/.gitignore index aaa9b1e3..e2a52608 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +/.cache /node_modules /scripts/release/node_modules *.log diff --git a/package.json b/package.json index 797a0b80..47d40a21 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "eslint-plugin-import": "2.9.0", "eslint-plugin-prettier": "2.6.0", "eslint-plugin-react": "7.7.0", + "execa": "0.10.0", "jest": "21.1.0", "mkdirp": "0.5.1", "prettier": "1.13.5", diff --git a/scripts/build/build.js b/scripts/build/build.js index 82f1fee6..7ec597cd 100644 --- a/scripts/build/build.js +++ b/scripts/build/build.js @@ -1,10 +1,15 @@ "use strict"; const chalk = require("chalk"); +const execa = require("execa"); +const minimist = require("minimist"); +const path = require("path"); const stringWidth = require("string-width"); + const bundler = require("./bundler"); const bundleConfigs = require("./config"); const util = require("./util"); +const Cache = require("./cache"); // Errors in promises should be fatal. const loggedErrors = new Set(); @@ -16,8 +21,9 @@ process.on("unhandledRejection", err => { process.exit(1); }); -const OK = chalk.reset.inverse.bold.green(" DONE "); -const FAIL = chalk.reset.inverse.bold.red(" FAIL "); +const CACHED = chalk.bgYellow.black(" CACHED "); +const OK = chalk.bgGreen.black(" DONE "); +const FAIL = chalk.bgRed.black(" FAIL "); function fitTerminal(input) { const columns = Math.min(process.stdout.columns || 40, 80); @@ -28,18 +34,22 @@ function fitTerminal(input) { return input; } -async function createBundle(bundleConfig) { +async function createBundle(bundleConfig, cache) { const { output } = bundleConfig; process.stdout.write(fitTerminal(output)); - try { - await bundler(bundleConfig, output); - } catch (error) { - process.stdout.write(`${FAIL}\n\n`); - handleError(error); - } - - process.stdout.write(`${OK}\n`); + return bundler(bundleConfig, cache) + .catch(error => { + console.log(FAIL + "\n"); + handleError(error); + }) + .then(result => { + if (result.cached) { + console.log(CACHED); + } else { + console.log(OK); + } + }); } function handleError(error) { @@ -48,6 +58,22 @@ function handleError(error) { throw error; } +async function cacheFiles() { + // Copy built files to .cache + try { + await execa("rm", ["-rf", path.join(".cache", "files")]); + await execa("mkdir", ["-p", path.join(".cache", "files")]); + for (const bundleConfig of bundleConfigs) { + await execa("cp", [ + path.join("dist", bundleConfig.output), + path.join(".cache", "files") + ]); + } + } catch (err) { + // Don't fail the build + } +} + async function preparePackage() { const pkg = await util.readJson("package.json"); pkg.bin = "./bin-prettier.js"; @@ -64,15 +90,30 @@ async function preparePackage() { await util.copyFile("./README.md", "./dist/README.md"); } -async function run() { - await util.asyncRimRaf("dist"); +async function run(params) { + await execa("rm", ["-rf", "dist"]); + await execa("mkdir", ["-p", "dist"]); + + if (params["purge-cache"]) { + await execa("rm", ["-rf", ".cache"]); + } + + const bundleCache = new Cache(".cache/", "v1"); + await bundleCache.load(); console.log(chalk.inverse(" Building packages ")); for (const bundleConfig of bundleConfigs) { - await createBundle(bundleConfig); + await createBundle(bundleConfig, bundleCache); } + await bundleCache.save(); + await cacheFiles(); + await preparePackage(); } -run(); +run( + minimist(process.argv.slice(2), { + boolean: ["purge-cache"] + }) +); diff --git a/scripts/build/bundler.js b/scripts/build/bundler.js index 5a60f84c..ea76580f 100644 --- a/scripts/build/bundler.js +++ b/scripts/build/bundler.js @@ -1,5 +1,6 @@ "use strict"; +const execa = require("execa"); const path = require("path"); const { rollup } = require("rollup"); const webpack = require("webpack"); @@ -186,11 +187,29 @@ function runWebpack(config) { }); } -module.exports = async function createBundle(bundle) { +module.exports = async function createBundle(bundle, cache) { + const useCache = await cache.checkBundle( + bundle.output, + getRollupConfig(bundle) + ); + if (useCache) { + try { + await execa("cp", [ + path.join(cache.cacheDir, "files", bundle.output), + "dist" + ]); + return { cached: true }; + } catch (err) { + // Proceed to build + } + } + if (bundle.bundler === "webpack") { await runWebpack(getWebpackConfig(bundle)); } else { const result = await rollup(getRollupConfig(bundle)); await result.write(getRollupOutputOptions(bundle)); } + + return { bundled: true }; }; diff --git a/scripts/build/cache.js b/scripts/build/cache.js new file mode 100644 index 00000000..f3a51b4c --- /dev/null +++ b/scripts/build/cache.js @@ -0,0 +1,129 @@ +"use strict"; + +const util = require("util"); +const assert = require("assert"); +const crypto = require("crypto"); +const fs = require("fs"); +const path = require("path"); +const { rollup } = require("rollup"); + +const readFile = util.promisify(fs.readFile); +const writeFile = util.promisify(fs.writeFile); + +const ROOT = path.join(__dirname, "..", ".."); + +function Cache(cacheDir, version) { + this.cacheDir = path.resolve(cacheDir || required("cacheDir")); + this.manifest = path.join(this.cacheDir, "manifest.json"); + this.version = version || required("version"); + this.checksums = {}; + this.files = {}; + this.updated = { + version: this.version, + checksums: {}, + files: {} + }; +} + +// Loads the manifest.json file with the information from the last build +Cache.prototype.load = async function() { + // This should never throw, if it does, let it fail the build + const lockfile = await readFile("yarn.lock", "utf-8"); + const lockfileHash = hashString(lockfile); + this.updated.checksums["yarn.lock"] = lockfileHash; + + try { + const manifest = await readFile(this.manifest, "utf-8"); + const { version, checksums, files } = JSON.parse(manifest); + + // Ignore the cache if the version changed + assert.equal(this.version, version); + + assert.ok(typeof checksums === "object"); + // If yarn.lock changed, rebuild everything + assert.equal(lockfileHash, checksums["yarn.lock"]); + this.checksums = checksums; + + assert.ok(typeof files === "object"); + this.files = files; + + for (const files of Object.values(this.files)) { + assert.ok(Array.isArray(files)); + } + } catch (err) { + this.checksums = {}; + this.files = {}; + } +}; + +// Run rollup to get the list of files included in the bundle and check if +// any (or the list itself) have changed. +// This takes the same rollup config used for bundling to include files that are +// resolved by specific plugins. +Cache.prototype.checkBundle = async function(output, rollupConfig) { + const result = await rollup(getRollupConfig(rollupConfig)); + const modules = result.modules + .filter(mod => !/\0/.test(mod.id)) + .map(mod => [path.relative(ROOT, mod.id), mod.originalCode]); + + const files = new Set(this.files[output]); + const newFiles = (this.updated.files[output] = []); + + let dirty = false; + + for (const [id, code] of modules) { + newFiles.push(id); + // If we already checked this file for another bundle, reuse the hash + if (!this.updated.checksums[id]) { + this.updated.checksums[id] = hashString(code); + } + const hash = this.updated.checksums[id]; + + // Check if this file changed + if (!this.checksums[id] || this.checksums[id] !== hash) { + dirty = true; + } + + // Check if this file is new + if (!files.delete(id)) { + dirty = true; + } + } + + // Final check: if any file was removed, `files` is not empty + return !dirty && files.size === 0; +}; + +Cache.prototype.save = async function() { + try { + await writeFile(this.manifest, JSON.stringify(this.updated, null, 2)); + } catch (err) { + // Don't fail the build + } +}; + +function required(name) { + throw new Error(name + " is required"); +} + +function hashString(string) { + return crypto + .createHash("md5") + .update(string) + .digest("hex"); +} + +function getRollupConfig(rollupConfig) { + return Object.assign({}, rollupConfig, { + onwarn() {}, + plugins: rollupConfig.plugins.filter( + plugin => + // We're not interested in dependencies, we already check `yarn.lock` + plugin.name !== "node-resolve" && + // This is really slow, we need this "preflight" to be fast + plugin.name !== "babel" + ) + }); +} + +module.exports = Cache; diff --git a/scripts/build/util.js b/scripts/build/util.js index 1275b84f..9a9af587 100644 --- a/scripts/build/util.js +++ b/scripts/build/util.js @@ -1,7 +1,6 @@ "use strict"; const fs = require("fs"); -const rimraf = require("rimraf"); const promisify = require("util").promisify; const readFile = promisify(fs.readFile); @@ -23,7 +22,6 @@ async function copyFile(from, to) { } module.exports = { - asyncRimRaf: promisify(rimraf), readJson, writeJson, copyFile, diff --git a/scripts/release/steps/generate-bundles.js b/scripts/release/steps/generate-bundles.js index 87b6534a..2b96073b 100644 --- a/scripts/release/steps/generate-bundles.js +++ b/scripts/release/steps/generate-bundles.js @@ -4,7 +4,7 @@ const chalk = require("chalk"); const { runYarn, logPromise, readJson } = require("../utils"); module.exports = async function({ version }) { - await logPromise("Generating bundles", runYarn("build")); + await logPromise("Generating bundles", runYarn(["build", "--purge-cache"])); const builtPkg = await readJson("dist/package.json"); if (builtPkg.version !== version) { diff --git a/scripts/release/utils.js b/scripts/release/utils.js index e6a4ea89..5b09eea7 100644 --- a/scripts/release/utils.js +++ b/scripts/release/utils.js @@ -34,7 +34,10 @@ function logPromise(name, promise) { } function runYarn(script) { - return execa("yarn", ["--silent", script]).catch(error => { + if (typeof script === "string") { + script = [script]; + } + return execa("yarn", ["--silent"].concat(script)).catch(error => { throw Error(`\`yarn ${script}\` failed\n${error.stdout}`); }); } diff --git a/yarn.lock b/yarn.lock index 1a616cab..3cd2c4a0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1651,6 +1651,16 @@ cross-spawn@^5.0.1, cross-spawn@^5.1.0: shebang-command "^1.2.0" which "^1.2.9" +cross-spawn@^6.0.0: + version "6.0.5" + resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-6.0.5.tgz#4a5ec7c64dfae22c3a14124dbacdee846d80cbc4" + dependencies: + nice-try "^1.0.4" + path-key "^2.0.1" + semver "^5.5.0" + shebang-command "^1.2.0" + which "^1.2.9" + cryptiles@2.x.x: version "2.0.5" resolved "https://registry.yarnpkg.com/cryptiles/-/cryptiles-2.0.5.tgz#3bdfecdc608147c1c67202fa291e7dca59eaa3b8" @@ -2239,6 +2249,18 @@ exec-sh@^0.2.0: dependencies: merge "^1.1.3" +execa@0.10.0: + version "0.10.0" + resolved "https://registry.yarnpkg.com/execa/-/execa-0.10.0.tgz#ff456a8f53f90f8eccc71a96d11bdfc7f082cb50" + dependencies: + cross-spawn "^6.0.0" + get-stream "^3.0.0" + is-stream "^1.1.0" + npm-run-path "^2.0.0" + p-finally "^1.0.0" + signal-exit "^3.0.0" + strip-eof "^1.0.0" + execa@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/execa/-/execa-0.7.0.tgz#944becd34cc41ee32a63a9faf27ad5a65fc59777" @@ -4065,6 +4087,10 @@ next-tick@1: version "1.0.0" resolved "https://registry.yarnpkg.com/next-tick/-/next-tick-1.0.0.tgz#ca86d1fe8828169b0120208e3dc8424b9db8342c" +nice-try@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/nice-try/-/nice-try-1.0.4.tgz#d93962f6c52f2c1558c0fbda6d512819f1efe1c4" + node-fetch@^1.0.1: version "1.7.3" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-1.7.3.tgz#980f6f72d85211a5347c6b2bc18c5b84c3eb47ef" @@ -4422,7 +4448,7 @@ path-is-inside@^1.0.1, path-is-inside@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/path-is-inside/-/path-is-inside-1.0.2.tgz#365417dede44430d1c11af61027facf074bdfc53" -path-key@^2.0.0: +path-key@^2.0.0, path-key@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/path-key/-/path-key-2.0.1.tgz#411cadb574c5a140d3a4b1910d40d80cc9f40b40" @@ -5146,7 +5172,7 @@ sax@^1.2.4: version "5.4.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.4.1.tgz#e059c09d8571f0540823733433505d3a2f00b18e" -semver@5.5.0, semver@^5.4.1: +semver@5.5.0, semver@^5.4.1, semver@^5.5.0: version "5.5.0" resolved "https://registry.yarnpkg.com/semver/-/semver-5.5.0.tgz#dc4bbc7a6ca9d916dee5d43516f0092b58f7b8ab"