build: fetch Node.js sources from tar.gz archive instead of Git

Bug: #204, #212
This commit is contained in:
Jesse Chan 2021-11-06 15:20:32 -07:00
parent 45300a7b6c
commit da210e4c6f
6 changed files with 179 additions and 35 deletions

View File

@ -7,7 +7,7 @@ USER root:root
WORKDIR /root/pkg-fetch/
RUN apk add --no-cache build-base git linux-headers npm python2 python3 yarn
RUN apk add --no-cache build-base linux-headers npm python2 python3 yarn
# https://gitlab.alpinelinux.org/alpine/aports/-/issues/8626
ENV CFLAGS=-U_FORTIFY_SOURCE

View File

@ -10,7 +10,7 @@ RUN yum upgrade -y
RUN yum install -y \
devtoolset-10 glibc-headers kernel-headers \
git make patch python2 \
make patch python2 \
rh-python36-python
RUN curl -fsSL https://rpm.nodesource.com/setup_lts.x | bash -

View File

@ -1,11 +1,16 @@
import { createGunzip } from 'zlib';
import crypto from 'crypto';
import fs from 'fs-extra';
import os from 'os';
import path from 'path';
import { pipeline } from 'stream';
import { promisify } from 'util';
import tar from 'tar-fs';
import { hash, spawn } from './utils';
import { cachePath } from './places';
import { downloadUrl, hash, spawn } from './utils';
import { hostArch, hostPlatform } from './system';
import { log } from './log';
import { log, wasReported } from './log';
import patchesJson from '../patches/patches.json';
const buildPath = path.resolve(
@ -14,7 +19,9 @@ const buildPath = path.resolve(
);
const nodePath = path.join(buildPath, 'node');
const patchesPath = path.resolve(__dirname, '../patches');
const nodeRepo = 'https://github.com/nodejs/node';
const nodeRepo = 'https://nodejs.org/dist';
const nodeArchivePath = path.join(cachePath, 'node');
function getMajor(nodeVersion: string) {
const [, version] = nodeVersion.match(/^v?(\d+)/) || ['', 0];
@ -66,37 +73,66 @@ function getConfigureArgs(major: number, targetPlatform: string): string[] {
return args;
}
async function gitClone(nodeVersion: string) {
log.info('Cloning Node.js repository from GitHub...');
async function tarFetch(nodeVersion: string) {
log.info('Fetching Node.js source archive from nodejs.org...');
const args = [
'clone',
'-b',
nodeVersion,
'--depth',
'1',
'--single-branch',
'--bare',
'--progress',
nodeRepo,
'node/.git',
];
const distUrl = `${nodeRepo}/${nodeVersion}`;
const tarName = `node-${nodeVersion}.tar.gz`;
await spawn('git', args, { cwd: buildPath, stdio: 'inherit' });
const archivePath = path.join(nodeArchivePath, tarName);
const hashPath = path.join(nodeArchivePath, `${tarName}.sha256sum`);
if (fs.existsSync(hashPath) && fs.existsSync(archivePath)) {
return;
}
await fs.remove(hashPath).catch(() => undefined);
await fs.remove(archivePath).catch(() => undefined);
await downloadUrl(`${distUrl}/SHASUMS256.txt`, hashPath);
await fs.writeFile(
hashPath,
(await fs.readFile(hashPath, 'utf8'))
.split('\n')
.filter((l) => l.includes(tarName))[0]
);
await downloadUrl(`${distUrl}/${tarName}`, archivePath);
}
async function gitResetHard(nodeVersion: string) {
log.info(`Checking out ${nodeVersion}`);
async function tarExtract(nodeVersion: string) {
log.info('Extracting Node.js source archive...');
const patches = patchesJson[nodeVersion as keyof typeof patchesJson] as
| string[]
| { commit?: string };
const tarName = `node-${nodeVersion}.tar.gz`;
const commit =
'commit' in patches && patches.commit ? patches.commit : nodeVersion;
const args = ['--work-tree', '.', 'reset', '--hard', commit];
const expectedHash = (
await fs.readFile(
path.join(nodeArchivePath, `${tarName}.sha256sum`),
'utf8'
)
).split(' ')[0];
const actualHash = await hash(path.join(nodeArchivePath, tarName));
await spawn('git', args, { cwd: nodePath, stdio: 'inherit' });
if (expectedHash !== actualHash) {
await fs.remove(path.join(nodeArchivePath, tarName));
await fs.remove(path.join(nodeArchivePath, `${tarName}.sha256sum`));
throw wasReported(`Hash mismatch for ${tarName}`);
}
const pipe = promisify(pipeline);
const source = fs.createReadStream(path.join(nodeArchivePath, tarName));
const gunzip = createGunzip();
const extract = tar.extract(nodePath, {
strip: 1,
map: (header) => {
log.info(header.name);
return header;
},
});
await pipe(source, gunzip, extract);
}
async function applyPatches(nodeVersion: string) {
@ -249,10 +285,11 @@ export default async function build(
local: string
) {
await fs.remove(buildPath);
await fs.mkdirp(buildPath);
await fs.mkdirp(nodePath);
await fs.mkdirp(nodeArchivePath);
await gitClone(nodeVersion);
await gitResetHard(nodeVersion);
await tarFetch(nodeVersion);
await tarExtract(nodeVersion);
await applyPatches(nodeVersion);
const output = await compile(nodeVersion, targetArch, targetPlatform);

View File

@ -5,7 +5,8 @@ import path from 'path';
const { PKG_CACHE_PATH } = process.env;
const IGNORE_TAG = Boolean(process.env.PKG_IGNORE_TAG);
const cachePath = PKG_CACHE_PATH || path.join(os.homedir(), '.pkg-cache');
export const cachePath =
PKG_CACHE_PATH || path.join(os.homedir(), '.pkg-cache');
function tagFromVersion(version: string) {
const mj = major(version);

View File

@ -20,6 +20,7 @@
"node-fetch": "^2.6.6",
"progress": "^2.0.3",
"semver": "^7.3.5",
"tar-fs": "^2.1.1",
"yargs": "^16.2.0"
},
"devDependencies": {
@ -28,6 +29,7 @@
"@types/node-fetch": "^2.5.12",
"@types/progress": "^2.0.5",
"@types/semver": "^7.3.9",
"@types/tar-fs": "^2.0.1",
"@types/yargs": "^16.0.4",
"@typescript-eslint/eslint-plugin": "^4.33.0",
"@typescript-eslint/parser": "^4.33.0",

108
yarn.lock
View File

@ -132,6 +132,21 @@
resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.9.tgz#152c6c20a7688c30b967ec1841d31ace569863fc"
integrity sha512-L/TMpyURfBkf+o/526Zb6kd/tchUP3iBDEPjqjb+U2MAJhVRxxrmr2fwpe08E7QsV7YLcpq0tUaQ9O9x97ZIxQ==
"@types/tar-fs@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@types/tar-fs/-/tar-fs-2.0.1.tgz#6391dcad1b03dea2d79fac07371585ab54472bb1"
integrity sha512-qlsQyIY9sN7p221xHuXKNoMfUenOcvEBN4zI8dGsYbYCqHtTarXOEXSIgUnK+GcR0fZDse6pAIc5pIrCh9NefQ==
dependencies:
"@types/node" "*"
"@types/tar-stream" "*"
"@types/tar-stream@*":
version "2.2.2"
resolved "https://registry.yarnpkg.com/@types/tar-stream/-/tar-stream-2.2.2.tgz#be9d0be9404166e4b114151f93e8442e6ab6fb1d"
integrity sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ==
dependencies:
"@types/node" "*"
"@types/yargs-parser@*":
version "20.2.0"
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-20.2.0.tgz#dd3e6699ba3237f0348cd085e4698780204842f9"
@ -342,6 +357,20 @@ balanced-match@^1.0.0:
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c=
base64-js@^1.3.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==
bl@^4.0.3:
version "4.1.0"
resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a"
integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==
dependencies:
buffer "^5.5.0"
inherits "^2.0.4"
readable-stream "^3.4.0"
brace-expansion@^1.1.7:
version "1.1.11"
resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
@ -357,6 +386,14 @@ braces@^3.0.1:
dependencies:
fill-range "^7.0.1"
buffer@^5.5.0:
version "5.7.1"
resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0"
integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==
dependencies:
base64-js "^1.3.1"
ieee754 "^1.1.13"
call-bind@^1.0.0, call-bind@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c"
@ -395,6 +432,11 @@ chalk@^4.1.0, chalk@^4.1.2:
ansi-styles "^4.1.0"
supports-color "^7.1.0"
chownr@^1.1.1:
version "1.1.4"
resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b"
integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==
clean-stack@^2.0.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b"
@ -576,7 +618,7 @@ emoji-regex@^8.0.0:
resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
end-of-stream@^1.1.0:
end-of-stream@^1.1.0, end-of-stream@^1.4.1:
version "1.4.4"
resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0"
integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==
@ -943,6 +985,11 @@ form-data@^3.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
fs-constants@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad"
integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==
fs-extra@^9.1.0:
version "9.1.0"
resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-9.1.0.tgz#5954460c764a8da2094ba3554bf839e6b9a7c86d"
@ -1111,6 +1158,11 @@ human-signals@^1.1.1:
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3"
integrity sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==
ieee754@^1.1.13:
version "1.2.1"
resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352"
integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==
ignore@^4.0.6:
version "4.0.6"
resolved "https://registry.yarnpkg.com/ignore/-/ignore-4.0.6.tgz#750e3db5862087b4737ebac8207ffd1ef27b25fc"
@ -1152,7 +1204,7 @@ inflight@^1.0.4:
once "^1.3.0"
wrappy "1"
inherits@2:
inherits@2, inherits@^2.0.3, inherits@^2.0.4:
version "2.0.4"
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==
@ -1510,6 +1562,11 @@ minimist@^1.2.0:
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.5.tgz#67d66014b66a6a8aaa0c083c5fd58df4e4e97602"
integrity sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==
mkdirp-classic@^0.5.2:
version "0.5.3"
resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
ms@2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@ -1739,6 +1796,15 @@ queue-microtask@^1.2.2:
resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243"
integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==
readable-stream@^3.1.1, readable-stream@^3.4.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
dependencies:
inherits "^2.0.3"
string_decoder "^1.1.1"
util-deprecate "^1.0.1"
regexpp@^3.1.0:
version "3.1.0"
resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.1.0.tgz#206d0ad0a5648cffbdb8ae46438f3dc51c9f78e2"
@ -1801,6 +1867,11 @@ rxjs@^7.4.0:
dependencies:
tslib "~2.1.0"
safe-buffer@~5.2.0:
version "5.2.1"
resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
semver-compare@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/semver-compare/-/semver-compare-1.0.0.tgz#0dee216a1c941ab37e9efb1788f6afc5ff5537fc"
@ -1911,6 +1982,13 @@ string.prototype.trimstart@^1.0.4:
call-bind "^1.0.2"
define-properties "^1.1.3"
string_decoder@^1.1.1:
version "1.3.0"
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e"
integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==
dependencies:
safe-buffer "~5.2.0"
stringify-object@^3.3.0:
version "3.3.0"
resolved "https://registry.yarnpkg.com/stringify-object/-/stringify-object-3.3.0.tgz#703065aefca19300d3ce88af4f5b3956d7556629"
@ -1967,6 +2045,27 @@ table@^6.0.9:
string-width "^4.2.3"
strip-ansi "^6.0.1"
tar-fs@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784"
integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==
dependencies:
chownr "^1.1.1"
mkdirp-classic "^0.5.2"
pump "^3.0.0"
tar-stream "^2.1.4"
tar-stream@^2.1.4:
version "2.2.0"
resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287"
integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==
dependencies:
bl "^4.0.3"
end-of-stream "^1.4.1"
fs-constants "^1.0.0"
inherits "^2.0.3"
readable-stream "^3.1.1"
text-table@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
@ -2070,6 +2169,11 @@ uri-js@^4.2.2:
dependencies:
punycode "^2.1.0"
util-deprecate@^1.0.1:
version "1.0.2"
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=
v8-compile-cache@^2.0.3:
version "2.3.0"
resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee"