#!/usr/bin/env python3 USAGE = 'Usage: dev/chain_build.py [!][-][=] [-][=][!] ...' HELP = f''' {USAGE} Build a chain of packages, each having all previous packages as 'extra dependency deb' as in build.sh. Fails when one in the chain fails to build, and picks up where it stopped next time by checking which packages have been recently built. `-` means a versioned package, i.e. src/-. It's considered different than ``. If multiple instances of the same crate are given, the first versioned (crate=ver) overrides others. This is to ease manually specifying versions before autogenerated ones (e.g. debcargo build-order), otherwise the generated first occurence would have been built. If a crate is in the apt cache or has built debs, its build is skipped. Prefix or suffix it with '!' to force build. The target crate (last one) is always built. This script needs python-apt to work. This script expects to run at the root of the debcargo-conf repository. Env vars: - DISTRO - CHROOT - SBUILD_OPTS - EXTRA_DEBS Those env vars are passed to build.sh, read it for their descriptions. - REPACKAGE Use ./repackage.sh instead of ./update.sh to prepare the source package ''' import re from sys import argv, stdout from subprocess import run from os import getcwd, chdir, environ, makedirs from os.path import basename, exists, join from glob import glob from dataclasses import dataclass from typing import Sequence, Self try: from apt.cache import Cache as AptCache except Exception: print('python-apt is needed, apt install python3-apt and re-run') exit(1) UNKNOWN_VERSION = '*' COLL_LINE = 'collapse_features = true' DCH_VER_RE = re.compile(r'\((.*?)\)') VER_SUFFIX_RE = re.compile(r'([\w-]+)-(\d\S*)$') aptc = AptCache() if stdout.isatty(): def _print(*args): print('\n\x1b[34;100m[chain_build]\x1b[;m', *args) # ]] nvim.. else: def _print(*args): print('\n[chain_build]', *args) @dataclass(frozen = True) class CrateSpec: name: str ver_suffix: str ver: str force: bool @property def dash_name(self) -> str: return self.name.replace('_', '-') @property def suffixed(self) -> str: # due to its usage, always use dashed name if self.ver_suffix: return f'{self.dash_name}-{self.ver_suffix}' else: return self.dash_name @property def dch_path(self) -> str: return join('src', self.suffixed, 'debian', 'changelog') @property def debcargo_toml_path(self) -> str: return join('src', self.suffixed, 'debian', 'debcargo.toml') def dch_version(self) -> str: line0 = open(self.dch_path).readline() search = DCH_VER_RE.search(line0) if search: return search.group(1) return UNKNOWN_VERSION def match_deb(self, deb: str) -> bool: match_ver = f'-dev_{self.ver}' in deb if self.ver != UNKNOWN_VERSION else True return deb.startswith(f'librust-{self.suffixed}-dev') and match_ver @classmethod def parse(cls, raw: str) -> Self: crate, ver = (raw.split('=') + ['*'])[:2] # filter out 1.2.3+surplus-version-part if '+' in ver: ver = ver.split('+')[0] suffix_search = VER_SUFFIX_RE.search(crate) ver_suffix = '' if suffix_search: crate, ver_suffix = suffix_search.groups() force = False if crate[0] == '!': force = True crate = crate[1:] if crate[-1] == '!': force = True crate = crate[:-1] return cls(crate, ver_suffix, ver, force) @dataclass class CrateSource: spec: CrateSpec deb_or_ver: str kind: str # 'apt' or 'build' def find_existing(specs: Sequence[CrateSpec]) -> tuple[CrateSource, ...]: # get all debs first, so we needn't walk again and again chdir('build') debs = glob('*.deb') chdir('..') built = [] for spec in specs: ver = spec.ver if spec.ver != UNKNOWN_VERSION else spec.dch_version() pkg = aptc.get(f'librust-{spec.suffixed}-dev') if pkg is not None and pkg.candidate is not None: cand_ver = pkg.candidate.version if ver == UNKNOWN_VERSION or cand_ver.startswith(ver): built.append(CrateSource(spec, cand_ver, 'apt')) continue if ver == UNKNOWN_VERSION: # version isn't specified, and d/changelog doesn't exist, # means it's yet to be `./update.sh`d, move on continue for deb in debs: if spec.match_deb(deb): built.append(CrateSource(spec, deb, 'build')) return tuple(built) def collapse_features(spec: CrateSpec) -> bool: f = open(spec.debcargo_toml_path, 'r+') toml = f.read() if COLL_LINE in toml: return False _print(f'writing {COLL_LINE} for {spec.suffixed}') lines = toml.split('\n') for i, line in enumerate(lines): # avoid inserting at end ending up in [some.directive] if line.startswith('['): # ] to work around auto indent in my nvim lines.insert(i, COLL_LINE) lines.insert(i + 1, '') f.seek(0) f.write('\n'.join(lines)) f.close() return True f.write('\n') f.write(COLL_LINE) f.close() return True def build_one(spec: CrateSpec, prev_debs: set[str]) -> None: args: tuple[str, ...] = (spec.name,) if spec.ver_suffix: args = (spec.name, spec.ver_suffix) env = environ.copy() if spec.ver != UNKNOWN_VERSION: env['REALVER'] = spec.ver # prevent git from stopping us with a pager env['GIT_PAGER'] = 'cat' # TODO: make repackage.sh the default if 'REPACKAGE' in env: run(('./repackage.sh',) + args, env=env, check=True) else: # \n is for when update.sh stops for confirmation run(('./update.sh',) + args, env=env, input=b'\n', check=True) # if not set before, rerun ./update.sh to enable it if collapse_features(spec): run(('./update.sh',) + args, env=env, input=b'\n', check=True) env['EXTRA_DEBS'] = ','.join(prev_debs) chdir('build') run(('./build.sh',) + args, env=env, check=True) chdir('..') def try_build(spec: CrateSpec, debs: set[str]) -> None: try: build_one(spec, debs) except Exception as e: print(e) _print(f'Failed to build {spec.suffixed}. Fix it then press any key to continue.') input() if basename(getcwd()) == 'build': chdir('..') try_build(spec, debs) def chain_build(specs: Sequence[CrateSpec]) -> None: found = find_existing(specs) env = environ.copy() extra_debs = env.get('EXTRA_DEBS') built, debs = set(), set() target = specs[-1] if found: _print('Existing debs:') for source in found: if source.spec == target: continue built.add(source.spec) if source.kind == 'build': print(source.spec.suffixed, source.deb_or_ver) debs.add(source.deb_or_ver) elif source.kind == 'apt': print(source.spec.suffixed, source.deb_or_ver, 'in apt repository') _print('To be built:') for spec in specs: if spec not in built: print(spec.suffixed, spec.ver, 'FORCE BUILD' if spec.force else '') else: built, debs = set(), set() _print('No recently built packages') if extra_debs: _print('EXTRA_DEBS:') for deb in extra_debs.split(' '): print(deb) debs.add(deb) _print('Press any key to start chain build, Ctrl-C to abort') input() for spec in specs: if spec in built: continue _print('Building', spec.suffixed, 'version', spec.ver, 'with previous debs', debs) try_build(spec, debs) built.add(spec) chdir('build') all_debs = glob('*.deb') chdir('..') for deb in all_debs: if spec.match_deb(deb): debs.add(deb) def main() -> None: if len(argv) <= 1: print(HELP) exit() cwd = getcwd() if not (exists(join(cwd, 'repackage.sh')) and exists(join(cwd, 'build.sh'))): _print('Please run this script at root of debcargo-conf') exit(1) # Make sure build directory is present makedirs('build', exist_ok=True) # flatten shell substituted args i = 1 while i < len(argv): if ' ' in argv[i]: argv[i:] = (list(filter(lambda a: a != '', argv[1].split(' '))) + argv[i + 1 :]) i += 1 chain_build(tuple(map(CrateSpec.parse, argv[1:]))) if __name__ == '__main__': try: main() except KeyboardInterrupt: _print('Exitting due to Ctrl-C')