debcargo-conf/dev/chain_build.py

#!/usr/bin/env python3

USAGE = 'Usage: dev/chain_build.py [!]<CRATE>[-<SEMVER>][=<REALVER>] <CRATE2>[-<SEMVER>][=<REALVER>][!] ...'

HELP = f'''
{USAGE}

Build a chain of packages, each having all previous packages as 'extra
dependency deb' as in build.sh. Fails when one in the chain fails to build, and
picks up where it stopped next time by checking which packages have been
recently built.

`<CRATE>-<SEMVER>` means a versioned package, i.e. src/<CRATE>-<SEMVER>.
It's considered different than `<CRATE>`.

If multiple instances of the same crate are given, the first versioned
(crate=ver) overrides others. This is to ease manually specifying versions
before autogenerated ones (e.g. debcargo build-order), otherwise the generated
first occurence would have been built.

If a crate is in the apt cache or has built debs, its build is skipped. Prefix
or suffix it with '!' to force build. The target crate (last one) is always
built.

This script needs python-apt to work.

This script expects to run at the root of the debcargo-conf repository.

Env vars:
- DISTRO
- CHROOT
- SBUILD_OPTS
- EXTRA_DEBS
    Those env vars are passed to build.sh, read it for their descriptions.
- REPACKAGE
    Use ./repackage.sh instead of ./update.sh to prepare the source package
'''

import re
from sys import argv, stdout
from subprocess import run
from os import getcwd, chdir, environ, makedirs
from os.path import basename, exists, join
from glob import glob
from dataclasses import dataclass
from typing import Sequence, Self

try:
    from apt.cache import Cache as AptCache
except Exception:
    print('python-apt is needed, apt install python3-apt and re-run')
    exit(1)


UNKNOWN_VERSION = '*'
COLL_LINE = 'collapse_features = true'
DCH_VER_RE = re.compile(r'\((.*?)\)')
VER_SUFFIX_RE = re.compile(r'([\w-]+)-(\d\S*)$')


aptc = AptCache()


if stdout.isatty():
    def _print(*args):
        print('\n\x1b[34;100m[chain_build]\x1b[;m', *args) # ]] nvim..
else:
    def _print(*args):
        print('\n[chain_build]', *args)


@dataclass(frozen = True)
class CrateSpec:
    name: str
    ver_suffix: str
    ver: str
    force: bool

    @property
    def dash_name(self) -> str:
        return self.name.replace('_', '-')

    @property
    def suffixed(self) -> str:
        # due to its usage, always use dashed name
        if self.ver_suffix:
            return f'{self.dash_name}-{self.ver_suffix}'
        else:
            return self.dash_name

    @property
    def dch_path(self) -> str:
        return join('src', self.suffixed, 'debian', 'changelog')

    @property
    def debcargo_toml_path(self) -> str:
        return join('src', self.suffixed, 'debian', 'debcargo.toml')

    def dch_version(self) -> str:
        line0 = open(self.dch_path).readline()
        search = DCH_VER_RE.search(line0)
        if search:
            return search.group(1)
        return UNKNOWN_VERSION

    def match_deb(self, deb: str) -> bool:
        match_ver = f'-dev_{self.ver}' in deb if self.ver != UNKNOWN_VERSION else True
        return deb.startswith(f'librust-{self.suffixed}-dev') and match_ver

    @classmethod
    def parse(cls, raw: str) -> Self:
        crate, ver = (raw.split('=') + ['*'])[:2]
        # filter out 1.2.3+surplus-version-part
        if '+' in ver:
            ver = ver.split('+')[0]

        suffix_search = VER_SUFFIX_RE.search(crate)
        ver_suffix = ''
        if suffix_search:
            crate, ver_suffix = suffix_search.groups()

        force = False
        if crate[0] == '!':
            force = True
            crate = crate[1:]
        if crate[-1] == '!':
            force = True
            crate = crate[:-1]

        return cls(crate, ver_suffix, ver, force)


@dataclass
class CrateSource:
    spec: CrateSpec
    deb_or_ver: str
    kind: str # 'apt' or 'build'


def find_existing(specs: Sequence[CrateSpec]) -> tuple[CrateSource, ...]:
    # get all debs first, so we needn't walk again and again
    chdir('build')
    debs = glob('*.deb')
    chdir('..')

    built = []

    for spec in specs:
        ver = spec.ver if spec.ver != UNKNOWN_VERSION else spec.dch_version()
        pkg = aptc.get(f'librust-{spec.suffixed}-dev')
        if pkg is not None and pkg.candidate is not None:
            cand_ver = pkg.candidate.version
            if ver == UNKNOWN_VERSION or cand_ver.startswith(ver):
                built.append(CrateSource(spec, cand_ver, 'apt'))
                continue
        if ver == UNKNOWN_VERSION:
            # version isn't specified, and d/changelog doesn't exist,
            # means it's yet to be `./update.sh`d, move on
            continue
        for deb in debs:
            if spec.match_deb(deb):
                built.append(CrateSource(spec, deb, 'build'))
    return tuple(built)


def collapse_features(spec: CrateSpec) -> bool:
    f = open(spec.debcargo_toml_path, 'r+')
    toml = f.read()
    if COLL_LINE in toml:
        return False

    _print(f'writing {COLL_LINE} for {spec.suffixed}')
    lines = toml.split('\n')
    for i, line in enumerate(lines):
        # avoid inserting at end ending up in [some.directive]
        if line.startswith('['):  # ] to work around auto indent in my nvim
            lines.insert(i, COLL_LINE)
            lines.insert(i + 1, '')
            f.seek(0)
            f.write('\n'.join(lines))
            f.close()
            return True
    f.write('\n')
    f.write(COLL_LINE)
    f.close()
    return True


def build_one(spec: CrateSpec, prev_debs: set[str]) -> None:
    args: tuple[str, ...] = (spec.name,)
    if spec.ver_suffix:
        args = (spec.name, spec.ver_suffix)

    env = environ.copy()
    if spec.ver != UNKNOWN_VERSION:
        env['REALVER'] = spec.ver
    # prevent git from stopping us with a pager
    env['GIT_PAGER'] = 'cat'

    # TODO: make repackage.sh the default
    if 'REPACKAGE' in env:
        run(('./repackage.sh',) + args, env=env, check=True)
    else:
        # \n is for when update.sh stops for confirmation
        run(('./update.sh',) + args, env=env, input=b'\n', check=True)
        # if not set before, rerun ./update.sh to enable it
        if collapse_features(spec):
            run(('./update.sh',) + args, env=env, input=b'\n', check=True)

    env['EXTRA_DEBS'] = ','.join(prev_debs)

    chdir('build')
    run(('./build.sh',) + args, env=env, check=True)
    chdir('..')


def try_build(spec: CrateSpec, debs: set[str]) -> None:
    try:
        build_one(spec, debs)
    except Exception as e:
        print(e)
        _print(f'Failed to build {spec.suffixed}. Fix it then press any key to continue.')
        input()
        if basename(getcwd()) == 'build':
            chdir('..')
        try_build(spec, debs)


def chain_build(specs: Sequence[CrateSpec]) -> None:
    found = find_existing(specs)
    env = environ.copy()
    extra_debs = env.get('EXTRA_DEBS')
    built, debs = set(), set()
    target = specs[-1]
    if found:
        _print('Existing debs:')
        for source in found:
            if source.spec == target:
                continue
            built.add(source.spec)
            if source.kind == 'build':
                print(source.spec.suffixed, source.deb_or_ver)
                debs.add(source.deb_or_ver)
            elif source.kind == 'apt':
                print(source.spec.suffixed, source.deb_or_ver, 'in apt repository')
        _print('To be built:')
        for spec in specs:
            if spec not in built:
                print(spec.suffixed, spec.ver, 'FORCE BUILD' if spec.force else '')
    else:
        built, debs = set(), set()
        _print('No recently built packages')
    if extra_debs:
        _print('EXTRA_DEBS:')
        for deb in extra_debs.split(' '):
            print(deb)
            debs.add(deb)

    _print('Press any key to start chain build, Ctrl-C to abort')
    input()

    for spec in specs:
        if spec in built:
            continue
        _print('Building', spec.suffixed, 'version', spec.ver, 'with previous debs', debs)
        try_build(spec, debs)
        built.add(spec)

        chdir('build')
        all_debs = glob('*.deb')
        chdir('..')
        for deb in all_debs:
            if spec.match_deb(deb):
                debs.add(deb)


def main() -> None:
    if len(argv) <= 1:
        print(HELP)
        exit()

    cwd = getcwd()
    if not (exists(join(cwd, 'repackage.sh')) and exists(join(cwd, 'build.sh'))):
        _print('Please run this script at root of debcargo-conf')
        exit(1)

    # Make sure build directory is present
    makedirs('build', exist_ok=True)

    # flatten shell substituted args
    i = 1
    while i < len(argv):
        if ' ' in argv[i]:
            argv[i:] = (list(filter(lambda a: a != '', argv[1].split(' '))) + argv[i + 1 :])
        i += 1
    chain_build(tuple(map(CrateSpec.parse, argv[1:])))


if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        _print('Exitting due to Ctrl-C')