sync scripts with unstable

Signed-off-by: Fabian Grünbichler <f.gruenbichler@proxmox.com>

build.sh

@@ -12,7 +12,7 @@
 #     this when giving extra dependency debs.
 # SKIP_SIGN=1
 #     Skip signing built packages.
-# DISTRIBUTION=$suite
+# DISTRO=$suite
 #     Release to something other than unstable, e.g. experimental
 # CHROOT=$chroot
 #     Build using another schroot than debcargo-unstable-amd64-sbuild
@@ -51,7 +51,7 @@ if test -z "$VER" -o -f "$VER"; then
 else
 	shift 2
 fi
-DISTRIBUTION="${DISTRIBUTION:-unstable}"
+DISTRO="${DISTRO:-unstable}"
 
 PKGNAME=$($DEBCARGO deb-src-name "$CRATE" $VER || abort 1 "couldn't find crate $CRATE")
 DEBVER=$(dpkg-parsechangelog -l $PKGNAME/debian/changelog -SVersion)
@@ -63,14 +63,8 @@ BUILDNAME="${DEBSRC}_${DEBVER}_${DEB_HOST_ARCH}"
 
 if [ -z "$CHROOT" ]; then
 	if [ "$CHROOT_MODE" = "unshare" ]; then
-		CHROOT="$(find ~/.cache/sbuild -iname "debcargo-*" | head -n1)"
-		if [ -z "$CHROOT" ]; then
-			CHROOT="unstable-${DEB_HOST_ARCH}"
-			echo >&2 "Automatically using sbuild tarball unstable-${DEB_HOST_ARCH}; however it's"
-			echo >&2 "strongly recommended to create a separate tarball debcargo-unstable-${DEB_HOST_ARCH}"
-			echo >&2 "so your builds won't have to re-download & re-install cargo, rustc, and llvm every time."
-			echo >&2 "See README.rst section \"Build environment\" for details."
-		fi
+		# nothing to do here, mmdebstrap will create a new chroot tarball on demand
+		CHROOT="debcargo-unstable-${DEB_HOST_ARCH}-sbuild"
 	elif schroot -i -c "debcargo-unstable-${DEB_HOST_ARCH}-sbuild" >/dev/null 2>&1; then
 		CHROOT="debcargo-unstable-${DEB_HOST_ARCH}-sbuild"
 	elif schroot -i -c "unstable-${DEB_HOST_ARCH}-sbuild" >/dev/null 2>&1; then
@@ -100,21 +94,22 @@ Dir "$PWD/aptroot";
 Acquire::Languages "none";
 END
 {
-echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ $DISTRIBUTION main";
-case $DISTRIBUTION in
+echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ $DISTRO main";
+echo "deb-src [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ $DISTRO main";
+case $DISTRO in
 	experimental|rc-buggy)
 		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ unstable main"
 		;;
 	unstable|sid|testing) : ;;
 	*-backports)
-		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ ${DISTRIBUTION%-backports} main";
-		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ ${DISTRIBUTION%-backports}-updates main";
-		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://security.debian.org/debian-security/ ${DISTRIBUTION%-backports}-security main";
+		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ ${DISTRO%-backports} main";
+		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ ${DISTRO%-backports}-updates main";
+		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://security.debian.org/debian-security/ ${DISTRO%-backports}-security main";
 		;;
 	*)
 		# assume stable release
-		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ $DISTRIBUTION-updates main";
-		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://security.debian.org/debian-security/ $DISTRIBUTION-security main";
+		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://deb.debian.org/debian/ $DISTRO-updates main";
+		echo "deb [signed-by=/usr/share/keyrings/debian-archive-keyring.gpg] http://security.debian.org/debian-security/ $DISTRO-security main";
 		;;
 esac;
 } > "./aptroot/etc/apt/sources.list"
@@ -166,7 +161,7 @@ check_build_deps() {
 	if shouldbuild dpkg-dummy/status "$(eval "$(apt-config shell v Dir::State::Lists/d)"; printf "$v")"; then
 		# pretend dpkg status file that marks all packages as installed
 		# this is because dpkg-checkbuilddeps only works on installed pkgs
-		( apt-cache dumpavail -o APT::Default-Release=$DISTRIBUTION && \
+		( apt-cache dumpavail -o APT::Default-Release=$DISTRO && \
 			for i in ${EXTRA_DEBS[*]}; do apt-cache show $(echo $i | cut -d_ -f1); done ) | \
 		sed -e 's/Package: .*/\0\nStatus: install ok installed/g' > dpkg-dummy/status.tmp
 		if ! test -s dpkg-dummy/status.tmp; then
@@ -179,10 +174,14 @@ check_build_deps() {
 
 if ! check_build_deps; then
 	if [ "$IGNORE_MISSING_BUILD_DEPS" != 1 ]; then
-		abort 1 "Missing build-dependencies, but maybe try '{apt,cargo} update'"
+		abort 1 'Missing build dependencies, possibly due to one of these reasons:' \
+			'- Version mismatch, which could be worked around ("relaxed") by patching Cargo.toml' \
+			'- Disabled feature, which requires either enabling in the dependency or disabling locally'
 	fi
 fi
 
+unset APT_CONFIG
+
 if [ "$SOURCEONLY" = 1 ]; then
 	exit
 fi
@@ -200,7 +199,8 @@ if [ -n "${EXTRA_DEBS[*]}" ]; then
 fi
 
 if [ "$CHROOT_MODE" = "unshare" ]; then
-	AUTOPKGTEST_OPTS=("--run-autopkgtest" "--autopkgtest-root-arg=" "--autopkgtest-opts=--apt-upgrade -- unshare -t ${CHROOT} ${DISTRIBUTION:+-r $DISTRIBUTION}")
+	CACHE_DIR=${XDG_CACHE_HOME:-${HOME}/.cache}
+	AUTOPKGTEST_OPTS=("--run-autopkgtest" "--autopkgtest-root-arg=" "--autopkgtest-opts=--apt-upgrade -- unshare -t ${CACHE_DIR}/sbuild/${CHROOT}.tar ${DISTRO:+-r $DISTRO}")
 else
 	AUTOPKGTEST_OPTS=("--run-autopkgtest" "--autopkgtest-root-arg=" "--autopkgtest-opts=-- schroot ${CHROOT}")
 fi
@@ -219,7 +219,7 @@ fi
 SBUILD_CONFIG="$SCRIPTDIR/dev/sbuildrc" sbuild --no-source --arch-any --arch-all \
   ${CHROOT:+-c $CHROOT} \
   ${CHROOT_MODE:+--chroot-mode "$CHROOT_MODE"} \
-  ${DISTRIBUTION:+-d $DISTRIBUTION} \
+  ${DISTRO:+-d $DISTRO} \
   "${EXTRA_DEBS_SBUILD[@]}" \
   "${EXTRA_DEBS_AUTOPKGTEST_OPTS[@]}" \
   "${AUTOPKGTEST_OPTS[@]}" \

dev/chain_build.py

@@ -1,34 +1,39 @@
 #!/usr/bin/env python3
 
-USAGE = 'Usage: dev/chain_build.py [!]<CRATE>[=<REALVER>] <CRATE2>[=<REALVER>] ...'
+USAGE = 'Usage: dev/chain_build.py [!]<CRATE>[-<SEMVER>][=<REALVER>] <CRATE2>[-<SEMVER>][=<REALVER>][!] ...'
 
 HELP = f'''
 {USAGE}
 
-Build a chain of packages, each having all previous packages as "extra
-dependency deb" as in build.sh. It fails when one in the chain fails to build,
-and picks up where it stopped next time by checking which packages have been
+Build a chain of packages, each having all previous packages as 'extra
+dependency deb' as in build.sh. Fails when one in the chain fails to build, and
+picks up where it stopped next time by checking which packages have been
 recently built.
 
+`<CRATE>-<SEMVER>` means a versioned package, i.e. src/<CRATE>-<SEMVER>.
+It's considered different than `<CRATE>`.
+
 If multiple instances of the same crate are given, the first versioned
 (crate=ver) overrides others. This is to ease manually specifying versions
 before autogenerated ones (e.g. debcargo build-order), otherwise the generated
 first occurence would have been built.
 
-Prefixing a crate with "!" forces it to be built.
+If a crate is in the apt cache or has built debs, its build is skipped. Prefix
+or suffix it with '!' to force build. The target crate (last one) is always
+built.
 
 This script needs python-apt to work.
 
 This script expects to run at the root of the debcargo-conf repository.
 
 Env vars:
-- DISTRIBUTION
+- DISTRO
 - CHROOT
 - SBUILD_OPTS
 - EXTRA_DEBS
-Those env vars are passed to build.sh, read it for their descriptions.
-
-- REPACKAGE: Use ./repackage.sh instead of ./update.sh to prepare the source package
+    Those env vars are passed to build.sh, read it for their descriptions.
+- REPACKAGE
+    Use ./repackage.sh instead of ./update.sh to prepare the source package
 '''
 
 import re
@@ -36,215 +41,264 @@ from sys import argv, stdout
 from subprocess import run
 from os import getcwd, chdir, environ, makedirs
 from os.path import basename, exists, join
+from glob import glob
+from dataclasses import dataclass
+from typing import Sequence, Self
+
 try:
-	from apt.cache import Cache as AptCache
-except:
-	print('This scripts depends on python-apt to work, apt install python3-apt and rerun')
-	exit(1)
+    from apt.cache import Cache as AptCache
+except Exception:
+    print('python-apt is needed, apt install python3-apt and re-run')
+    exit(1)
+
+
+UNKNOWN_VERSION = '*'
+COLL_LINE = 'collapse_features = true'
+DCH_VER_RE = re.compile(r'\((.*?)\)')
+VER_SUFFIX_RE = re.compile(r'([\w-]+)-(\d\S*)$')
 
 
 aptc = AptCache()
 
 
-def _todash(crate: str) -> str:
-	return crate.replace('_', '-')
+if stdout.isatty():
+    def _print(*args):
+        print('\n\x1b[34;100m[chain_build]\x1b[;m', *args) # ]] nvim..
+else:
+    def _print(*args):
+        print('\n[chain_build]', *args)
 
 
-def _print(*args):
-	if stdout.isatty():
-		print('\n\x1b[34;100m[chain_build]\x1b[;m', *args)
-	else:
-		print('[chain_build]', *args, '\n')
+@dataclass(frozen = True)
+class CrateSpec:
+    name: str
+    ver_suffix: str
+    ver: str
+    force: bool
+
+    @property
+    def dash_name(self) -> str:
+        return self.name.replace('_', '-')
+
+    @property
+    def suffixed(self) -> str:
+        # due to its usage, always use dashed name
+        if self.ver_suffix:
+            return f'{self.dash_name}-{self.ver_suffix}'
+        else:
+            return self.dash_name
+
+    @property
+    def dch_path(self) -> str:
+        return join('src', self.suffixed, 'debian', 'changelog')
+
+    @property
+    def debcargo_toml_path(self) -> str:
+        return join('src', self.suffixed, 'debian', 'debcargo.toml')
+
+    def dch_version(self) -> str:
+        line0 = open(self.dch_path).readline()
+        search = DCH_VER_RE.search(line0)
+        if search:
+            return search.group(1)
+        return UNKNOWN_VERSION
+
+    def match_deb(self, deb: str) -> bool:
+        match_ver = f'-dev_{self.ver}' in deb if self.ver != UNKNOWN_VERSION else True
+        return deb.startswith(f'librust-{self.suffixed}-dev') and match_ver
+
+    @classmethod
+    def parse(cls, raw: str) -> Self:
+        crate, ver = (raw.split('=') + ['*'])[:2]
+        # filter out 1.2.3+surplus-version-part
+        if '+' in ver:
+            ver = ver.split('+')[0]
+
+        suffix_search = VER_SUFFIX_RE.search(crate)
+        ver_suffix = ''
+        if suffix_search:
+            crate, ver_suffix = suffix_search.groups()
+
+        force = False
+        if crate[0] == '!':
+            force = True
+            crate = crate[1:]
+        if crate[-1] == '!':
+            force = True
+            crate = crate[:-1]
+
+        return cls(crate, ver_suffix, ver, force)
 
 
-# this is actually faster than os.walk
-def _find(pattern: str):
-	 return run(f'ls {pattern}', shell=True, capture_output=True, text=True).stdout.strip().split('\n')
+@dataclass
+class CrateSource:
+    spec: CrateSpec
+    deb_or_ver: str
+    kind: str # 'apt' or 'build'
 
 
-DCH_VER_RE = re.compile(r'\((.*?)\)')
-def _get_dch_version(crate: str) -> str:
-	# normally we check if there is a match, but a valid d/changelog should
-	# always have one
-	return DCH_VER_RE.search(open(join('src', _todash(crate), 'debian', 'changelog')).readline()).group(1)
+def find_existing(specs: Sequence[CrateSpec]) -> tuple[CrateSource, ...]:
+    # get all debs first, so we needn't walk again and again
+    chdir('build')
+    debs = glob('*.deb')
+    chdir('..')
+
+    built = []
+
+    for spec in specs:
+        ver = spec.ver if spec.ver != UNKNOWN_VERSION else spec.dch_version()
+        pkg = aptc.get(f'librust-{spec.suffixed}-dev')
+        if pkg is not None and pkg.candidate is not None:
+            cand_ver = pkg.candidate.version
+            if ver == UNKNOWN_VERSION or cand_ver.startswith(ver):
+                built.append(CrateSource(spec, cand_ver, 'apt'))
+                continue
+        if ver == UNKNOWN_VERSION:
+            # version isn't specified, and d/changelog doesn't exist,
+            # means it's yet to be `./update.sh`d, move on
+            continue
+        for deb in debs:
+            if spec.match_deb(deb):
+                built.append(CrateSource(spec, deb, 'build'))
+    return tuple(built)
 
 
-def find_built(specs: list[tuple[str, str]]) -> list[tuple[str, str, str]]:
-	# get all debs first, so we needn't walk again and again
-	chdir('build')
-	debs = _find('*.deb')
-	chdir('..')
-	built = []
-	_print('Conducting search in apt cache and build/ directory for existing debs')
-	for crate, ver in specs:
-		_crate = _todash(crate)
-		pkg_re = re.compile(f'librust-{_crate}(?:\+.*?)?-dev_{ver}')
-		if ver == '*':
-			try:
-				ver = _get_dch_version(crate)
-			except:
-				pass
-		pkg = aptc.get(f'librust-{_crate}-dev')
-        #		if pkg is not None and pkg.candidate is not None and (ver == '*' or pkg.candidate.version.startswith(ver)):
-        #			built.append((crate, pkg.candidate.version, 'apt'))
-        #			continue
-		if ver == '*':
-			# version isn't specified, and d/changelog doesn't exist,
-			# means it's yet to be `./update.sh`d, move on
-			continue
-		for deb in debs:
-			if pkg_re.match(deb):
-				built.append((crate, deb, 'build'))
-	return built
+def collapse_features(spec: CrateSpec) -> bool:
+    f = open(spec.debcargo_toml_path, 'r+')
+    toml = f.read()
+    if COLL_LINE in toml:
+        return False
+
+    _print(f'writing {COLL_LINE} for {spec.suffixed}')
+    lines = toml.split('\n')
+    for i, line in enumerate(lines):
+        # avoid inserting at end ending up in [some.directive]
+        if line.startswith('['):  # ] to work around auto indent in my nvim
+            lines.insert(i, COLL_LINE)
+            lines.insert(i + 1, '')
+            f.seek(0)
+            f.write('\n'.join(lines))
+            f.close()
+            return True
+    f.write('\n')
+    f.write(COLL_LINE)
+    f.close()
+    return True
 
 
-COLL_LINE = 'collapse_features = true'
-def collapse_features(crate: str):
-	f = open(join('src', _todash(crate), 'debian', 'debcargo.toml'), 'r+')
-	toml = f.read()
-	if COLL_LINE not in toml:
-		_print('writing collapse_features for', crate)
-		lines = toml.split('\n')
-		for i, line in enumerate(lines):
-			# avoid inserting at end ending up in [some.directive]
-			if line.startswith('['): #] to work around auto indent in my nvim
-				lines.insert(i, COLL_LINE)
-				lines.insert(i + 1, '')
-				f.seek(0)
-				f.write('\n'.join(lines))
-				f.close()
-				return True
-		f.write('\n')
-		f.write(COLL_LINE)
-		f.close()
-		return True
+def build_one(spec: CrateSpec, prev_debs: set[str]) -> None:
+    args: tuple[str, ...] = (spec.name,)
+    if spec.ver_suffix:
+        args = (spec.name, spec.ver_suffix)
+
+    env = environ.copy()
+    if spec.ver != UNKNOWN_VERSION:
+        env['REALVER'] = spec.ver
+    # prevent git from stopping us with a pager
+    env['GIT_PAGER'] = 'cat'
+
+    # TODO: make repackage.sh the default
+    if 'REPACKAGE' in env:
+        run(('./repackage.sh',) + args, env=env, check=True)
+    else:
+        # \n is for when update.sh stops for confirmation
+        run(('./update.sh',) + args, env=env, input=b'\n', check=True)
+        # if not set before, rerun ./update.sh to enable it
+        if collapse_features(spec):
+            run(('./update.sh',) + args, env=env, input=b'\n', check=True)
+
+    env['EXTRA_DEBS'] = ','.join(prev_debs)
+
+    chdir('build')
+    run(('./build.sh',) + args, env=env, check=True)
+    chdir('..')
 
 
-def build_one(crate: str, ver: str, prev_debs: list[str]):
-	env = environ.copy()
-	if ver != '*':
-		env['REALVER'] = ver
-	# prevent git from stopping us with a pager
-	env['GIT_PAGER'] = 'cat'
-	if 'REPACKAGE' in env:
-		run(('./repackage.sh', crate), env=env, check=True)
-	else:
-		# \n is for when update.sh stops for confirmation
-		run(('./update.sh', crate), env=env, input=b'\n', check=True)
-		# if not set before, rerun ./update.sh to enable it
-		if collapse_features(crate):
-			run(('./update.sh', crate), env=env, input=b'\n', check=True)
-	env['EXTRA_DEBS'] = ','.join(prev_debs)
-	chdir('build')
-	run(('./build.sh', crate), env=env, check=True)
-	chdir('..')
+def try_build(spec: CrateSpec, debs: set[str]) -> None:
+    try:
+        build_one(spec, debs)
+    except Exception as e:
+        print(e)
+        _print(f'Failed to build {spec.suffixed}. Fix it then press any key to continue.')
+        input()
+        if basename(getcwd()) == 'build':
+            chdir('..')
+        try_build(spec, debs)
 
 
-def parse_specs(specs: tuple[str]) -> list[tuple[str, str]]:
-	recorded = []
-	versions = {}
-	for spec in specs:
-		crate, ver = (spec.split('=') + ['*'])[:2]
-		# filter out 1.2.3+surplus-version-part
-		if '+' in ver:
-			ver = ver.split('+')[0]
+def chain_build(specs: Sequence[CrateSpec]) -> None:
+    found = find_existing(specs)
+    env = environ.copy()
+    extra_debs = env.get('EXTRA_DEBS')
+    built, debs = set(), set()
+    target = specs[-1]
+    if found:
+        _print('Existing debs:')
+        for source in found:
+            if source.spec == target:
+                continue
+            built.add(source.spec)
+            if source.kind == 'build':
+                print(source.spec.suffixed, source.deb_or_ver)
+                debs.add(source.deb_or_ver)
+            elif source.kind == 'apt':
+                print(source.spec.suffixed, source.deb_or_ver, 'in apt repository')
+        _print('To be built:')
+        for spec in specs:
+            if spec not in built:
+                print(spec.suffixed, spec.ver, 'FORCE BUILD' if spec.force else '')
+    else:
+        built, debs = set(), set()
+        _print('No recently built packages')
+    if extra_debs:
+        _print('EXTRA_DEBS:')
+        for deb in extra_debs.split(' '):
+            print(deb)
+            debs.add(deb)
 
-		if crate in recorded:
-			if versions.get(crate, '*') != '*':
-				continue
-		else:
-			recorded.append(crate)
-		versions[crate] = ver
+    _print('Press any key to start chain build, Ctrl-C to abort')
+    input()
 
-	return [(crate, versions[crate]) for crate in recorded]
+    for spec in specs:
+        if spec in built:
+            continue
+        _print('Building', spec.suffixed, 'version', spec.ver, 'with previous debs', debs)
+        try_build(spec, debs)
+        built.add(spec)
+
+        chdir('build')
+        all_debs = glob('*.deb')
+        chdir('..')
+        for deb in all_debs:
+            if spec.match_deb(deb):
+                debs.add(deb)
 
 
-def chain_build(specs):
-	specs = parse_specs(specs)
-	found = find_built(specs)
-	env = environ.copy()
-	extra_debs = env.get('EXTRA_DEBS')
-	built, debs = set(), set()
-	if found:
-		_print('Existing debs:')
-		for crate, deb_or_ver, kind in found:
-			built.add(crate)
-			if kind == 'build':
-				print(crate, deb_or_ver)
-				debs.add(deb_or_ver)
-			elif kind == 'apt':
-				print(crate, deb_or_ver, 'in apt repository')
-		_print('To be built:')
-		for crate, ver in specs:
-			if crate not in built:
-				if ver == '*':
-					try:
-						ver = _get_dch_version(crate)
-					except:
-						pass
-				print(crate, ver, 'FORCE BUILD' if crate[0] == '!' else '')
-	else:
-		built, debs = set(), set()
-		_print('No recently built packages')
-	if extra_debs:
-		_print('EXTRA_DEBS:')
-		for deb in extra_debs.split(' '):
-			print(deb)
-			debs.add(deb)
+def main() -> None:
+    if len(argv) <= 1:
+        print(HELP)
+        exit()
 
-	_print('Starting chain build, press any key to continue, Ctrl+C to abort')
-	input()
+    cwd = getcwd()
+    if not (exists(join(cwd, 'repackage.sh')) and exists(join(cwd, 'build.sh'))):
+        _print('Please run this script at root of debcargo-conf')
+        exit(1)
 
-	def try_build(crate, ver, debs):
-		try:
-			build_one(crate, ver, debs)
-		except Exception as e:
-			print(e)
-			_print(f'Failed to build crate {crate}. Please fix it then press any key to continue.')
-			input()
-			if basename(getcwd()) == 'build':
-				chdir('..')
-			try_build(crate, ver, debs)
+    # Make sure build directory is present
+    makedirs('build', exist_ok=True)
 
-	for crate, ver in specs:
-		if crate in built:
-			continue
-		if crate[0] == '!':
-			crate = crate[1:]
-		_print('Start building', crate, 'version', ver, 'with previous debs', debs)
-		try_build(crate, ver, debs)
-		built.add(crate)
-		if ver == '*':
-			# used in a glob, so
-			ver = ''
-		_crate = _todash(crate)
-		pkg_re = re.compile(f'librust-{_crate}(?:\+.*?)?-dev_{ver}')
-		chdir('build')
-		all_debs = _find('*.deb')
-		chdir('..')
-		for deb in all_debs:
-			if pkg_re.match(deb):
-				debs.add(deb)
+    # flatten shell substituted args
+    i = 1
+    while i < len(argv):
+        if ' ' in argv[i]:
+            argv[i:] = (list(filter(lambda a: a != '', argv[1].split(' '))) + argv[i + 1 :])
+        i += 1
+    chain_build(tuple(map(CrateSpec.parse, argv[1:])))
 
 
 if __name__ == '__main__':
-	if len(argv) <= 2:
-		print(HELP)
-		exit()
-
-	cwd = getcwd()
-	if not (exists(join(cwd, 'repackage.sh')) and exists(join(cwd, 'build.sh'))):
-		_print('Please run this script at root of debcargo-conf')
-		exit(1)
-
-	# Make sure build directory is present
-	makedirs('build', exist_ok=True)
-
-	# flatten shell substituted args
-	i = 1
-	while i < len(argv):
-		if ' ' in argv[i]:
-			argv[i:] = list(filter(lambda a: a != '', argv[1].split(' '))) + argv[i + 1:]
-		i += 1
-	chain_build(argv[1:])
+    try:
+        main()
+    except KeyboardInterrupt:
+        _print('Exitting due to Ctrl-C')
 

dev/create-rfs.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+set -eu
+CRATE="$1"
+touch "src/$CRATE/debian/RFS"
+git add "src/$CRATE/debian/RFS"
+git commit -vm "$CRATE: create RFS file"

dev/grant-dm-perms.sh

@@ -1,6 +1,16 @@
 #!/bin/bash
 # Grant DM upload permissions on all crates maintained by a DM.
 
+# this script needs dcut-ng
+# and deb-src entries in /etc/apt/sources.list
+
+if [ "$1" = "--dry-run" ] ; then
+	DRY_RUN=true
+	shift
+else
+	DRY_RUN=false
+fi
+
 if [ "$#" -ne 1 ]; then
     echo "Syntax: $0 <email>"
     exit 1
@@ -8,9 +18,25 @@ fi
 
 m="$1"
 shift
-grep -l "$m" src/*/debian/copyright src/*/debian/debcargo.toml \
+pkgs=$(grep -l "$m" src/*/debian/copyright src/*/debian/debcargo.toml \
 | sed -nre 's,src/(.*)/debian/.*,\1,gp' \
 | sort -u \
 | dev/filter-package-in-debian.sh \
-| sed -nre 's/^(.*)\s[1-9][0-9]*$/rust-\1/gp' \
-| xargs -r dcut "$@" dm --uid "$m" --allow
+| sed -nre 's/^(.*)\s[1-9][0-9]*$/rust-\1/gp')
+
+if [ -n "$pkgs" ] ; then
+	if ! $DRY_RUN ; then
+		echo "Do you really want to grant DM upload permissions to $m for these packages:"
+		echo $pkgs
+		echo
+		echo "Press enter to grant DM upload permissions to $m for these packages, else press CTRL-C to abort."
+		read a
+		dcut "$@" dm --uid "$m" --allow $pkgs
+	else
+		echo "Run the following command (with dput-ng installed) to grant DM upload permissions to $m for their packages:"
+		echo
+		echo dcut "$@" dm --uid "$m" --allow $pkgs
+	fi
+else
+	echo "No packages for $m found, either the email address is wrong or maybe you have no deb-src entries in sources.list?"
+fi

dev/list-binpkgs.sh

@@ -1,7 +1,19 @@
 #!/bin/bash
 # List all packages that produce rust binaries, from this repo.
+
+# If the first command line argument is set to --uploaded, only show programs that have
+# been uploaded
+check_uploaded=${1:-disable}
+
 grep -l "\[packages.bin\]" src/*/debian/debcargo.toml \
   | cut -d/ -f2 \
-  | sed -e 's/^/rust-/g' \
+  | while read pkg; do
+      # If check is enabled and 'uploaded' is found, or if check is disabled
+      if [ "$check_uploaded" == "--uploaded" ] && grep -q "unstable" "src/$pkg/debian/changelog"; then
+          echo "rust-$pkg"
+      elif [ "$check_uploaded" == "disable" ]; then
+          echo "rust-$pkg"
+      fi
+    done \
   | tr '\n' ',' \
   | sed -e 's/,$/\n/'

dev/list-outdated.sh

@@ -13,13 +13,14 @@ check_command curl jq
 
 pushd src >/dev/null
 
-NAMES=*
-if [ $# -ne 0 ]; then
-    NAMES=$@
+if [ ${#@} -eq 0 ]; then
+	names="$(ls | grep --invert-match --perl-regexp '\w-\d')"
+else
+	names=$@
 fi
 
 echo -e "crate\tpackaged\tcrates.io"
-for name in $NAMES; do
+for name in $names; do
     if [ ! -e $name/debian/changelog ]; then
         continue
     fi

dev/list-rdeps.sh

@@ -18,34 +18,10 @@ type gawk >/dev/null || abort 1 "gawk not found, install gawk"
 ARCHIVE="${ARCHIVE:-unstable}"
 ARCHIVT="${ARCHIVT:-testing}"
 
-grep_sources_entry() {
-	grep '^deb[^#]*://[^#[:space:]]*[[:space:]]*'"$@"
-}
-
-if ! grep_sources_entry "$ARCHIVE" -qR /etc/apt/sources.list /etc/apt/sources.list.d/ || \
-   ! grep_sources_entry "$ARCHIVT" -qR /etc/apt/sources.list /etc/apt/sources.list.d/; then
-	cat <<-eof
-To make this script work, you will need Debian Testing *AND* Debian Unstable
-in your sources.list. If you want your system to prefer Debian Testing, be
-sure to also add these lines to your /etc/apt/apt.conf:
-
-APT::Default-Release "$ARCHIVT";
-
-After these changes, make sure to re-run \`apt-get update\`.
-eof
-	exit 1
-fi
-
-if [ $(($(date +%s) - $(stat -c %Y /var/cache/apt/pkgcache.bin))) -gt 7200 ]; then
-	read -p "APT cache is a bit old, update? [Y/n] " x
-	if [ "$x" != "n" ]; then sudo apt update; fi
-fi
-
 apt_versions() {
 	aptitude versions --disable-columns -F '%e %p %t' --group-by=none "~rnative $1" || true
 }
 
-all_rust_packages="$(apt_versions "~e^rust-")"
 quick_apt_versions() {
 	printf "%s\n" "$all_rust_packages" | gawk "\$1 ~ /$1/ && \$2 ~ /$2/ && \$3 ~ /$3/ && \$4 ~ /$4/ { ${5:-print} }"
 }
@@ -61,13 +37,36 @@ src_version() {
 	echo "${srcver[$src]}"
 }
 
+tmpdir=$(mktemp -d)
+# https://stackoverflow.com/a/14812383 inside "trap" avoids running handler twice
+trap 'excode=$?; rm -rf "'"$tmpdir"'"; trap - EXIT' EXIT HUP INT QUIT PIPE TERM
+
 if [ -n "$INST_CACHE" ]; then
 	inst_cache="$INST_CACHE"
 else
-	inst_cache=$(mktemp)
-	# https://stackoverflow.com/a/14812383 inside "trap" avoids running handler twice
-	trap 'excode=$?; rm -rf "'"$inst_cache"'"; trap - EXIT' EXIT HUP INT QUIT PIPE TERM
+	inst_cache="$tmpdir/cache"
 fi
+touch "$inst_cache"
+
+mkdir -p "$tmpdir/aptroot/etc/apt/apt.conf.d" "$tmpdir/aptroot/var/lib/apt/lists/" "$tmpdir/aptroot/etc/apt/preferences.d" "$tmpdir/aptroot/etc/apt/sources.list.d"
+cat << END > "$tmpdir/aptroot/apt.conf"
+Apt::Architecture "$(dpkg --print-architecture)";
+Apt::Architectures "$(dpkg --print-architecture)";
+Dir "$tmpdir/aptroot";
+Acquire::Languages "none";
+END
+cat << END > "$tmpdir/aptroot/etc/apt/sources.list.d/debian.sources"
+Types: deb deb-src
+URIs: http://deb.debian.org/debian/
+Suites: $ARCHIVE $ARCHIVT
+Components: main
+Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+END
+APT_CONFIG="$tmpdir/aptroot/apt.conf"
+export APT_CONFIG
+apt-get update
+
+all_rust_packages="$(apt_versions "~e^rust-")"
 
 installability() {
 	local r=$(grep -F "$1=$2" "$inst_cache" | cut -f2)
@@ -141,7 +140,7 @@ list_rdeps() {
 		local binpkg="${binver/=*/}"
 		binpkg="$(apt-cache show "$binver" | grep-dctrl -F Package -ns Package -s Provides -e "${binpkg//\+/\\+}" | tr '\n' '|' | sed -e 's/ \+\(([^)]*)\)\?,\? */|/g' -e 's/+/\\+/g' -e 's/|\+$//g' -e 's/|{2,}/|/g')"
 		# check for bin package + all its provided virtual feature packages in one go
-		grep-dctrl -F Testsuite-Triggers -s Package,Version -w "$binpkg" /var/lib/apt/lists/*_dists_"$ARCHIVE"_*_source_Sources* \
+		grep-dctrl -F Testsuite-Triggers -s Package,Version -w "$binpkg" "$tmpdir"/aptroot/var/lib/apt/lists/*_dists_"$ARCHIVE"_*_source_Sources* \
 		  | cut -d: -f2 | cut '-d ' -f2- \
 		  | sed -z -e 's/\n\n/\t/g' -e 's/\n/ /g' -e 's/\t/\n/g' \
 		  | while read triggered ver; do

dev/list-rfs.sh

@@ -3,6 +3,7 @@ for i in src/*/debian/RFS; do
     echo "$(git log -1 --pretty="format:%ct" "$i")" "$i"
 done | sort  | while read t i; do
     pkg=$(basename "$(dirname "$(dirname "$i")")")
+    if test $pkg == "."; then continue; fi
     upstream_pkg=$(grep Upstream-Name src/$pkg/debian/copyright|awk '{print $2}')
     if test -z $upstream_pkg; then
         echo "Could not find upstream package name. is  src/$pkg/debian/copyright correct?"

dev/sbuildrc

@@ -1,6 +1,13 @@
 # The following options can be only appended to from the command line.
 # Let's reset them to empty lists to override the users's ~/.sbuildrc.
 
+$build_dir = undef;
 $lintian_opts = [];
 $autopkgtest_root_args = [];
 $autopkgtest_opts = [];
+
+$unshare_mmdebstrap_auto_create = 1;
+$unshare_mmdebstrap_keep_tarball = 1;
+$unshare_mmdebstrap_extra_args = [
+	"debcargo-unstable-%a-sbuild" => ["--include=ccache,gnupg,dh-cargo,cargo,lintian,perl-openssl-defaults"]
+];

dev/shouldnot.list

@@ -0,0 +1,48 @@
+# This list consists of one RFC822 stanza per entry, like d/control.
+#
+# Currently there are 3 fields: Crate, Version, Reason.
+# - Crate: Name of the described crate.
+# - Version: Applicable version(s), can be omitted if entire crate applies.
+# - Reason: Why it's included, can span multiple lines.
+#
+# Stanzas are ordered by crate names, alphabetically.
+
+Crate: aes-ctr
+Reason: Deprecated upstream, use aes.
+
+Crate: aes-soft
+Reason: Deprecated upstream, use aes.
+
+Crate: block-modes
+Reason: Deprecated upstream, use https://github.com/RustCrypto/block-modes.
+
+Crate: cargo-vendor
+Reason: The functionality now already exists as part of cargo, so there is no need to package it.
+
+Crate: criterion
+Reason: Packaged by Jonas at https://salsa.debian.org/debian/rust-criterion
+
+Crate: criterion-plot
+Version: 0.5+
+Reason: Packaged by Jonas at https://salsa.debian.org/debian/rust-criterion
+
+Crate: gcc
+Reason: Deprecated upstream, use cc.
+
+Crate: miniz-sys
+Reason: Please don't package this for the time being. flate2 was patched to not use miniz, and instead the system zlib. This seems to be working fine. Typically, crates depend on flate2 rather than miniz directly. If a crate depends on miniz directly, try persuading them to use flate2 instead. (If this really needs to be packaged, we should first package miniz as a C static library Debian package, and *then* package miniz-sys here.)
+
+Crate: tempdir
+Reason: Deprecated upstream, use tempfile.
+
+Crate: rustfmt
+Reason: Deprecated upstream.
+
+Crate: synom
+Reason: This is only needed for old versions of syn.
+
+Crate: lazy-regex-proc-macros
+Reason: Packaged by jonas in https://salsa.debian.org/debian/rust-lazy-regex
+
+Crate: webpki-roots
+Reason: duplicates ca-certificates, rdeps should use native-certs or similar crates/features to pick up ca-certificates contents instead (https://bugs.debian.org/1069946)

dev/shouldnot.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+import re
+from email import message_from_string
+from sys import argv
+
+help_text = '''Usage: dev/shouldnot.py CRATE [CRATE [CRATE ...]]
+
+Check if given crate should *not* be packaged. Reasons include
+
+- Already packaged elsewhere
+- Deprecated or archived upstream
+- The crate can explode
+
+Entries are stored in dev/shouldnot.list, read for its format.
+'''
+
+
+# TODO: improve stanza splitting
+RE_NON_COMMENT_LINE = re.compile(r'^[^#]')
+RE_WS_ONLY = re.compile(r'\s*')
+blocks = open('dev/shouldnot.list').read().split('\n\n')
+entries = {}
+for block in blocks:
+	# skip blocks with only whitespace, likely unknown extra empty lines
+	if RE_WS_ONLY.fullmatch(block):
+		continue
+	# skip comment only blocks
+	if block[0] == '#' and not RE_NON_COMMENT_LINE.match(block):
+		continue
+	
+	msg = message_from_string(block)
+	crate = msg.get('Crate')
+	version = msg.get('Version')
+	reason = msg.get('Reason')
+
+	if crate is None or reason is None:
+		print('Invalid stanza:\n')
+		print(block)
+		print('\nPlease fix it in the list file and retry.')
+		exit(2)
+	
+	if crate not in entries:
+		entries[crate] = []
+	entries[crate].append((version, reason))
+
+
+def show(crate):
+	if crate not in entries:
+		print(f'Crate {crate} is not in shouldnot.list')
+		return
+	print(f'Crate {crate} has {len(entries[crate])} entries:')
+	for entry in entries[crate]:
+		if entry[0] is not None:
+			print(f'\tVersion: {entry[0]}')
+		print(f'\t{entry[1]}')
+		print()
+
+if __name__ == '__main__':
+	crates = argv[1:]
+
+	if len(crates) == 0:
+		print(help_text)
+		exit(0)
+
+	for crate in crates:
+		show(crate)
+

release.sh

@@ -12,11 +12,6 @@
 #     NEW upload. This is a dumb consequence of two independently-thought-out
 #     policies but nobody on either team has expressed interest in fixing it,
 #     claiming "not my department".
-# NOUPDATE=1
-#     Tell debcargo not to attempt to update to the latest version, i.e.
-#     autodetect REALVER. Set this if you get unexpected diffs when releasing.
-#     We probably want to switch this on by default, please complain in our IRC
-#     channel if you agree.
 # REUSE_EXISTING_ORIG_TARBALL=1
 #     Re-use the existing .orig tarball. This is needed if it was previously
 #     generated with an old version of debcargo, otherwise you'll get
@@ -50,14 +45,12 @@ type dch >/dev/null || \
 abort 1 "Install devscripts, we need to run dch."
 
 RELBRANCH="pending-$PKGNAME"
-git fetch origin --prune
+timeout --foreground 15 git fetch origin --prune || abort 1 "Failed to fetch upstream to check whether we are synced, please check network"
 
 git merge-base --is-ancestor origin/master HEAD || \
 abort 1 "You are not synced with origin/master, please do so before running this script."
 
-if [ "$RERELEASE" = 1 -o "$NOUPDATE" = 1 ]; then
-	REALVER="$(get_existing_version "$PKGDIR")"
-fi
+REALVER="$(get_existing_version "$PKGDIR")"
 
 if head -n1 "$PKGDIR/debian/changelog" | grep -qv UNRELEASED-FIXME-AUTOGENERATED-DEBCARGO; then
 	if [ "$RERELEASE" = 1 ]; then
@@ -100,9 +93,6 @@ if [ "$CHANGEDBY" != "$UPLOADER" ]; then
 	cat <<eof
 Changes by $CHANGEDBY to be sponsored by $UPLOADER.
 eof
-	if ! test -e $PKGDIR/debian/RFS; then
-		abort 1 "Sponsored upload, but $PKGDIR/debian/RFS does not exist"
-	fi
 fi
 
 ( cd "$PKGDIR"
@@ -134,17 +124,13 @@ if git diff -- "$PKGDIR_REL/debian/copyright.debcargo.hint" | patch -r - --no-ba
 	git add "$PKGDIR_REL/debian/copyright.debcargo.hint" "$PKGDIR_REL/debian/copyright"
 else
 	git diff -- "$PKGDIR_REL/debian/copyright.debcargo.hint"
-	revert_git_changes
-	abort 1 \
-	  "copyright file needs updating; apply the above diff to $PKGDIR_REL/debian/copyright" \
-	  "then commit your changes, and run me again."
+	echo "copyright file should be updated; consider applying the above diff to $PKGDIR_REL/debian/copyright and commiting your changes."
 fi
 
 if ! git diff --exit-code -- "$PKGDIR_REL"; then
 	revert_git_changes
 	abort 1 \
-	  "Release attempt resulted in git diffs to $PKGDIR_REL, probably you need to update the package (./update.sh $*)." \
-	  "Alternatively, set NOUPDATE=1 to override this requirement, but please have a good reason."
+	  "Release attempt resulted in git diffs to $PKGDIR_REL, probably you need to update the package (./update.sh $*)."
 fi
 
 if ! ( cd build && SOURCEONLY=1 ./build.sh "$CRATE" $VER ); then
@@ -154,12 +140,12 @@ if ! ( cd build && SOURCEONLY=1 ./build.sh "$CRATE" $VER ); then
 		"- packaged version is out-of-date => run \`./update.sh $*\`"
 fi
 
-git commit -m "Release package $PKGNAME"
-
 DEBVER=$(dpkg-parsechangelog -l $BUILDDIR/debian/changelog -SVersion)
 DEBSRC=$(dpkg-parsechangelog -l $BUILDDIR/debian/changelog -SSource)
 DEB_HOST_ARCH=$(dpkg-architecture -q DEB_HOST_ARCH)
 
+git commit -m "$PKGNAME: release $DEBVER"
+
 if [ "$RERELEASE" = 1 ]; then
 
 ( cd build && dput "${DEBSRC}_${DEBVER}_source.changes" )
@@ -175,7 +161,7 @@ else
 
 unstable_bin_packages="$(rmadison --noconf --suite unstable --source-and-binary "${DEBSRC}" | grep -v 'source$' | cut -d ' ' -f 1 | sort -u)"
 upload_bin_packages="$(grep '^Binary' "build/${DEBSRC}_${DEBVER}.dsc" | sed -e 's/^Binary: //' -e 's/, /,/g' | tr ',' '\n' | sort -u)"
-diff_bin_packages="$(diff -u0 <(echo "$unstable_bin_packages") <(echo "$upload_bin_packages") | tail -n-2)"
+diff_bin_packages="$(diff -u0 <(echo "$unstable_bin_packages") <(echo "$upload_bin_packages") | grep '^[+-]' || true)"
 new_bin_packages="$(echo "$diff_bin_packages" | grep '^+' | sed -e 's/^+//g')"
 rm_bin_packages="$(echo "$diff_bin_packages" | grep '^-' | sed -e 's/^-//g')"
 
@@ -207,16 +193,26 @@ ${NC}
 eof
 show_build_notice
 
+if test $(echo "$upload_bin_packages"|grep librust|wc -l) -ge 2; then
+    # We have more than one package
+    echo "${RED}"
+    echo "collapse_features = true missing in $PKGDIR_REL/debian/debcargo.toml"
+    echo "${NC}To add it:"
+    echo "git checkout - && git branch -D $RELBRANCH && echo 'collapse_features = true' >> $PKGDIR_REL/debian/debcargo.toml && git commit -m '$PKGDIR_REL: add collapse_features = true' $PKGDIR_REL/debian/debcargo.toml"
+    echo "${NC}"
+    exit 1
+fi
+
 elif [ -z "$new_bin_packages" ]; then
 cat <<eof
 Since the source package is already in Debian and this version does not introduce
 new binaries, then you can just go ahead and directly dput the source package.
 
-  cd build && dput ${DEBSRC}_${DEBVER}_source.changes && git checkout - && git merge -
+  cd build && dput ${DEBSRC}_${DEBVER}_source.changes && cd - && git checkout master
 
 If you want to build and test it, run:
 
-  cd build && ./build.sh $CRATE && dput ${DEBSRC}_${DEBVER}_source.changes && git checkout - && git merge - && cd -
+  cd build && ./build.sh $CRATE && dput ${DEBSRC}_${DEBVER}_source.changes && cd - && git checkout master
 
 For your reference, this source package builds $(echo "$upload_bin_packages" | wc -l) binary package(s):
 $upload_bin_packages

update.sh

@@ -6,12 +6,33 @@
 # Envvars:
 # See also ./vars.sh.frag for its envvars, which we pass through.
 
+
+# Run this before sourcing vars.sh.frag so the user reads it first
+exec_basename="$(basename "$0")"
+case "$exec_basename" in
+new-package.sh|package.sh)
+	echo >&2
+	echo >&2 "WARNING! $exec_basename is deprecated. Please use update.sh instead."
+	echo >&2
+	# Add 1 second sleep so users read the deprecation notice,
+	# get annoyed by the wait and start using update.sh
+	sleep 1
+	;;
+esac
+
 . ./vars.sh.frag
 
 case "$(git rev-parse --abbrev-ref HEAD)" in
 pending-*)	abort 1 "You are on a pending-release branch, $0 can only be run on another branch, like master";;
 esac
 
+timeout --foreground 15 git fetch origin --prune || abort 1 "Failed to fetch upstream to check pending branches, please check network"
+pending_branches=$(git branch --all --list "*pending-$PKGNAME")
+if [ -n "$pending_branches" ]
+then
+	abort 1 "Please resolve these existing pending branches before updating this crate:$(printf "\n$pending_branches")"
+fi
+
 if [ -n "$VER" ]; then
 	if [ ! -d "$PWD/src/$PKGBASE" ]; then
 		abort 1 "Using crate $CRATE with version $VER but default-version is not packaged." \
@@ -24,7 +45,6 @@ if [ ! -d "$PKGDIR/debian" ]; then
 	overlay = "."
 	uploaders = ["$DEBFULLNAME <$DEBEMAIL>"]
 	eof
-	git add "$PKGDIR"
 fi
 if [ ! -f "$PKGDIR/debian/copyright" ]; then
 	cat <<-eof > "$PKGDIR/debian/copyright"
@@ -51,7 +71,6 @@ if ! grep -q uploaders "$PKGCFG"; then
 fi
 
 run_debcargo
-git add -N "$PKGDIR"
 
 if ! git diff --quiet -- "$PKGDIR_REL"; then
 	read -p "Update wrote some changes to $PKGDIR_REL, press enter to git diff..." x || true
@@ -75,7 +94,7 @@ and directly git-add without editing:
 	    src/$PKGNAME/debian/copyright.debcargo.hint
 
 When done, git-add all your changes plus any unmodified hint files, and re-run
-this command (\`./update.sh $*\`).
+this command (\`./update.sh $*\`).  Do NOT run git commit at this stage.
 
 For issues with debian/control, edit src/$PKGNAME/debian/debcargo.toml instead.
 You can find docs for that in debcargo.toml.example in the debcargo git repo.
@@ -94,6 +113,7 @@ You can test-build your package by running:
 
   cd build && ./build.sh $CRATE $VER
 
+You should not run git commit before doing this; it uses files git-added.
 This assumes that you have set up sbuild; see "Build environment" in README.rst
 for details. Try to fix any lintian errors, but note that some errors are due
 to lintian being out-of-date and/or are expected at this stage of the process
@@ -118,14 +138,18 @@ When satisfied with all of these outputs:
 - of lintian after running ./build.sh
 - of dev/list-rdeps.sh $CRATE
 
-then you can commit and push all your changes.
+then you can commit. For example:
+git commit -m "$CRATE: new upstream release" src/$PKGNAME/
+and push all your changes.
 
 Then, ask a Debian Developer to run \`./release.sh $*\`. This finalises your
 changes in the changelog, and allows them to build and upload the package. If
 you're not a Debian Developer and are unable to upload, please don't run that
 script or else you will need to revert the changes that it makes to your git.
-Instead, add an empty RFS file inside the created debian directory; see
-TODO.rst's  "Ready for upload" section for more details.
+Instead, add an empty RFS file inside the created debian directory. If there
+are other issues that need to be addressed such as missing dependencies, they
+can be addressed by adding a comment into the RFS file; see CONTRIBUTING.rst's
+"Ready for upload" section for more details.
 eof
 if [ -n "$VER" ]; then
 cat >&2 <<eof