From cc8eb5ca8c391e597eb51c2e1d8543b2a4246dfc Mon Sep 17 00:00:00 2001 From: Mario Limonciello Date: Thu, 16 Sep 2021 22:48:46 -0500 Subject: [PATCH] trivial: Merge python steps from contrib/setup into helper script This avoids having to hardcode profile targets in multiple places and also fixes the confusing entry points into scripts both by arguments and environment variables. It also makes the setup script a lot more debuggable and scalable. OS detection is a lot more robust, where it will try to use pip to set up the distro python package, and if pip is missing try to install it. If OS detection fails now, a user can use --os on contrib/setup for specifying it. --- .github/workflows/main.yml | 2 +- contrib/ci/README.md | 2 +- contrib/ci/arch.sh | 2 +- contrib/ci/debian.sh | 2 +- contrib/ci/fedora.sh | 2 +- contrib/ci/fwupd_setup_helpers.py | 165 ++++++++++++++++++++++++++++ contrib/ci/generate_dependencies.py | 82 +------------- contrib/ci/generate_docker.py | 2 +- contrib/ci/ubuntu.sh | 2 +- contrib/ci/void.sh | 2 +- contrib/setup | 76 +++++++++---- 11 files changed, 231 insertions(+), 108 deletions(-) create mode 100755 contrib/ci/fwupd_setup_helpers.py mode change 100755 => 120000 contrib/ci/generate_dependencies.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b93d91375..983fa0468 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,7 +31,7 @@ jobs: - name: Refresh dependencies run: sudo apt update - name: Install dependencies - run: ./contrib/ci/generate_dependencies.py | sudo xargs apt install -y + run: sudo ./contrib/ci/fwupd_setup_helpers.py install-dependencies -o ubuntu --yes - name: Check ABI run: ./contrib/ci/check-abi $(git describe --abbrev=0 --tags) $(git rev-parse HEAD) diff --git a/contrib/ci/README.md b/contrib/ci/README.md index 76140f9e3..f0a9e891a 100644 --- a/contrib/ci/README.md +++ b/contrib/ci/README.md @@ -99,7 +99,7 @@ Each distribution will have `package` elements and `control` elements. * `inclusive` elements represent an inclusive list of architectures to be installed on * `exclusive` elements represent an exclusive list of architectures to not be installed on -For convenience there is also a standalone script __generate_dependencies.py__ that parses `dependencies.xml`. +For convenience there is also a helper script `./contrib/ci/fwupd_setup_helpers.p install-dependencies` that parses `dependencies.xml`. ### Dockerfile.in diff --git a/contrib/ci/arch.sh b/contrib/ci/arch.sh index d479f81d6..f196616dd 100755 --- a/contrib/ci/arch.sh +++ b/contrib/ci/arch.sh @@ -7,7 +7,7 @@ shopt -s extglob . ./contrib/ci/get_test_firmware.sh #install anything missing from the container -./contrib/ci/generate_dependencies.py | xargs pacman -S --noconfirm --needed +./contrib/ci/fwupd_setup_helpers.py install-dependencies -o arch # prepare the build tree rm -rf build diff --git a/contrib/ci/debian.sh b/contrib/ci/debian.sh index d8615def2..014dbdf02 100755 --- a/contrib/ci/debian.sh +++ b/contrib/ci/debian.sh @@ -36,7 +36,7 @@ sed s/quilt/native/ debian/source/format -i #check if we have all deps available #if some are missing, we're going to use subproject instead and #packaging CI will fail -./contrib/ci/generate_dependencies.py | xargs apt install -y || true +./contrib/ci/fwupd_setup_helpers.py install-dependencies -o debian --yes || true if ! dpkg-checkbuilddeps; then ./contrib/ci/ubuntu.sh exit 0 diff --git a/contrib/ci/fedora.sh b/contrib/ci/fedora.sh index 24811aa48..5e822aee7 100755 --- a/contrib/ci/fedora.sh +++ b/contrib/ci/fedora.sh @@ -3,7 +3,7 @@ set -e set -x #get any missing deps from the container -./contrib/ci/generate_dependencies.py | xargs dnf install -y +./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o fedora #generate a tarball git config tar.tar.xz.command "xz -c" diff --git a/contrib/ci/fwupd_setup_helpers.py b/contrib/ci/fwupd_setup_helpers.py new file mode 100755 index 000000000..32bf4f8a7 --- /dev/null +++ b/contrib/ci/fwupd_setup_helpers.py @@ -0,0 +1,165 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2017 Dell, Inc. +# Copyright (C) 2020 Intel, Inc. +# Copyright (C) 2021 Mario Limonciello +# +# SPDX-License-Identifier: LGPL-2.1+ +# +import os +import sys +import argparse + +# Minimum version of markdown required +MINIMUM_MARKDOWN = (3, 3, 3) + + +def get_possible_profiles(): + return ["fedora", "centos", "debian", "ubuntu", "arch", "void"] + + +def detect_profile(): + try: + import distro + + target = distro.id() + if not target in get_possible_profiles(): + target = distro.like() + except ModuleNotFoundError: + target = "" + return target + + +def test_markdown(): + import markdown + + new_enough = markdown.__version_info__ >= MINIMUM_MARKDOWN + if not new_enough: + print("python3-markdown must be upgraded") + sys.exit(not new_enough) + + +def parse_dependencies(OS, SUBOS, requested_type): + import xml.etree.ElementTree as etree + + deps = [] + dep = "" + directory = os.path.dirname(sys.argv[0]) + tree = etree.parse(os.path.join(directory, "dependencies.xml")) + root = tree.getroot() + for child in root: + if "type" not in child.attrib or "id" not in child.attrib: + continue + for distro in child: + if "id" not in distro.attrib: + continue + if distro.attrib["id"] != OS: + continue + packages = distro.findall("package") + for package in packages: + if SUBOS: + if "variant" not in package.attrib: + continue + if package.attrib["variant"] != SUBOS: + continue + if package.text: + dep = package.text + else: + dep = child.attrib["id"] + if child.attrib["type"] == requested_type and dep: + deps.append(dep) + return deps + + +def get_build_dependencies(os, variant): + return parse_dependencies(os, variant, "build") + + +def _get_installer_cmd(os, yes): + if os == "debian" or os == "ubuntu": + installer = ["apt", "install"] + elif os == "fedora": + installer = ["dnf", "install"] + elif os == "arch": + installer = ["pacman", "-Syu", "--noconfirm", "--needed"] + elif os == "void": + installer = ["xbps-install", "-Syu"] + else: + print("unable to detect OS profile, use --os= to specify") + print("\tsupported profiles: %s" % get_possible_profiles()) + sys.exit(1) + if yes: + installer += ["-y"] + return installer + + +def install_packages(os, variant, yes, debugging, packages): + import subprocess + + if packages == "build-dependencies": + packages = get_build_dependencies(os, variant) + installer = _get_installer_cmd(os, yes) + installer += packages + if debugging: + print(installer) + subprocess.call(installer) + + +if __name__ == "__main__": + + command = None + # compat mode for old training documentation + if "generate_dependencies.py" in sys.argv[0]: + command = "get-dependencies" + + parser = argparse.ArgumentParser() + if not command: + parser.add_argument( + "command", + choices=[ + "get-dependencies", + "test-markdown", + "detect-profile", + "install-dependencies", + "install-pip", + ], + help="command to run", + ) + parser.add_argument( + "-o", + "--os", + default=detect_profile(), + choices=get_possible_profiles(), + help="calculate dependencies for OS profile", + ) + parser.add_argument( + "-v", "--variant", help="optional machine variant for the OS profile" + ) + parser.add_argument( + "-y", "--yes", action="store_true", help="Don't prompt to install" + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="Display all launched commands" + ) + args = parser.parse_args() + + # these require variants to be set + if not args.variant and (args.os == "ubuntu" or args.os == "debian"): + args.variant = os.uname().machine + if not command: + command = args.command + + # command to run + if command == "test-markdown": + test_markdown() + elif command == "detect-profile": + print(detect_profile()) + elif command == "get-dependencies": + dependencies = get_build_dependencies(args.os, args.variant) + print(*dependencies, sep="\n") + elif command == "install-dependencies": + install_packages( + args.os, args.variant, args.yes, args.debug, "build-dependencies" + ) + elif command == "install-pip": + install_packages(args.os, args.variant, args.yes, args.debug, ["python3-pip"]) diff --git a/contrib/ci/generate_dependencies.py b/contrib/ci/generate_dependencies.py deleted file mode 100755 index 4aa99d2b4..000000000 --- a/contrib/ci/generate_dependencies.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2017 Dell, Inc. -# Copyright (C) 2020 Intel, Inc. -# -# SPDX-License-Identifier: LGPL-2.1+ -# -import os -import sys -import argparse -import xml.etree.ElementTree as etree - - -def parse_dependencies(OS, SUBOS, requested_type): - deps = [] - dep = "" - directory = os.path.dirname(sys.argv[0]) - tree = etree.parse(os.path.join(directory, "dependencies.xml")) - root = tree.getroot() - for child in root: - if "type" not in child.attrib or "id" not in child.attrib: - continue - for distro in child: - if "id" not in distro.attrib: - continue - if distro.attrib["id"] != OS: - continue - packages = distro.findall("package") - for package in packages: - if SUBOS: - if "variant" not in package.attrib: - continue - if package.attrib["variant"] != SUBOS: - continue - if package.text: - dep = package.text - else: - dep = child.attrib["id"] - if child.attrib["type"] == requested_type and dep: - deps.append(dep) - return deps - - -if __name__ == "__main__": - possible_targets = ["fedora", "centos", "debian", "ubuntu", "arch"] - - try: - import distro - - target = distro.id() - if not target in possible_targets: - target = distro.like() - except ModuleNotFoundError: - target = None - - parser = argparse.ArgumentParser() - parser.add_argument( - "-o", - "--os", - default=target, - choices=possible_targets, - help="dependencies for OS", - ) - args = parser.parse_args() - - # these require variants to be set - if args.os == "ubuntu" or args.os == "debian": - args.os = "%s-%s" % (args.os, os.uname().machine) - - target = os.getenv("OS", args.os) - if target is None: - print("Missing OS environment variable") - sys.exit(1) - - _os = target.lower() - _sub_os = "" - split = target.split("-") - if len(split) >= 2: - _os, _sub_os = split[:2] - dependencies = parse_dependencies(_os, _sub_os, "build") - print(*dependencies, sep="\n") diff --git a/contrib/ci/generate_dependencies.py b/contrib/ci/generate_dependencies.py new file mode 120000 index 000000000..cc1f5adae --- /dev/null +++ b/contrib/ci/generate_dependencies.py @@ -0,0 +1 @@ +fwupd_setup_helpers.py \ No newline at end of file diff --git a/contrib/ci/generate_docker.py b/contrib/ci/generate_docker.py index 7390cb661..9640ab1c0 100755 --- a/contrib/ci/generate_docker.py +++ b/contrib/ci/generate_docker.py @@ -8,7 +8,7 @@ import os import subprocess import sys import shutil -from generate_dependencies import parse_dependencies +from fwupd_setup_helpers import parse_dependencies def get_container_cmd(): diff --git a/contrib/ci/ubuntu.sh b/contrib/ci/ubuntu.sh index 2a7c01ec2..1688ace83 100755 --- a/contrib/ci/ubuntu.sh +++ b/contrib/ci/ubuntu.sh @@ -6,7 +6,7 @@ set -x . ./contrib/ci/get_test_firmware.sh #check for and install missing dependencies -./contrib/ci/generate_dependencies.py | xargs apt install -y +./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o ubuntu #evaluate using Ubuntu's buildflags #evaluate using Debian/Ubuntu's buildflags diff --git a/contrib/ci/void.sh b/contrib/ci/void.sh index 81739897a..9aa03786f 100755 --- a/contrib/ci/void.sh +++ b/contrib/ci/void.sh @@ -4,7 +4,7 @@ set -x #install dependencies xbps-install -Suy python3 -./contrib/ci/generate_dependencies.py | xargs xbps-install -y +./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o void #clone test firmware if necessary . ./contrib/ci/get_test_firmware.sh diff --git a/contrib/setup b/contrib/setup index 0abef17c2..1c4bd36d3 100755 --- a/contrib/setup +++ b/contrib/setup @@ -3,21 +3,14 @@ cd "$(dirname "$0")/.." +HELPER=./contrib/ci/fwupd_setup_helpers.py +HELPER_ARGS="-y" + setup_deps() { - OS=$1 - read -p "Install build dependencies? (y/n) " question + read -p "Install build dependencies? (y/n) " question if [ "$question" = "y" ]; then - DEPS=$(./contrib/ci/generate_dependencies.py) - if [ "$OS" = "debian" ] || [ "$OS" = "ubuntu" ]; then - sudo apt install $DEPS - elif [ "$OS" = "fedora" ]; then - sudo dnf install $DEPS - elif [ "$OS" = "arch" ]; then - pacman -Syu --noconfirm --needed $DEPS - elif [ "$OS" = "void" ]; then - xbps-install -Syu $DEPS - fi + $(which sudo) python3 $HELPER install-dependencies $HELPER_ARGS -y fi } @@ -47,34 +40,79 @@ setup_git() git config include.path ../.gitconfig } +install_pip() +{ + package=$1 + args=$2 + if ! python3 -m pip install $package $args; then + $(which sudo) python3 $HELPER install-pip $HELPER_ARGS -y + fi + #try once more + python3 -m pip install $package +} + setup_precommit() { echo "Configuring pre-commit hooks" python3 -m venv venv source venv/bin/activate - python3 -m pip install pre-commit + install_pip pre-commit pre-commit install } check_markdown() { - if python3 -c "import markdown; import sys; sys.exit(markdown.__version_info__ >= (3,3,3))"; then - echo "Upgrading python3-markdown version" - python3 -m pip install markdown --upgrade + if ! python3 $HELPER test-markdown; then + install_pip markdown --upgrade fi } +detect_os() +{ + for i in "$@"; do + case $i in + --os=*) + OS="${i#*=}" + shift + ;; + --debug) + DEBUG=1 + shift + ;; + *) + ;; + esac + done + if [ -z $OS ]; then + OS=$(python3 $HELPER detect-profile) + if [ -z "$OS" ]; then + install_pip distro + OS=$(python3 $HELPER detect-profile) + fi + echo "Using OS profile $OS to setup" + fi + if [ -n "$OS" ];then + HELPER_ARGS="$HELPER_ARGS --os $OS" + fi + if [ -n "$DEBUG" ]; then + set -x + HELPER_ARGS="$HELPER_ARGS --debug" + fi +} + +#needed for arguments for some commands +detect_os "$@" + #if interactive install build deps and prepare environment if [ -t 2 ]; then - OS=$(python3 ./contrib/ci/detect_os_profile.py) case $OS in debian|ubuntu|arch|fedora) - setup_deps $OS + setup_deps setup_run_dev ;; void) - setup_deps $OS + setup_deps ;; esac check_markdown