diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b93d91375..983fa0468 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,7 +31,7 @@ jobs: - name: Refresh dependencies run: sudo apt update - name: Install dependencies - run: ./contrib/ci/generate_dependencies.py | sudo xargs apt install -y + run: sudo ./contrib/ci/fwupd_setup_helpers.py install-dependencies -o ubuntu --yes - name: Check ABI run: ./contrib/ci/check-abi $(git describe --abbrev=0 --tags) $(git rev-parse HEAD) diff --git a/contrib/ci/README.md b/contrib/ci/README.md index 76140f9e3..f0a9e891a 100644 --- a/contrib/ci/README.md +++ b/contrib/ci/README.md @@ -99,7 +99,7 @@ Each distribution will have `package` elements and `control` elements. * `inclusive` elements represent an inclusive list of architectures to be installed on * `exclusive` elements represent an exclusive list of architectures to not be installed on -For convenience there is also a standalone script __generate_dependencies.py__ that parses `dependencies.xml`. +For convenience there is also a helper script `./contrib/ci/fwupd_setup_helpers.p install-dependencies` that parses `dependencies.xml`. ### Dockerfile.in diff --git a/contrib/ci/arch.sh b/contrib/ci/arch.sh index d479f81d6..f196616dd 100755 --- a/contrib/ci/arch.sh +++ b/contrib/ci/arch.sh @@ -7,7 +7,7 @@ shopt -s extglob . ./contrib/ci/get_test_firmware.sh #install anything missing from the container -./contrib/ci/generate_dependencies.py | xargs pacman -S --noconfirm --needed +./contrib/ci/fwupd_setup_helpers.py install-dependencies -o arch # prepare the build tree rm -rf build diff --git a/contrib/ci/debian.sh b/contrib/ci/debian.sh index d8615def2..014dbdf02 100755 --- a/contrib/ci/debian.sh +++ b/contrib/ci/debian.sh @@ -36,7 +36,7 @@ sed s/quilt/native/ debian/source/format -i #check if we have all deps available #if some are missing, we're going to use subproject instead and #packaging CI will fail -./contrib/ci/generate_dependencies.py | xargs apt install -y || true +./contrib/ci/fwupd_setup_helpers.py install-dependencies -o debian --yes || true if ! dpkg-checkbuilddeps; then ./contrib/ci/ubuntu.sh exit 0 diff --git a/contrib/ci/fedora.sh b/contrib/ci/fedora.sh index 24811aa48..5e822aee7 100755 --- a/contrib/ci/fedora.sh +++ b/contrib/ci/fedora.sh @@ -3,7 +3,7 @@ set -e set -x #get any missing deps from the container -./contrib/ci/generate_dependencies.py | xargs dnf install -y +./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o fedora #generate a tarball git config tar.tar.xz.command "xz -c" diff --git a/contrib/ci/fwupd_setup_helpers.py b/contrib/ci/fwupd_setup_helpers.py new file mode 100755 index 000000000..32bf4f8a7 --- /dev/null +++ b/contrib/ci/fwupd_setup_helpers.py @@ -0,0 +1,165 @@ +#!/usr/bin/python3 +# +# Copyright (C) 2017 Dell, Inc. +# Copyright (C) 2020 Intel, Inc. +# Copyright (C) 2021 Mario Limonciello +# +# SPDX-License-Identifier: LGPL-2.1+ +# +import os +import sys +import argparse + +# Minimum version of markdown required +MINIMUM_MARKDOWN = (3, 3, 3) + + +def get_possible_profiles(): + return ["fedora", "centos", "debian", "ubuntu", "arch", "void"] + + +def detect_profile(): + try: + import distro + + target = distro.id() + if not target in get_possible_profiles(): + target = distro.like() + except ModuleNotFoundError: + target = "" + return target + + +def test_markdown(): + import markdown + + new_enough = markdown.__version_info__ >= MINIMUM_MARKDOWN + if not new_enough: + print("python3-markdown must be upgraded") + sys.exit(not new_enough) + + +def parse_dependencies(OS, SUBOS, requested_type): + import xml.etree.ElementTree as etree + + deps = [] + dep = "" + directory = os.path.dirname(sys.argv[0]) + tree = etree.parse(os.path.join(directory, "dependencies.xml")) + root = tree.getroot() + for child in root: + if "type" not in child.attrib or "id" not in child.attrib: + continue + for distro in child: + if "id" not in distro.attrib: + continue + if distro.attrib["id"] != OS: + continue + packages = distro.findall("package") + for package in packages: + if SUBOS: + if "variant" not in package.attrib: + continue + if package.attrib["variant"] != SUBOS: + continue + if package.text: + dep = package.text + else: + dep = child.attrib["id"] + if child.attrib["type"] == requested_type and dep: + deps.append(dep) + return deps + + +def get_build_dependencies(os, variant): + return parse_dependencies(os, variant, "build") + + +def _get_installer_cmd(os, yes): + if os == "debian" or os == "ubuntu": + installer = ["apt", "install"] + elif os == "fedora": + installer = ["dnf", "install"] + elif os == "arch": + installer = ["pacman", "-Syu", "--noconfirm", "--needed"] + elif os == "void": + installer = ["xbps-install", "-Syu"] + else: + print("unable to detect OS profile, use --os= to specify") + print("\tsupported profiles: %s" % get_possible_profiles()) + sys.exit(1) + if yes: + installer += ["-y"] + return installer + + +def install_packages(os, variant, yes, debugging, packages): + import subprocess + + if packages == "build-dependencies": + packages = get_build_dependencies(os, variant) + installer = _get_installer_cmd(os, yes) + installer += packages + if debugging: + print(installer) + subprocess.call(installer) + + +if __name__ == "__main__": + + command = None + # compat mode for old training documentation + if "generate_dependencies.py" in sys.argv[0]: + command = "get-dependencies" + + parser = argparse.ArgumentParser() + if not command: + parser.add_argument( + "command", + choices=[ + "get-dependencies", + "test-markdown", + "detect-profile", + "install-dependencies", + "install-pip", + ], + help="command to run", + ) + parser.add_argument( + "-o", + "--os", + default=detect_profile(), + choices=get_possible_profiles(), + help="calculate dependencies for OS profile", + ) + parser.add_argument( + "-v", "--variant", help="optional machine variant for the OS profile" + ) + parser.add_argument( + "-y", "--yes", action="store_true", help="Don't prompt to install" + ) + parser.add_argument( + "-d", "--debug", action="store_true", help="Display all launched commands" + ) + args = parser.parse_args() + + # these require variants to be set + if not args.variant and (args.os == "ubuntu" or args.os == "debian"): + args.variant = os.uname().machine + if not command: + command = args.command + + # command to run + if command == "test-markdown": + test_markdown() + elif command == "detect-profile": + print(detect_profile()) + elif command == "get-dependencies": + dependencies = get_build_dependencies(args.os, args.variant) + print(*dependencies, sep="\n") + elif command == "install-dependencies": + install_packages( + args.os, args.variant, args.yes, args.debug, "build-dependencies" + ) + elif command == "install-pip": + install_packages(args.os, args.variant, args.yes, args.debug, ["python3-pip"]) diff --git a/contrib/ci/generate_dependencies.py b/contrib/ci/generate_dependencies.py deleted file mode 100755 index 4aa99d2b4..000000000 --- a/contrib/ci/generate_dependencies.py +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/python3 -# -# Copyright (C) 2017 Dell, Inc. -# Copyright (C) 2020 Intel, Inc. -# -# SPDX-License-Identifier: LGPL-2.1+ -# -import os -import sys -import argparse -import xml.etree.ElementTree as etree - - -def parse_dependencies(OS, SUBOS, requested_type): - deps = [] - dep = "" - directory = os.path.dirname(sys.argv[0]) - tree = etree.parse(os.path.join(directory, "dependencies.xml")) - root = tree.getroot() - for child in root: - if "type" not in child.attrib or "id" not in child.attrib: - continue - for distro in child: - if "id" not in distro.attrib: - continue - if distro.attrib["id"] != OS: - continue - packages = distro.findall("package") - for package in packages: - if SUBOS: - if "variant" not in package.attrib: - continue - if package.attrib["variant"] != SUBOS: - continue - if package.text: - dep = package.text - else: - dep = child.attrib["id"] - if child.attrib["type"] == requested_type and dep: - deps.append(dep) - return deps - - -if __name__ == "__main__": - possible_targets = ["fedora", "centos", "debian", "ubuntu", "arch"] - - try: - import distro - - target = distro.id() - if not target in possible_targets: - target = distro.like() - except ModuleNotFoundError: - target = None - - parser = argparse.ArgumentParser() - parser.add_argument( - "-o", - "--os", - default=target, - choices=possible_targets, - help="dependencies for OS", - ) - args = parser.parse_args() - - # these require variants to be set - if args.os == "ubuntu" or args.os == "debian": - args.os = "%s-%s" % (args.os, os.uname().machine) - - target = os.getenv("OS", args.os) - if target is None: - print("Missing OS environment variable") - sys.exit(1) - - _os = target.lower() - _sub_os = "" - split = target.split("-") - if len(split) >= 2: - _os, _sub_os = split[:2] - dependencies = parse_dependencies(_os, _sub_os, "build") - print(*dependencies, sep="\n") diff --git a/contrib/ci/generate_dependencies.py b/contrib/ci/generate_dependencies.py new file mode 120000 index 000000000..cc1f5adae --- /dev/null +++ b/contrib/ci/generate_dependencies.py @@ -0,0 +1 @@ +fwupd_setup_helpers.py \ No newline at end of file diff --git a/contrib/ci/generate_docker.py b/contrib/ci/generate_docker.py index 7390cb661..9640ab1c0 100755 --- a/contrib/ci/generate_docker.py +++ b/contrib/ci/generate_docker.py @@ -8,7 +8,7 @@ import os import subprocess import sys import shutil -from generate_dependencies import parse_dependencies +from fwupd_setup_helpers import parse_dependencies def get_container_cmd(): diff --git a/contrib/ci/ubuntu.sh b/contrib/ci/ubuntu.sh index 2a7c01ec2..1688ace83 100755 --- a/contrib/ci/ubuntu.sh +++ b/contrib/ci/ubuntu.sh @@ -6,7 +6,7 @@ set -x . ./contrib/ci/get_test_firmware.sh #check for and install missing dependencies -./contrib/ci/generate_dependencies.py | xargs apt install -y +./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o ubuntu #evaluate using Ubuntu's buildflags #evaluate using Debian/Ubuntu's buildflags diff --git a/contrib/ci/void.sh b/contrib/ci/void.sh index 81739897a..9aa03786f 100755 --- a/contrib/ci/void.sh +++ b/contrib/ci/void.sh @@ -4,7 +4,7 @@ set -x #install dependencies xbps-install -Suy python3 -./contrib/ci/generate_dependencies.py | xargs xbps-install -y +./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o void #clone test firmware if necessary . ./contrib/ci/get_test_firmware.sh diff --git a/contrib/setup b/contrib/setup index 0abef17c2..1c4bd36d3 100755 --- a/contrib/setup +++ b/contrib/setup @@ -3,21 +3,14 @@ cd "$(dirname "$0")/.." +HELPER=./contrib/ci/fwupd_setup_helpers.py +HELPER_ARGS="-y" + setup_deps() { - OS=$1 - read -p "Install build dependencies? (y/n) " question + read -p "Install build dependencies? (y/n) " question if [ "$question" = "y" ]; then - DEPS=$(./contrib/ci/generate_dependencies.py) - if [ "$OS" = "debian" ] || [ "$OS" = "ubuntu" ]; then - sudo apt install $DEPS - elif [ "$OS" = "fedora" ]; then - sudo dnf install $DEPS - elif [ "$OS" = "arch" ]; then - pacman -Syu --noconfirm --needed $DEPS - elif [ "$OS" = "void" ]; then - xbps-install -Syu $DEPS - fi + $(which sudo) python3 $HELPER install-dependencies $HELPER_ARGS -y fi } @@ -47,34 +40,79 @@ setup_git() git config include.path ../.gitconfig } +install_pip() +{ + package=$1 + args=$2 + if ! python3 -m pip install $package $args; then + $(which sudo) python3 $HELPER install-pip $HELPER_ARGS -y + fi + #try once more + python3 -m pip install $package +} + setup_precommit() { echo "Configuring pre-commit hooks" python3 -m venv venv source venv/bin/activate - python3 -m pip install pre-commit + install_pip pre-commit pre-commit install } check_markdown() { - if python3 -c "import markdown; import sys; sys.exit(markdown.__version_info__ >= (3,3,3))"; then - echo "Upgrading python3-markdown version" - python3 -m pip install markdown --upgrade + if ! python3 $HELPER test-markdown; then + install_pip markdown --upgrade fi } +detect_os() +{ + for i in "$@"; do + case $i in + --os=*) + OS="${i#*=}" + shift + ;; + --debug) + DEBUG=1 + shift + ;; + *) + ;; + esac + done + if [ -z $OS ]; then + OS=$(python3 $HELPER detect-profile) + if [ -z "$OS" ]; then + install_pip distro + OS=$(python3 $HELPER detect-profile) + fi + echo "Using OS profile $OS to setup" + fi + if [ -n "$OS" ];then + HELPER_ARGS="$HELPER_ARGS --os $OS" + fi + if [ -n "$DEBUG" ]; then + set -x + HELPER_ARGS="$HELPER_ARGS --debug" + fi +} + +#needed for arguments for some commands +detect_os "$@" + #if interactive install build deps and prepare environment if [ -t 2 ]; then - OS=$(python3 ./contrib/ci/detect_os_profile.py) case $OS in debian|ubuntu|arch|fedora) - setup_deps $OS + setup_deps setup_run_dev ;; void) - setup_deps $OS + setup_deps ;; esac check_markdown