trivial: Merge python steps from contrib/setup into helper script

This avoids having to hardcode profile targets in multiple places
and also fixes the confusing entry points into scripts both by
arguments and environment variables.

It also makes the setup script a lot more debuggable and scalable.

OS detection is a lot more robust, where it will try to use pip to
set up the distro python package, and if pip is missing try to install
it.

If OS detection fails now, a user can use --os on contrib/setup for
specifying it.

.github/workflows/main.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Refresh dependencies
         run: sudo apt update
       - name: Install dependencies
-        run: ./contrib/ci/generate_dependencies.py | sudo xargs apt install -y
+        run: sudo ./contrib/ci/fwupd_setup_helpers.py install-dependencies -o ubuntu --yes
       - name: Check ABI
         run: ./contrib/ci/check-abi $(git describe --abbrev=0 --tags) $(git rev-parse HEAD)
 

contrib/ci/README.md

@@ -99,7 +99,7 @@ Each distribution will have `package` elements and `control` elements.
 * `inclusive` elements represent an inclusive list of architectures to be installed on
 * `exclusive` elements represent an exclusive list of architectures to not be installed on
 
-For convenience there is also a standalone script __generate_dependencies.py__ that parses `dependencies.xml`.
+For convenience there is also a helper script `./contrib/ci/fwupd_setup_helpers.p install-dependencies` that parses `dependencies.xml`.
 
 ### Dockerfile.in
 

contrib/ci/arch.sh

@@ -7,7 +7,7 @@ shopt -s extglob
 . ./contrib/ci/get_test_firmware.sh
 
 #install anything missing from the container
-./contrib/ci/generate_dependencies.py | xargs pacman -S --noconfirm --needed
+./contrib/ci/fwupd_setup_helpers.py install-dependencies -o arch
 
 # prepare the build tree
 rm -rf build

contrib/ci/debian.sh

@@ -36,7 +36,7 @@ sed s/quilt/native/ debian/source/format -i
 #check if we have all deps available
 #if some are missing, we're going to use subproject instead and
 #packaging CI will fail
-./contrib/ci/generate_dependencies.py  | xargs apt install -y || true
+./contrib/ci/fwupd_setup_helpers.py install-dependencies -o debian --yes || true
 if ! dpkg-checkbuilddeps; then
 	./contrib/ci/ubuntu.sh
 	exit 0

contrib/ci/fedora.sh

@@ -3,7 +3,7 @@ set -e
 set -x
 
 #get any missing deps from the container
-./contrib/ci/generate_dependencies.py | xargs dnf install -y
+./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o fedora
 
 #generate a tarball
 git config tar.tar.xz.command "xz -c"

contrib/ci/fwupd_setup_helpers.py

@@ -0,0 +1,165 @@
+#!/usr/bin/python3
+#
+# Copyright (C) 2017 Dell, Inc.
+# Copyright (C) 2020 Intel, Inc.
+# Copyright (C) 2021 Mario Limonciello
+#
+# SPDX-License-Identifier: LGPL-2.1+
+#
+import os
+import sys
+import argparse
+
+# Minimum version of markdown required
+MINIMUM_MARKDOWN = (3, 3, 3)
+
+
+def get_possible_profiles():
+    return ["fedora", "centos", "debian", "ubuntu", "arch", "void"]
+
+
+def detect_profile():
+    try:
+        import distro
+
+        target = distro.id()
+        if not target in get_possible_profiles():
+            target = distro.like()
+    except ModuleNotFoundError:
+        target = ""
+    return target
+
+
+def test_markdown():
+    import markdown
+
+    new_enough = markdown.__version_info__ >= MINIMUM_MARKDOWN
+    if not new_enough:
+        print("python3-markdown must be upgraded")
+    sys.exit(not new_enough)
+
+
+def parse_dependencies(OS, SUBOS, requested_type):
+    import xml.etree.ElementTree as etree
+
+    deps = []
+    dep = ""
+    directory = os.path.dirname(sys.argv[0])
+    tree = etree.parse(os.path.join(directory, "dependencies.xml"))
+    root = tree.getroot()
+    for child in root:
+        if "type" not in child.attrib or "id" not in child.attrib:
+            continue
+        for distro in child:
+            if "id" not in distro.attrib:
+                continue
+            if distro.attrib["id"] != OS:
+                continue
+            packages = distro.findall("package")
+            for package in packages:
+                if SUBOS:
+                    if "variant" not in package.attrib:
+                        continue
+                    if package.attrib["variant"] != SUBOS:
+                        continue
+                if package.text:
+                    dep = package.text
+                else:
+                    dep = child.attrib["id"]
+                if child.attrib["type"] == requested_type and dep:
+                    deps.append(dep)
+    return deps
+
+
+def get_build_dependencies(os, variant):
+    return parse_dependencies(os, variant, "build")
+
+
+def _get_installer_cmd(os, yes):
+    if os == "debian" or os == "ubuntu":
+        installer = ["apt", "install"]
+    elif os == "fedora":
+        installer = ["dnf", "install"]
+    elif os == "arch":
+        installer = ["pacman", "-Syu", "--noconfirm", "--needed"]
+    elif os == "void":
+        installer = ["xbps-install", "-Syu"]
+    else:
+        print("unable to detect OS profile, use --os= to specify")
+        print("\tsupported profiles: %s" % get_possible_profiles())
+        sys.exit(1)
+    if yes:
+        installer += ["-y"]
+    return installer
+
+
+def install_packages(os, variant, yes, debugging, packages):
+    import subprocess
+
+    if packages == "build-dependencies":
+        packages = get_build_dependencies(os, variant)
+    installer = _get_installer_cmd(os, yes)
+    installer += packages
+    if debugging:
+        print(installer)
+    subprocess.call(installer)
+
+
+if __name__ == "__main__":
+
+    command = None
+    # compat mode for old training documentation
+    if "generate_dependencies.py" in sys.argv[0]:
+        command = "get-dependencies"
+
+    parser = argparse.ArgumentParser()
+    if not command:
+        parser.add_argument(
+            "command",
+            choices=[
+                "get-dependencies",
+                "test-markdown",
+                "detect-profile",
+                "install-dependencies",
+                "install-pip",
+            ],
+            help="command to run",
+        )
+    parser.add_argument(
+        "-o",
+        "--os",
+        default=detect_profile(),
+        choices=get_possible_profiles(),
+        help="calculate dependencies for OS profile",
+    )
+    parser.add_argument(
+        "-v", "--variant", help="optional machine variant for the OS profile"
+    )
+    parser.add_argument(
+        "-y", "--yes", action="store_true", help="Don't prompt to install"
+    )
+    parser.add_argument(
+        "-d", "--debug", action="store_true", help="Display all launched commands"
+    )
+    args = parser.parse_args()
+
+    # these require variants to be set
+    if not args.variant and (args.os == "ubuntu" or args.os == "debian"):
+        args.variant = os.uname().machine
+    if not command:
+        command = args.command
+
+    # command to run
+    if command == "test-markdown":
+        test_markdown()
+    elif command == "detect-profile":
+        print(detect_profile())
+    elif command == "get-dependencies":
+        dependencies = get_build_dependencies(args.os, args.variant)
+        print(*dependencies, sep="\n")
+    elif command == "install-dependencies":
+        install_packages(
+            args.os, args.variant, args.yes, args.debug, "build-dependencies"
+        )
+    elif command == "install-pip":
+        install_packages(args.os, args.variant, args.yes, args.debug, ["python3-pip"])

contrib/ci/generate_dependencies.py

@@ -1,81 +0,0 @@
-#!/usr/bin/python3
-#
-# Copyright (C) 2017 Dell, Inc.
-# Copyright (C) 2020 Intel, Inc.
-#
-# SPDX-License-Identifier: LGPL-2.1+
-#
-import os
-import sys
-import argparse
-import xml.etree.ElementTree as etree
-
-
-def parse_dependencies(OS, SUBOS, requested_type):
-    deps = []
-    dep = ""
-    directory = os.path.dirname(sys.argv[0])
-    tree = etree.parse(os.path.join(directory, "dependencies.xml"))
-    root = tree.getroot()
-    for child in root:
-        if "type" not in child.attrib or "id" not in child.attrib:
-            continue
-        for distro in child:
-            if "id" not in distro.attrib:
-                continue
-            if distro.attrib["id"] != OS:
-                continue
-            packages = distro.findall("package")
-            for package in packages:
-                if SUBOS:
-                    if "variant" not in package.attrib:
-                        continue
-                    if package.attrib["variant"] != SUBOS:
-                        continue
-                if package.text:
-                    dep = package.text
-                else:
-                    dep = child.attrib["id"]
-                if child.attrib["type"] == requested_type and dep:
-                    deps.append(dep)
-    return deps
-
-
-if __name__ == "__main__":
-    possible_targets = ["fedora", "centos", "debian", "ubuntu", "arch"]
-
-    try:
-        import distro
-
-        target = distro.id()
-        if not target in possible_targets:
-            target = distro.like()
-    except ModuleNotFoundError:
-        target = None
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-o",
-        "--os",
-        default=target,
-        choices=possible_targets,
-        help="dependencies for OS",
-    )
-    args = parser.parse_args()
-
-    # these require variants to be set
-    if args.os == "ubuntu" or args.os == "debian":
-        args.os = "%s-%s" % (args.os, os.uname().machine)
-
-    target = os.getenv("OS", args.os)
-    if target is None:
-        print("Missing OS environment variable")
-        sys.exit(1)
-
-    _os = target.lower()
-    _sub_os = ""
-    split = target.split("-")
-    if len(split) >= 2:
-        _os, _sub_os = split[:2]
-    dependencies = parse_dependencies(_os, _sub_os, "build")
-    print(*dependencies, sep="\n")

contrib/ci/generate_dependencies.py

@@ -0,0 +1 @@
+fwupd_setup_helpers.py

contrib/ci/generate_docker.py

@@ -8,7 +8,7 @@ import os
 import subprocess
 import sys
 import shutil
-from generate_dependencies import parse_dependencies
+from fwupd_setup_helpers import parse_dependencies
 
 
 def get_container_cmd():

contrib/ci/ubuntu.sh

@@ -6,7 +6,7 @@ set -x
 . ./contrib/ci/get_test_firmware.sh
 
 #check for and install missing dependencies
-./contrib/ci/generate_dependencies.py | xargs apt install -y
+./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o ubuntu
 
 #evaluate using Ubuntu's buildflags
 #evaluate using Debian/Ubuntu's buildflags

contrib/ci/void.sh

@@ -4,7 +4,7 @@ set -x
 
 #install dependencies
 xbps-install -Suy python3
-./contrib/ci/generate_dependencies.py | xargs xbps-install -y
+./contrib/ci/fwupd_setup_helpers.py install-dependencies --yes -o void
 
 #clone test firmware if necessary
 . ./contrib/ci/get_test_firmware.sh

contrib/setup

@@ -3,21 +3,14 @@
 
 cd "$(dirname "$0")/.."
 
+HELPER=./contrib/ci/fwupd_setup_helpers.py
+HELPER_ARGS="-y"
+
 setup_deps()
 {
-    OS=$1
-    read -p  "Install build dependencies? (y/n) " question
+    read -p "Install build dependencies? (y/n) " question
     if [ "$question" = "y" ]; then
-        DEPS=$(./contrib/ci/generate_dependencies.py)
-        if [ "$OS" = "debian" ] || [ "$OS" = "ubuntu" ]; then
-            sudo apt install $DEPS
-        elif [ "$OS" = "fedora" ]; then
-            sudo dnf install $DEPS
-        elif [ "$OS" = "arch" ]; then
-            pacman -Syu --noconfirm --needed $DEPS
-        elif [ "$OS" = "void" ]; then
-            xbps-install -Syu $DEPS
-        fi
+        $(which sudo) python3 $HELPER install-dependencies $HELPER_ARGS -y
     fi
 }
 
@@ -47,34 +40,79 @@ setup_git()
     git config include.path ../.gitconfig
 }
 
+install_pip()
+{
+    package=$1
+    args=$2
+    if ! python3 -m pip install $package $args; then
+        $(which sudo) python3 $HELPER install-pip $HELPER_ARGS -y
+    fi
+    #try once more
+    python3 -m pip install $package
+}
+
 setup_precommit()
 {
     echo "Configuring pre-commit hooks"
     python3 -m venv venv
     source venv/bin/activate
 
-    python3 -m pip install pre-commit
+    install_pip pre-commit
     pre-commit install
 }
 
 check_markdown()
 {
-    if python3 -c "import markdown; import sys; sys.exit(markdown.__version_info__ >= (3,3,3))"; then
-        echo "Upgrading python3-markdown version"
-        python3 -m pip install markdown --upgrade
+    if ! python3 $HELPER test-markdown; then
+        install_pip markdown --upgrade
     fi
 }
 
+detect_os()
+{
+    for i in "$@"; do
+        case $i in
+            --os=*)
+                OS="${i#*=}"
+                shift
+                ;;
+            --debug)
+                DEBUG=1
+                shift
+                ;;
+            *)
+                ;;
+        esac
+    done
+    if [ -z $OS ]; then
+        OS=$(python3 $HELPER detect-profile)
+        if [ -z "$OS" ]; then
+            install_pip distro
+            OS=$(python3 $HELPER detect-profile)
+        fi
+        echo "Using OS profile $OS to setup"
+    fi
+    if [ -n "$OS" ];then
+        HELPER_ARGS="$HELPER_ARGS --os $OS"
+    fi
+    if [ -n "$DEBUG" ]; then
+        set -x
+        HELPER_ARGS="$HELPER_ARGS --debug"
+    fi
+}
+
+#needed for arguments for some commands
+detect_os "$@"
+
 #if interactive install build deps and prepare environment
 if [ -t 2 ]; then
-    OS=$(python3 ./contrib/ci/detect_os_profile.py)
     case $OS in
         debian|ubuntu|arch|fedora)
-            setup_deps $OS
+            setup_deps
             setup_run_dev
             ;;
         void)
-            setup_deps $OS
+            setup_deps
             ;;
     esac
     check_markdown