initial commit

Signed-off-by: Fabian Ebner <f.ebner@proxmox.com>

.cargo/config

@@ -0,0 +1,5 @@
+[source]
+[source.debian-packages]
+directory = "/usr/share/cargo/registry"
+[source.crates-io]
+replace-with = "debian-packages"

.gitignore

@@ -0,0 +1,4 @@
+Cargo.lock
+target/
+tests/sources.list.d.actual
+tests/sources.list.d.digest

Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "proxmox-apt"
+version = "0.1.0"
+authors = [
+    "Fabian Ebner <f.ebner@proxmox.com>",
+    "Proxmox Support Team <support@proxmox.com>",
+]
+edition = "2018"
+license = "AGPL-3"
+description = "Proxmox library for APT"
+homepage = "https://www.proxmox.com"
+
+exclude = [ "debian" ]
+
+[lib]
+name = "proxmox_apt"
+path = "src/lib.rs"
+
+[dependencies]
+anyhow = "1.0"
+openssl = "0.10"
+proxmox = { version = "0.11.5", features = [ "api-macro" ] }
+serde = { version = "1.0", features = ["derive"] }

rustfmt.toml

@@ -0,0 +1 @@
+edition = "2018"

src/lib.rs

@@ -0,0 +1 @@
+pub mod repositories;

src/repositories/file.rs

@@ -0,0 +1,274 @@
+use std::convert::TryFrom;
+use std::fmt::Display;
+use std::path::{Path, PathBuf};
+
+use anyhow::{format_err, Error};
+use serde::{Deserialize, Serialize};
+
+use crate::repositories::repository::{APTRepository, APTRepositoryFileType};
+
+use proxmox::api::api;
+
+mod list_parser;
+use list_parser::APTListFileParser;
+
+mod sources_parser;
+use sources_parser::APTSourcesFileParser;
+
+trait APTRepositoryParser {
+    /// Parse all repositories including the disabled ones and push them onto
+    /// the provided vector.
+    fn parse_repositories(&mut self) -> Result<Vec<APTRepository>, Error>;
+}
+
+#[api(
+    properties: {
+        "file-type": {
+            type: APTRepositoryFileType,
+        },
+        repositories: {
+            description: "List of APT repositories.",
+            type: Array,
+            items: {
+                type: APTRepository,
+            },
+        },
+        digest: {
+            description: "Digest for the content of the file.",
+            optional: true,
+            type: Array,
+            items: {
+                description: "Digest byte.",
+                type: Integer,
+            },
+        },
+    },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Represents an abstract APT repository file.
+pub struct APTRepositoryFile {
+    /// The path to the file.
+    pub path: String,
+
+    /// The type of the file.
+    pub file_type: APTRepositoryFileType,
+
+    /// List of repositories in the file.
+    pub repositories: Vec<APTRepository>,
+
+    /// Digest of the original contents.
+    pub digest: Option<[u8; 32]>,
+}
+
+#[api]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "kebab-case")]
+/// Error type for problems with APT repository files.
+pub struct APTRepositoryFileError {
+    /// The path to the problematic file.
+    pub path: String,
+
+    /// The error message.
+    pub error: String,
+}
+
+impl Display for APTRepositoryFileError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "proxmox-apt error for '{}' - {}", self.path, self.error)
+    }
+}
+
+impl std::error::Error for APTRepositoryFileError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        None
+    }
+}
+
+impl APTRepositoryFile {
+    /// Creates a new `APTRepositoryFile` without parsing.
+    ///
+    /// If the file is hidden, the path points to a directory, or the extension
+    /// is usually ignored by APT (e.g. `.orig`), `Ok(None)` is returned, while
+    /// invalid file names yield an error.
+    pub fn new<P: AsRef<Path>>(path: P) -> Result<Option<Self>, APTRepositoryFileError> {
+        let path: PathBuf = path.as_ref().to_path_buf();
+
+        let new_err = |path_string: String, err: &str| APTRepositoryFileError {
+            path: path_string,
+            error: err.to_string(),
+        };
+
+        let path_string = path
+            .clone()
+            .into_os_string()
+            .into_string()
+            .map_err(|os_string| {
+                new_err(
+                    os_string.to_string_lossy().to_string(),
+                    "path is not valid unicode",
+                )
+            })?;
+
+        let new_err = |err| new_err(path_string.clone(), err);
+
+        if path.is_dir() {
+            return Ok(None);
+        }
+
+        let file_name = match path.file_name() {
+            Some(file_name) => file_name
+                .to_os_string()
+                .into_string()
+                .map_err(|_| new_err("invalid path"))?,
+            None => return Err(new_err("invalid path")),
+        };
+
+        if file_name.starts_with('.') || file_name.ends_with('~') {
+            return Ok(None);
+        }
+
+        let extension = match path.extension() {
+            Some(extension) => extension
+                .to_os_string()
+                .into_string()
+                .map_err(|_| new_err("invalid path"))?,
+            None => return Err(new_err("invalid extension")),
+        };
+
+        // See APT's apt-pkg/init.cc
+        if extension.starts_with("dpkg-")
+            || extension.starts_with("ucf-")
+            || matches!(
+                extension.as_str(),
+                "disabled" | "bak" | "save" | "orig" | "distUpgrade"
+            )
+        {
+            return Ok(None);
+        }
+
+        let file_type = APTRepositoryFileType::try_from(&extension[..])
+            .map_err(|_| new_err("invalid extension"))?;
+
+        if !file_name
+            .chars()
+            .all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '-' || x == '.')
+        {
+            return Err(new_err("invalid characters in file name"));
+        }
+
+        Ok(Some(Self {
+            path: path_string,
+            file_type,
+            repositories: vec![],
+            digest: None,
+        }))
+    }
+
+    /// Check if the file exists.
+    pub fn exists(&self) -> bool {
+        PathBuf::from(&self.path).exists()
+    }
+
+    pub fn read_with_digest(&self) -> Result<(Vec<u8>, [u8; 32]), APTRepositoryFileError> {
+        let content = std::fs::read(&self.path).map_err(|err| self.err(format_err!("{}", err)))?;
+
+        let digest = openssl::sha::sha256(&content);
+
+        Ok((content, digest))
+    }
+
+    /// Create an `APTRepositoryFileError`.
+    pub fn err(&self, error: Error) -> APTRepositoryFileError {
+        APTRepositoryFileError {
+            path: self.path.clone(),
+            error: error.to_string(),
+        }
+    }
+
+    /// Parses the APT repositories configured in the file on disk, including
+    /// disabled ones.
+    ///
+    /// Resets the current repositories and digest, even on failure.
+    pub fn parse(&mut self) -> Result<(), APTRepositoryFileError> {
+        self.repositories.clear();
+        self.digest = None;
+
+        let (content, digest) = self.read_with_digest()?;
+
+        let mut parser: Box<dyn APTRepositoryParser> = match self.file_type {
+            APTRepositoryFileType::List => Box::new(APTListFileParser::new(&content[..])),
+            APTRepositoryFileType::Sources => Box::new(APTSourcesFileParser::new(&content[..])),
+        };
+
+        let repos = parser.parse_repositories().map_err(|err| self.err(err))?;
+
+        for (n, repo) in repos.iter().enumerate() {
+            repo.basic_check()
+                .map_err(|err| self.err(format_err!("check for repository {} - {}", n + 1, err)))?;
+        }
+
+        self.repositories = repos;
+        self.digest = Some(digest);
+
+        Ok(())
+    }
+
+    /// Writes the repositories to the file on disk.
+    ///
+    /// If a digest is provided, checks that the current content of the file still
+    /// produces the same one.
+    pub fn write(&self) -> Result<(), APTRepositoryFileError> {
+        if let Some(digest) = self.digest {
+            if !self.exists() {
+                return Err(self.err(format_err!("digest specified, but file does not exist")));
+            }
+
+            let (_, current_digest) = self.read_with_digest()?;
+            if digest != current_digest {
+                return Err(self.err(format_err!("digest mismatch")));
+            }
+        }
+
+        if self.repositories.is_empty() {
+            return std::fs::remove_file(&self.path)
+                .map_err(|err| self.err(format_err!("unable to remove file - {}", err)));
+        }
+
+        let mut content = vec![];
+
+        for (n, repo) in self.repositories.iter().enumerate() {
+            repo.basic_check()
+                .map_err(|err| self.err(format_err!("check for repository {} - {}", n + 1, err)))?;
+
+            repo.write(&mut content)
+                .map_err(|err| self.err(format_err!("writing repository {} - {}", n + 1, err)))?;
+        }
+
+        let path = PathBuf::from(&self.path);
+        let dir = match path.parent() {
+            Some(dir) => dir,
+            None => return Err(self.err(format_err!("invalid path"))),
+        };
+
+        std::fs::create_dir_all(dir)
+            .map_err(|err| self.err(format_err!("unable to create parent dir - {}", err)))?;
+
+        let pid = std::process::id();
+        let mut tmp_path = path.clone();
+        tmp_path.set_extension("tmp");
+        tmp_path.set_extension(format!("{}", pid));
+
+        if let Err(err) = std::fs::write(&tmp_path, content) {
+            let _ = std::fs::remove_file(&tmp_path);
+            return Err(self.err(format_err!("writing {:?} failed - {}", path, err)));
+        }
+
+        if let Err(err) = std::fs::rename(&tmp_path, &path) {
+            let _ = std::fs::remove_file(&tmp_path);
+            return Err(self.err(format_err!("rename failed for {:?} - {}", path, err)));
+        }
+
+        Ok(())
+    }
+}

src/repositories/file/list_parser.rs

@@ -0,0 +1,172 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::{Iterator, Peekable};
+use std::str::SplitAsciiWhitespace;
+
+use anyhow::{bail, format_err, Error};
+
+use crate::repositories::{APTRepository, APTRepositoryFileType, APTRepositoryOption};
+
+use super::APTRepositoryParser;
+
+pub struct APTListFileParser<R: BufRead> {
+    input: R,
+    line_nr: usize,
+    comment: String,
+}
+
+impl<R: BufRead> APTListFileParser<R> {
+    pub fn new(reader: R) -> Self {
+        Self {
+            input: reader,
+            line_nr: 0,
+            comment: String::new(),
+        }
+    }
+
+    /// Helper to parse options from the existing token stream.
+    ///
+    /// Also returns `Ok(())` if there are no options.
+    ///
+    /// Errors when options are invalid or not closed by `']'`.
+    fn parse_options(
+        options: &mut Vec<APTRepositoryOption>,
+        tokens: &mut Peekable<SplitAsciiWhitespace>,
+    ) -> Result<(), Error> {
+        let mut option = match tokens.peek() {
+            Some(token) => {
+                match token.strip_prefix('[') {
+                    Some(option) => option,
+                    None => return Ok(()), // doesn't look like options
+                }
+            }
+            None => return Ok(()),
+        };
+
+        tokens.next(); // avoid reading the beginning twice
+
+        let mut finished = false;
+        loop {
+            if let Some(stripped) = option.strip_suffix(']') {
+                option = stripped;
+                if option.is_empty() {
+                    break;
+                }
+                finished = true; // but still need to handle the last one
+            };
+
+            if let Some(mid) = option.find('=') {
+                let (key, mut value_str) = option.split_at(mid);
+                value_str = &value_str[1..];
+
+                if key.is_empty() {
+                    bail!("option has no key: '{}'", option);
+                }
+
+                if value_str.is_empty() {
+                    bail!("option has no value: '{}'", option);
+                }
+
+                let values: Vec<String> = value_str
+                    .split(',')
+                    .map(|value| value.to_string())
+                    .collect();
+
+                options.push(APTRepositoryOption {
+                    key: key.to_string(),
+                    values,
+                });
+            } else if !option.is_empty() {
+                bail!("got invalid option - '{}'", option);
+            }
+
+            if finished {
+                break;
+            }
+
+            option = match tokens.next() {
+                Some(option) => option,
+                None => bail!("options not closed by ']'"),
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Parse a repository or comment in one-line format.
+    ///
+    /// Commented out repositories are also detected and returned with the
+    /// `enabled` property set to `false`.
+    ///
+    /// If the line contains a repository, `self.comment` is added to the
+    /// `comment` property.
+    ///
+    /// If the line contains a comment, it is added to `self.comment`.
+    fn parse_one_line(&mut self, mut line: &str) -> Result<Option<APTRepository>, Error> {
+        line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+        // check for commented out repository first
+        if let Some(commented_out) = line.strip_prefix('#') {
+            if let Ok(Some(mut repo)) = self.parse_one_line(commented_out) {
+                repo.set_enabled(false);
+                return Ok(Some(repo));
+            }
+        }
+
+        let mut repo = APTRepository::new(APTRepositoryFileType::List);
+
+        // now handle "real" comment
+        if let Some(comment_start) = line.find('#') {
+            let (line_start, comment) = line.split_at(comment_start);
+            self.comment = format!("{}{}\n", self.comment, &comment[1..]);
+            line = line_start;
+        }
+
+        let mut tokens = line.split_ascii_whitespace().peekable();
+
+        match tokens.next() {
+            Some(package_type) => {
+                repo.types.push(package_type.try_into()?);
+            }
+            None => return Ok(None), // empty line
+        }
+
+        Self::parse_options(&mut repo.options, &mut tokens)?;
+
+        // the rest of the line is just '<uri> <suite> [<components>...]'
+        let mut tokens = tokens.map(str::to_string);
+        repo.uris
+            .push(tokens.next().ok_or_else(|| format_err!("missing URI"))?);
+        repo.suites
+            .push(tokens.next().ok_or_else(|| format_err!("missing suite"))?);
+        repo.components.extend(tokens);
+
+        repo.comment = std::mem::take(&mut self.comment);
+
+        Ok(Some(repo))
+    }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTListFileParser<R> {
+    fn parse_repositories(&mut self) -> Result<Vec<APTRepository>, Error> {
+        let mut repos = vec![];
+        let mut line = String::new();
+
+        loop {
+            self.line_nr += 1;
+            line.clear();
+
+            match self.input.read_line(&mut line) {
+                Err(err) => bail!("input error - {}", err),
+                Ok(0) => break,
+                Ok(_) => match self.parse_one_line(&line) {
+                    Ok(Some(repo)) => repos.push(repo),
+                    Ok(None) => continue,
+                    Err(err) => bail!("malformed entry on line {} - {}", self.line_nr, err),
+                },
+            }
+        }
+
+        Ok(repos)
+    }
+}

src/repositories/file/sources_parser.rs

@@ -0,0 +1,204 @@
+use std::convert::TryInto;
+use std::io::BufRead;
+use std::iter::Iterator;
+
+use anyhow::{bail, Error};
+
+use crate::repositories::{
+    APTRepository, APTRepositoryFileType, APTRepositoryOption, APTRepositoryPackageType,
+};
+
+use super::APTRepositoryParser;
+
+pub struct APTSourcesFileParser<R: BufRead> {
+    input: R,
+    stanza_nr: usize,
+    comment: String,
+}
+
+/// See `man sources.list` and `man deb822` for the format specification.
+impl<R: BufRead> APTSourcesFileParser<R> {
+    pub fn new(reader: R) -> Self {
+        Self {
+            input: reader,
+            stanza_nr: 1,
+            comment: String::new(),
+        }
+    }
+
+    /// Based on APT's `StringToBool` in `strutl.cc`
+    fn string_to_bool(string: &str, default: bool) -> bool {
+        let string = string.trim_matches(|c| char::is_ascii_whitespace(&c));
+        let string = string.to_lowercase();
+
+        match &string[..] {
+            "1" | "yes" | "true" | "with" | "on" | "enable" => true,
+            "0" | "no" | "false" | "without" | "off" | "disable" => false,
+            _ => default,
+        }
+    }
+
+    /// Checks if `key` is valid according to deb822
+    fn valid_key(key: &str) -> bool {
+        if key.starts_with('-') {
+            return false;
+        };
+        return key.chars().all(|c| matches!(c, '!'..='9' | ';'..='~'));
+    }
+
+    /// Try parsing a repository in stanza format from `lines`.
+    ///
+    /// Returns `Ok(None)` when no stanza can be found.
+    ///
+    /// Comments are added to `self.comments`. If a stanza can be found,
+    /// `self.comment` is added to the repository's `comment` property.
+    ///
+    /// Fully commented out stanzas are treated as comments.
+    fn parse_stanza(&mut self, lines: &str) -> Result<Option<APTRepository>, Error> {
+        let mut repo = APTRepository::new(APTRepositoryFileType::Sources);
+
+        // Values may be folded into multiple lines.
+        // Those lines have to start with a space or a tab.
+        let lines = lines.replace("\n ", " ");
+        let lines = lines.replace("\n\t", " ");
+
+        let mut got_something = false;
+
+        for line in lines.lines() {
+            let line = line.trim_matches(|c| char::is_ascii_whitespace(&c));
+            if line.is_empty() {
+                continue;
+            }
+
+            if let Some(commented_out) = line.strip_prefix('#') {
+                self.comment = format!("{}{}\n", self.comment, commented_out);
+                continue;
+            }
+
+            if let Some(mid) = line.find(':') {
+                let (key, value_str) = line.split_at(mid);
+                let value_str = &value_str[1..];
+                let key = key.trim_matches(|c| char::is_ascii_whitespace(&c));
+
+                if key.is_empty() {
+                    bail!("option has no key: '{}'", line);
+                }
+
+                if value_str.is_empty() {
+                    // ignored by APT
+                    eprintln!("option has no value: '{}'", line);
+                    continue;
+                }
+
+                if !Self::valid_key(key) {
+                    // ignored by APT
+                    eprintln!("option with invalid key '{}'", key);
+                    continue;
+                }
+
+                let values: Vec<String> = value_str
+                    .split_ascii_whitespace()
+                    .map(|value| value.to_string())
+                    .collect();
+
+                match &key.to_lowercase()[..] {
+                    "types" => {
+                        if !repo.types.is_empty() {
+                            eprintln!("key 'Types' was defined twice");
+                        }
+                        let mut types = Vec::<APTRepositoryPackageType>::new();
+                        for package_type in values {
+                            types.push((&package_type[..]).try_into()?);
+                        }
+                        repo.types = types;
+                    }
+                    "uris" => {
+                        if !repo.uris.is_empty() {
+                            eprintln!("key 'URIs' was defined twice");
+                        }
+                        repo.uris = values;
+                    }
+                    "suites" => {
+                        if !repo.suites.is_empty() {
+                            eprintln!("key 'Suites' was defined twice");
+                        }
+                        repo.suites = values;
+                    }
+                    "components" => {
+                        if !repo.components.is_empty() {
+                            eprintln!("key 'Components' was defined twice");
+                        }
+                        repo.components = values;
+                    }
+                    "enabled" => {
+                        repo.set_enabled(Self::string_to_bool(value_str, true));
+                    }
+                    _ => repo.options.push(APTRepositoryOption {
+                        key: key.to_string(),
+                        values,
+                    }),
+                }
+            } else {
+                bail!("got invalid line - '{:?}'", line);
+            }
+
+            got_something = true;
+        }
+
+        if !got_something {
+            return Ok(None);
+        }
+
+        repo.comment = std::mem::take(&mut self.comment);
+
+        Ok(Some(repo))
+    }
+
+    /// Helper function for `parse_repositories`.
+    fn try_parse_stanza(
+        &mut self,
+        lines: &str,
+        repos: &mut Vec<APTRepository>,
+    ) -> Result<(), Error> {
+        match self.parse_stanza(lines) {
+            Ok(Some(repo)) => {
+                repos.push(repo);
+                self.stanza_nr += 1;
+            }
+            Ok(None) => (),
+            Err(err) => bail!("malformed entry in stanza {} - {}", self.stanza_nr, err),
+        }
+
+        Ok(())
+    }
+}
+
+impl<R: BufRead> APTRepositoryParser for APTSourcesFileParser<R> {
+    fn parse_repositories(&mut self) -> Result<Vec<APTRepository>, Error> {
+        let mut repos = vec![];
+        let mut lines = String::new();
+
+        loop {
+            let old_length = lines.len();
+            match self.input.read_line(&mut lines) {
+                Err(err) => bail!("input error - {}", err),
+                Ok(0) => {
+                    self.try_parse_stanza(&lines[..], &mut repos)?;
+                    break;
+                }
+                Ok(_) => {
+                    if (&lines[old_length..])
+                        .trim_matches(|c| char::is_ascii_whitespace(&c))
+                        .is_empty()
+                    {
+                        // detected end of stanza
+                        self.try_parse_stanza(&lines[..], &mut repos)?;
+                        lines.clear();
+                    }
+                }
+            }
+        }
+
+        Ok(repos)
+    }
+}

src/repositories/mod.rs

@@ -0,0 +1,107 @@
+use std::collections::BTreeMap;
+use std::path::PathBuf;
+
+use anyhow::{bail, Error};
+
+mod repository;
+pub use repository::{
+    APTRepository, APTRepositoryFileType, APTRepositoryOption, APTRepositoryPackageType,
+};
+
+mod file;
+pub use file::{APTRepositoryFile, APTRepositoryFileError};
+
+const APT_SOURCES_LIST_FILENAME: &str = "/etc/apt/sources.list";
+const APT_SOURCES_LIST_DIRECTORY: &str = "/etc/apt/sources.list.d/";
+
+/// Calculates a common digest for successfully parsed repository files.
+///
+/// The digest is invariant with respect to file order.
+///
+/// Files without a digest are ignored.
+fn common_digest(files: &[APTRepositoryFile]) -> [u8; 32] {
+    let mut digests = BTreeMap::new();
+
+    for file in files.iter() {
+        digests.insert(file.path.clone(), &file.digest);
+    }
+
+    let mut common_raw = Vec::<u8>::with_capacity(digests.len() * 32);
+    for digest in digests.values() {
+        match digest {
+            Some(digest) => common_raw.extend_from_slice(&digest[..]),
+            None => (),
+        }
+    }
+
+    openssl::sha::sha256(&common_raw[..])
+}
+
+/// Returns all APT repositories configured in `/etc/apt/sources.list` and
+/// in `/etc/apt/sources.list.d` including disabled repositories.
+///
+/// Returns the succesfully parsed files, a list of errors for files that could
+/// not be read or parsed and a common digest for the succesfully parsed files.
+///
+/// The digest is guaranteed to be set for each successfully parsed file.
+pub fn repositories() -> Result<
+    (
+        Vec<APTRepositoryFile>,
+        Vec<APTRepositoryFileError>,
+        [u8; 32],
+    ),
+    Error,
+> {
+    let to_result = |files: Vec<APTRepositoryFile>, errors: Vec<APTRepositoryFileError>| {
+        let common_digest = common_digest(&files);
+
+        (files, errors, common_digest)
+    };
+
+    let mut files = vec![];
+    let mut errors = vec![];
+
+    let sources_list_path = PathBuf::from(APT_SOURCES_LIST_FILENAME);
+
+    let sources_list_d_path = PathBuf::from(APT_SOURCES_LIST_DIRECTORY);
+
+    match APTRepositoryFile::new(sources_list_path) {
+        Ok(Some(mut file)) => match file.parse() {
+            Ok(()) => files.push(file),
+            Err(err) => errors.push(err),
+        },
+        _ => bail!("internal error with '{}'", APT_SOURCES_LIST_FILENAME),
+    }
+
+    if !sources_list_d_path.exists() {
+        return Ok(to_result(files, errors));
+    }
+
+    if !sources_list_d_path.is_dir() {
+        errors.push(APTRepositoryFileError {
+            path: APT_SOURCES_LIST_DIRECTORY.to_string(),
+            error: "not a directory!".to_string(),
+        });
+        return Ok(to_result(files, errors));
+    }
+
+    for entry in std::fs::read_dir(sources_list_d_path)? {
+        let path = entry?.path();
+
+        match APTRepositoryFile::new(path) {
+            Ok(Some(mut file)) => match file.parse() {
+                Ok(()) => {
+                    if file.digest.is_none() {
+                        bail!("internal error - digest not set");
+                    }
+                    files.push(file);
+                }
+                Err(err) => errors.push(err),
+            },
+            Ok(None) => (),
+            Err(err) => errors.push(err),
+        }
+    }
+
+    Ok(to_result(files, errors))
+}

src/repositories/repository.rs

@@ -0,0 +1,353 @@
+use std::convert::TryFrom;
+use std::fmt::Display;
+use std::io::Write;
+
+use anyhow::{bail, Error};
+use serde::{Deserialize, Serialize};
+
+use proxmox::api::api;
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum APTRepositoryFileType {
+    /// One-line-style format
+    List,
+    /// DEB822-style format
+    Sources,
+}
+
+impl TryFrom<&str> for APTRepositoryFileType {
+    type Error = Error;
+
+    fn try_from(string: &str) -> Result<Self, Error> {
+        match string {
+            "list" => Ok(APTRepositoryFileType::List),
+            "sources" => Ok(APTRepositoryFileType::Sources),
+            _ => bail!("invalid file type '{}'", string),
+        }
+    }
+}
+
+impl Display for APTRepositoryFileType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            APTRepositoryFileType::List => write!(f, "list"),
+            APTRepositoryFileType::Sources => write!(f, "sources"),
+        }
+    }
+}
+
+#[api]
+#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+pub enum APTRepositoryPackageType {
+    /// Debian package
+    Deb,
+    /// Debian source package
+    DebSrc,
+}
+
+impl TryFrom<&str> for APTRepositoryPackageType {
+    type Error = Error;
+
+    fn try_from(string: &str) -> Result<Self, Error> {
+        match string {
+            "deb" => Ok(APTRepositoryPackageType::Deb),
+            "deb-src" => Ok(APTRepositoryPackageType::DebSrc),
+            _ => bail!("invalid package type '{}'", string),
+        }
+    }
+}
+
+impl Display for APTRepositoryPackageType {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            APTRepositoryPackageType::Deb => write!(f, "deb"),
+            APTRepositoryPackageType::DebSrc => write!(f, "deb-src"),
+        }
+    }
+}
+
+#[api(
+    properties: {
+        Key: {
+            description: "Option key.",
+            type: String,
+        },
+        Values: {
+            description: "Option values.",
+            type: Array,
+            items: {
+                description: "Value.",
+                type: String,
+            },
+        },
+    },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")] // for consistency
+/// Additional options for an APT repository.
+/// Used for both single- and mutli-value options.
+pub struct APTRepositoryOption {
+    /// Option key.
+    pub key: String,
+    /// Option value(s).
+    pub values: Vec<String>,
+}
+
+#[api(
+    properties: {
+        Types: {
+            description: "List of package types.",
+            type: Array,
+            items: {
+                type: APTRepositoryPackageType,
+            },
+        },
+        URIs: {
+            description: "List of repository URIs.",
+            type: Array,
+            items: {
+                description: "Repository URI.",
+                type: String,
+            },
+        },
+        Suites: {
+            description: "List of distributions.",
+            type: Array,
+            items: {
+                description: "Package distribution.",
+                type: String,
+            },
+        },
+        Components: {
+            description: "List of repository components.",
+            type: Array,
+            items: {
+                description: "Repository component.",
+                type: String,
+            },
+        },
+        Options: {
+            type: Array,
+            optional: true,
+            items: {
+                type: APTRepositoryOption,
+            },
+        },
+        Comment: {
+            description: "Associated comment.",
+            type: String,
+            optional: true,
+        },
+        FileType: {
+            type: APTRepositoryFileType,
+        },
+        Enabled: {
+            description: "Whether the repository is enabled or not.",
+            type: Boolean,
+        },
+    },
+)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(rename_all = "PascalCase")]
+/// Describes an APT repository.
+pub struct APTRepository {
+    /// List of package types.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub types: Vec<APTRepositoryPackageType>,
+
+    /// List of repository URIs.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    #[serde(rename = "URIs")]
+    pub uris: Vec<String>,
+
+    /// List of package distributions.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub suites: Vec<String>,
+
+    /// List of repository components.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub components: Vec<String>,
+
+    /// Additional options.
+    #[serde(skip_serializing_if = "Vec::is_empty")]
+    pub options: Vec<APTRepositoryOption>,
+
+    /// Associated comment.
+    #[serde(skip_serializing_if = "String::is_empty")]
+    pub comment: String,
+
+    /// Format of the defining file.
+    pub file_type: APTRepositoryFileType,
+
+    /// Whether the repository is enabled or not.
+    pub enabled: bool,
+}
+
+impl APTRepository {
+    /// Crates an empty repository.
+    pub fn new(file_type: APTRepositoryFileType) -> Self {
+        Self {
+            types: vec![],
+            uris: vec![],
+            suites: vec![],
+            components: vec![],
+            options: vec![],
+            comment: String::new(),
+            file_type,
+            enabled: true,
+        }
+    }
+
+    /// Changes the `enabled` flag and makes sure the `Enabled` option for
+    /// `APTRepositoryPackageType::Sources` repositories is updated too.
+    pub fn set_enabled(&mut self, enabled: bool) {
+        self.enabled = enabled;
+
+        if self.file_type == APTRepositoryFileType::Sources {
+            let enabled_string = match enabled {
+                true => "true".to_string(),
+                false => "false".to_string(),
+            };
+            for option in self.options.iter_mut() {
+                if option.key == "Enabled" {
+                    option.values = vec![enabled_string];
+                    return;
+                }
+            }
+            self.options.push(APTRepositoryOption {
+                key: "Enabled".to_string(),
+                values: vec![enabled_string],
+            });
+        }
+    }
+
+    /// Makes sure that all basic properties of a repository are present and
+    /// not obviously invalid.
+    pub fn basic_check(&self) -> Result<(), Error> {
+        if self.types.is_empty() {
+            bail!("missing package type(s)");
+        }
+        if self.uris.is_empty() {
+            bail!("missing URI(s)");
+        }
+        if self.suites.is_empty() {
+            bail!("missing suite(s)");
+        }
+
+        for uri in self.uris.iter() {
+            if !uri.contains(':') || uri.len() < 3 {
+                bail!("invalid URI: '{}'", uri);
+            }
+        }
+
+        for suite in self.suites.iter() {
+            if !suite.ends_with('/') && self.components.is_empty() {
+                bail!("missing component(s)");
+            } else if suite.ends_with('/') && !self.components.is_empty() {
+                bail!("absolute suite '{}' does not allow component(s)", suite);
+            }
+        }
+
+        if self.file_type == APTRepositoryFileType::List {
+            if self.types.len() > 1 {
+                bail!("more than one package type");
+            }
+            if self.uris.len() > 1 {
+                bail!("more than one URI");
+            }
+            if self.suites.len() > 1 {
+                bail!("more than one suite");
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Writes a repository in the corresponding format followed by a blank.
+    ///
+    /// Expects that `basic_check()` for the repository was successful.
+    pub fn write(&self, w: &mut dyn Write) -> Result<(), Error> {
+        match self.file_type {
+            APTRepositoryFileType::List => write_one_line(self, w),
+            APTRepositoryFileType::Sources => write_stanza(self, w),
+        }
+    }
+}
+
+/// Writes a repository in one-line format followed by a blank line.
+///
+/// Expects that `repo.file_type == APTRepositoryFileType::List`.
+///
+/// Expects that `basic_check()` for the repository was successful.
+fn write_one_line(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+    if repo.file_type != APTRepositoryFileType::List {
+        bail!("not a .list repository");
+    }
+
+    if !repo.comment.is_empty() {
+        for line in repo.comment.lines() {
+            writeln!(w, "#{}", line)?;
+        }
+    }
+
+    if !repo.enabled {
+        write!(w, "# ")?;
+    }
+
+    write!(w, "{} ", repo.types[0])?;
+
+    if !repo.options.is_empty() {
+        write!(w, "[ ")?;
+        repo.options
+            .iter()
+            .try_for_each(|option| write!(w, "{}={} ", option.key, option.values.join(",")))?;
+        write!(w, "] ")?;
+    };
+
+    write!(w, "{} ", repo.uris[0])?;
+    write!(w, "{} ", repo.suites[0])?;
+    writeln!(w, "{}", repo.components.join(" "))?;
+
+    writeln!(w)?;
+
+    Ok(())
+}
+
+/// Writes a single stanza followed by a blank line.
+///
+/// Expects that `repo.file_type == APTRepositoryFileType::Sources`.
+fn write_stanza(repo: &APTRepository, w: &mut dyn Write) -> Result<(), Error> {
+    if repo.file_type != APTRepositoryFileType::Sources {
+        bail!("not a .sources repository");
+    }
+
+    if !repo.comment.is_empty() {
+        for line in repo.comment.lines() {
+            writeln!(w, "#{}", line)?;
+        }
+    }
+
+    write!(w, "Types:")?;
+    repo.types
+        .iter()
+        .try_for_each(|package_type| write!(w, " {}", package_type))?;
+    writeln!(w)?;
+
+    writeln!(w, "URIs: {}", repo.uris.join(" "))?;
+    writeln!(w, "Suites: {}", repo.suites.join(" "))?;
+
+    if !repo.components.is_empty() {
+        writeln!(w, "Components: {}", repo.components.join(" "))?;
+    }
+
+    for option in repo.options.iter() {
+        writeln!(w, "{}: {}", option.key, option.values.join(" "))?;
+    }
+
+    writeln!(w)?;
+
+    Ok(())
+}

tests/repositories.rs

@@ -0,0 +1,162 @@
+use std::path::PathBuf;
+
+use anyhow::{bail, format_err, Error};
+
+use proxmox_apt::repositories::APTRepositoryFile;
+
+#[test]
+fn test_parse_write() -> Result<(), Error> {
+    let test_dir = std::env::current_dir()?.join("tests");
+    let read_dir = test_dir.join("sources.list.d");
+    let write_dir = test_dir.join("sources.list.d.actual");
+    let expected_dir = test_dir.join("sources.list.d.expected");
+
+    if write_dir.is_dir() {
+        std::fs::remove_dir_all(&write_dir)
+            .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+    }
+
+    std::fs::create_dir_all(&write_dir)
+        .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+    let mut files = vec![];
+    let mut errors = vec![];
+
+    for entry in std::fs::read_dir(read_dir)? {
+        let path = entry?.path();
+
+        match APTRepositoryFile::new(&path)? {
+            Some(mut file) => match file.parse() {
+                Ok(()) => files.push(file),
+                Err(err) => errors.push(err),
+            },
+            None => bail!("unexpected None for '{:?}'", path),
+        }
+    }
+
+    assert!(errors.is_empty());
+
+    for file in files.iter_mut() {
+        let path = PathBuf::from(&file.path);
+        let new_path = write_dir.join(path.file_name().unwrap());
+        file.path = new_path.into_os_string().into_string().unwrap();
+        file.digest = None;
+        file.write()?;
+    }
+
+    let mut expected_count = 0;
+
+    for entry in std::fs::read_dir(expected_dir)? {
+        expected_count += 1;
+
+        let expected_path = entry?.path();
+        let actual_path = write_dir.join(expected_path.file_name().unwrap());
+
+        let expected_contents = std::fs::read(&expected_path)
+            .map_err(|err| format_err!("unable to read {:?} - {}", expected_path, err))?;
+
+        let actual_contents = std::fs::read(&actual_path)
+            .map_err(|err| format_err!("unable to read {:?} - {}", actual_path, err))?;
+
+        assert_eq!(
+            expected_contents, actual_contents,
+            "Use\n\ndiff {:?} {:?}\n\nif you're not fluent in byte decimals",
+            expected_path, actual_path
+        );
+    }
+
+    let actual_count = std::fs::read_dir(write_dir)?.count();
+
+    assert_eq!(expected_count, actual_count);
+
+    Ok(())
+}
+
+#[test]
+fn test_digest() -> Result<(), Error> {
+    let test_dir = std::env::current_dir()?.join("tests");
+    let read_dir = test_dir.join("sources.list.d");
+    let write_dir = test_dir.join("sources.list.d.digest");
+
+    if write_dir.is_dir() {
+        std::fs::remove_dir_all(&write_dir)
+            .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+    }
+
+    std::fs::create_dir_all(&write_dir)
+        .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+    let path = read_dir.join("standard.list");
+
+    let mut file = APTRepositoryFile::new(&path)?.unwrap();
+    file.parse()?;
+
+    let new_path = write_dir.join(path.file_name().unwrap());
+    file.path = new_path.clone().into_os_string().into_string().unwrap();
+
+    let old_digest = file.digest.unwrap();
+
+    // file does not exist yet...
+    assert!(file.read_with_digest().is_err());
+    assert!(file.write().is_err());
+
+    // ...but it should work if there's no digest
+    file.digest = None;
+    file.write()?;
+
+    // overwrite with old contents...
+    std::fs::copy(path, new_path)?;
+
+    // modify the repo
+    let mut repo = file.repositories.first_mut().unwrap();
+    repo.enabled = !repo.enabled;
+
+    // ...then it should work
+    file.digest = Some(old_digest);
+    file.write()?;
+
+    // expect a different digest, because the repo was modified
+    let (_, new_digest) = file.read_with_digest()?;
+    assert_ne!(old_digest, new_digest);
+
+    assert!(file.write().is_err());
+
+    Ok(())
+}
+
+#[test]
+fn test_empty_write() -> Result<(), Error> {
+    let test_dir = std::env::current_dir()?.join("tests");
+    let read_dir = test_dir.join("sources.list.d");
+    let write_dir = test_dir.join("sources.list.d.remove");
+
+    if write_dir.is_dir() {
+        std::fs::remove_dir_all(&write_dir)
+            .map_err(|err| format_err!("unable to remove dir {:?} - {}", write_dir, err))?;
+    }
+
+    std::fs::create_dir_all(&write_dir)
+        .map_err(|err| format_err!("unable to create dir {:?} - {}", write_dir, err))?;
+
+    let path = read_dir.join("standard.list");
+
+    let mut file = APTRepositoryFile::new(&path)?.unwrap();
+    file.parse()?;
+
+    let new_path = write_dir.join(path.file_name().unwrap());
+    file.path = new_path.clone().into_os_string().into_string().unwrap();
+
+    file.digest = None;
+
+    file.write()?;
+
+    assert!(file.exists());
+
+    file.repositories.clear();
+
+    file.write()?;
+
+    assert!(!file.exists());
+
+    Ok(())
+}

tests/sources.list.d.expected/absolute_suite.list

@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+deb http://packages.falcot.com/ updates/ 
+
+deb http://user.name@packages.falcot.com:80/ internal/ 
+

tests/sources.list.d.expected/absolute_suite.sources

@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+Types: deb
+URIs: http://packages.falcot.com/
+Suites: updates/ internal/
+

tests/sources.list.d.expected/case.sources

@@ -0,0 +1,16 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye-updates
+Components: main contrib
+languages: it de fr
+Enabled: false
+languages-Add: ja
+languages-Remove: de
+
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye
+Components: main contrib
+

tests/sources.list.d.expected/multiline.sources

@@ -0,0 +1,10 @@
+# comment in here
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye bullseye-updates
+Components: main contrib
+Languages: it de fr
+Enabled: false
+Languages-Add: ja
+Languages-Remove: de
+

tests/sources.list.d.expected/options_comment.list

@@ -0,0 +1,6 @@
+# comment
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian bullseye main contrib
+
+# non-free :(
+deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian bullseye non-free
+

tests/sources.list.d.expected/pbs-enterprise.list

@@ -0,0 +1,2 @@
+deb https://enterprise.proxmox.com/debian/pbs bullseye pbs-enterprise
+

tests/sources.list.d.expected/pve.list

@@ -0,0 +1,13 @@
+deb http://ftp.debian.org/debian bullseye main contrib
+
+deb http://ftp.debian.org/debian bullseye-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve bullseye pve-no-subscription
+
+# deb https://enterprise.proxmox.com/debian/pve bullseye pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security bullseye-security main contrib
+

tests/sources.list.d.expected/standard.list

@@ -0,0 +1,7 @@
+deb http://ftp.at.debian.org/debian bullseye main contrib
+
+deb http://ftp.at.debian.org/debian bullseye-updates main contrib
+
+# security updates
+deb http://security.debian.org bullseye-security main contrib
+

tests/sources.list.d.expected/standard.sources

@@ -0,0 +1,11 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye bullseye-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: bullseye-security
+Components: main contrib
+

tests/sources.list.d/absolute_suite.list

@@ -0,0 +1,4 @@
+# From Debian Administrator's Handbook
+deb http://packages.falcot.com/ updates/
+
+deb http://user.name@packages.falcot.com:80/ internal/

tests/sources.list.d/absolute_suite.sources

@@ -0,0 +1,5 @@
+# From Debian Administrator's Handbook
+Types: deb
+URIs: http://packages.falcot.com/
+Suites: updates/ internal/
+

tests/sources.list.d/case.sources

@@ -0,0 +1,17 @@
+tYpeS: deb deb-src
+uRis: http://ftp.at.debian.org/debian
+suiTes: bullseye-updates
+# comment in here
+CompOnentS: main contrib
+languages: it
+ de
+	fr
+Enabled: off
+languages-Add: ja
+languages-Remove: de
+
+types: deb deb-src
+Uris: http://ftp.at.debian.org/debian
+suites: bullseye
+# comment in here
+components: main contrib

tests/sources.list.d/multiline.sources

@@ -0,0 +1,11 @@
+Types: deb deb-src
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye bullseye-updates
+# comment in here
+Components: main contrib
+Languages: it
+ de
+	fr
+Enabled: off
+Languages-Add: ja
+Languages-Remove: de

tests/sources.list.d/options_comment.list

@@ -0,0 +1,3 @@
+deb [ lang=it,de arch=amd64 ] http://ftp.at.debian.org/debian bullseye main contrib # comment
+deb [ lang=it,de arch=amd64 lang+=fr lang-=de ] http://ftp.at.debian.org/debian bullseye non-free # non-free :(
+

tests/sources.list.d/pbs-enterprise.list

@@ -0,0 +1 @@
+deb https://enterprise.proxmox.com/debian/pbs bullseye pbs-enterprise

tests/sources.list.d/pve.list

@@ -0,0 +1,10 @@
+deb http://ftp.debian.org/debian bullseye main contrib
+deb http://ftp.debian.org/debian bullseye-updates main contrib
+
+# PVE pve-no-subscription repository provided by proxmox.com,
+# NOT recommended for production use
+deb http://download.proxmox.com/debian/pve bullseye pve-no-subscription
+# deb https://enterprise.proxmox.com/debian/pve bullseye pve-enterprise
+
+# security updates
+deb http://security.debian.org/debian-security bullseye-security main contrib

tests/sources.list.d/standard.list

@@ -0,0 +1,6 @@
+deb http://ftp.at.debian.org/debian bullseye main contrib
+
+deb http://ftp.at.debian.org/debian bullseye-updates main contrib
+
+# security updates
+deb http://security.debian.org bullseye-security main contrib

tests/sources.list.d/standard.sources

@@ -0,0 +1,10 @@
+Types: deb
+URIs: http://ftp.at.debian.org/debian
+Suites: bullseye bullseye-updates
+Components: main contrib
+
+# security updates
+Types: deb
+URIs: http://security.debian.org
+Suites: bullseye-security
+Components: main contrib