From f4f8dff05a01286b295d51be4a5eaad5311ef59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabian=20Gr=C3=BCnbichler?= Date: Thu, 17 Feb 2022 17:09:40 +0100 Subject: [PATCH] initial commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Fabian Grünbichler --- .cargo/config | 5 + Cargo.toml | 28 ++++ src/bin/proxmox_apt_mirror.rs | 83 ++++++++++ src/config.rs | 11 ++ src/lib.rs | 274 ++++++++++++++++++++++++++++++++++ src/verifier.rs | 94 ++++++++++++ 6 files changed, 495 insertions(+) create mode 100644 .cargo/config create mode 100644 Cargo.toml create mode 100644 src/bin/proxmox_apt_mirror.rs create mode 100644 src/config.rs create mode 100644 src/lib.rs create mode 100644 src/verifier.rs diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000..3b5b6e4 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,5 @@ +[source] +[source.debian-packages] +directory = "/usr/share/cargo/registry" +[source.crates-io] +replace-with = "debian-packages" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..c059434 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "proxmox-apt-mirror" +version = "0.1.0" +authors = ["Fabian Grünbichler "] +edition = "2021" +license = "AGPL-3" +description = "Proxmox APT repository mirror" + +exclude = ["debian"] + +[dependencies] +anyhow = "1.0" +bzip2 = "0.4" +flate2 = "1.0.22" +hex = "0.4.3" +hyper = "0.14" +openssl = "0.10" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +sequoia-openpgp = { version = "1.7" } +ureq = { version = "2.4.0", features = ["native-certs"] } +xz2 = "0.1" + +proxmox-apt = { path = "../proxmox-apt", version = "0.8.0" } +proxmox-async = "0.3" +proxmox-router = { version = "1.1", features = [ "cli" ] } +proxmox-schema = { version = "1.1", features = [ "api-macro" ] } +proxmox-sys = { version = "0.2.2" } \ No newline at end of file diff --git a/src/bin/proxmox_apt_mirror.rs b/src/bin/proxmox_apt_mirror.rs new file mode 100644 index 0000000..4eebdc7 --- /dev/null +++ b/src/bin/proxmox_apt_mirror.rs @@ -0,0 +1,83 @@ +use anyhow::Error; +use serde_json::Value; + +use proxmox_apt::repositories::APTRepository; +use proxmox_router::cli::{ + run_cli_command, CliCommand, CliCommandMap, CliEnvironment, OUTPUT_FORMAT, +}; +use proxmox_schema::api; +use proxmox_sys::fs::file_get_contents; + +#[api( + input: { + properties: { + repository: { + type: String, + description: "Repository string to parse.", + }, + key: { + type: String, + description: "Path to repository key." + }, + architectures: { + type: Array, + items: { + type: String, + description: "Architecture string (e.g., 'all', 'amd64', ..)", + }, + description: "Architectures to mirror (default: 'all' and 'amd64')", + optional: true, + }, + path: { + type: String, + description: "Output path. Contents will be re-used if still valid.", + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + } + }, + returns: { + type: APTRepository, + }, + )] +/// Parse a repository line. +async fn mirror( + repository: String, + key: String, + architectures: Option>, + path: String, + _param: Value, +) -> Result { + //let output_format = get_output_format(¶m); + + let repository = proxmox_apt_mirror::parse_repo(repository)?; + let key = file_get_contents(&key)?; + let architectures = architectures.unwrap_or_else(|| vec!["amd64".to_owned(), "all".to_owned()]); + + let config = proxmox_apt_mirror::config::MirrorConfig { + repository, + key, + path, + architectures, + }; + + proxmox_apt_mirror::mirror(&config)?; + + Ok(Value::Null) +} + +fn main() { + let rpcenv = CliEnvironment::new(); + + let mirror_cmd_def = CliCommand::new(&API_METHOD_MIRROR); + + let cmd_def = CliCommandMap::new().insert("mirror", mirror_cmd_def); + + run_cli_command( + cmd_def, + rpcenv, + Some(|future| proxmox_async::runtime::main(future)), + ); +} diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..99c5592 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,11 @@ +use std::fmt::Debug; + +use proxmox_apt::repositories::APTRepository; + +#[derive(Debug)] +pub struct MirrorConfig { + pub repository: APTRepository, + pub architectures: Vec, + pub path: String, + pub key: Vec, +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..6191de9 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,274 @@ +use std::{collections::HashMap, io::Read, path::PathBuf}; + +use anyhow::{bail, Error}; + +use config::MirrorConfig; +use flate2::bufread::GzDecoder; +use proxmox_apt::{ + deb822::{CompressionType, FileReference, FileReferenceType, PackagesFile, ReleaseFile}, + repositories::{ + APTRepository, APTRepositoryFile, APTRepositoryFileType, APTRepositoryPackageType, + }, +}; +use proxmox_sys::fs::{create_path, file_get_contents, replace_file, CreateOptions}; + +pub mod config; +mod verifier; + +/// Parse a single line in sources.list format +pub fn parse_repo(repo: String) -> Result { + let mut repo = APTRepositoryFile::with_content(repo, APTRepositoryFileType::List); + repo.parse()?; + Ok(repo.repositories[0].clone()) +} + +fn get_repo_url(repo: &APTRepository, path: &str) -> String { + let repo_root = format!("{}/dists/{}", repo.uris[0], repo.suites[0]); + + format!("{}/{}", repo_root, path) +} + +fn fetch_repo_file(uri: &str, max_size: Option) -> Result, Error> { + println!("-> GET '{}'..", uri); + + let response = ureq::get(uri).call()?.into_reader(); + + let mut content = Vec::new(); + let bytes = response + .take(max_size.unwrap_or(10_000_000)) + .read_to_end(&mut content)?; + println!("<- GOT {} bytes", bytes); + + Ok(content) +} + +pub fn fetch_release( + repo: &APTRepository, + key: &[u8], + output_dir: Option<&PathBuf>, + detached: bool, +) -> Result { + let (name, content, sig) = if detached { + println!("Fetching Release/Release.gpg files"); + let sig = fetch_repo_file(&get_repo_url(repo, "Release.gpg"), None)?; + let content = fetch_repo_file(&get_repo_url(repo, "Release"), Some(32_000_000))?; + ("Release(.gpg)", content, Some(sig)) + } else { + println!("Fetching InRelease file"); + let content = fetch_repo_file(&get_repo_url(repo, "InRelease"), Some(32_000_000))?; + ("InRelease", content, None) + }; + + println!("Verifying '{name}' signature using provided repository key.."); + let verified = verifier::verify_signature(&content[..], key, sig.as_deref())?; + println!("Success"); + + println!("Parsing '{name}'.."); + let parsed: ReleaseFile = verified[..].try_into()?; + println!( + "'{name}' file has {} referenced files..", + parsed.files.len() + ); + + if let Some(output_dir) = output_dir { + if detached { + let mut release_file = output_dir.clone(); + release_file.push("Release"); + replace_file(release_file, &content, CreateOptions::default(), true)?; + let mut release_sig = output_dir.clone(); + release_sig.push("Release.gpg"); + replace_file(release_sig, &sig.unwrap(), CreateOptions::default(), true)?; + } else { + let mut in_release = output_dir.clone(); + in_release.push("InRelease"); + replace_file(in_release, &content, CreateOptions::default(), true)?; + } + } + + Ok(parsed) +} + +pub fn fetch_referenced_file( + repo: &APTRepository, + output_dir: Option<&PathBuf>, + reference: &FileReference, +) -> Result, Error> { + let mut output = None; + let existing = if let Some(output_dir) = output_dir { + let mut path = output_dir.clone(); + path.push(&reference.path); + create_path(&path.parent().unwrap(), None, None)?; + output = Some(path.clone()); + + if let Ok(raw) = file_get_contents(&path) { + if let Ok(()) = reference.checksums.verify(&raw) { + output = None; + Some(raw) + } else { + None + } + } else { + None + } + } else { + None + }; + + let raw = if let Some(existing) = existing { + println!("Reused existing file '{}'", reference.path); + existing + } else { + let new = fetch_repo_file(&get_repo_url(repo, &reference.path), Some(100_000_000))?; + reference.checksums.verify(&new)?; + new + }; + let mut buf = Vec::new(); + + let decompressed = match reference.file_type.compression() { + None => &raw[..], + Some(CompressionType::Gzip) => { + let mut gz = GzDecoder::new(&raw[..]); + gz.read_to_end(&mut buf)?; + &buf[..] + } + Some(CompressionType::Bzip2) => { + let mut bz = bzip2::read::BzDecoder::new(&raw[..]); + bz.read_to_end(&mut buf)?; + &buf[..] + } + Some(CompressionType::Lzma) | Some(CompressionType::Xz) => { + let mut xz = xz2::read::XzDecoder::new(&raw[..]); + xz.read_to_end(&mut buf)?; + &buf[..] + } + }; + + if let Some(path) = output { + replace_file(path, &raw[..], CreateOptions::default(), true)?; + } + + Ok(decompressed.to_owned()) +} + +pub fn mirror(config: &MirrorConfig) -> Result<(), Error> { + let repo = &config.repository; + let output_dir = PathBuf::from(&config.path); + + if !output_dir.exists() { + proxmox_sys::fs::create_dir(&output_dir, CreateOptions::default())?; + } + + let release = fetch_release(repo, &config.key[..], Some(&output_dir), true)?; + let _release2 = fetch_release(repo, &config.key[..], Some(&output_dir), false)?; + + let mut per_component = HashMap::new(); + let mut others = Vec::new(); + let binary = repo.types.contains(&APTRepositoryPackageType::Deb); + let source = repo.types.contains(&APTRepositoryPackageType::DebSrc); + + for (basename, references) in &release.files { + let reference = references.first(); + let reference = if let Some(reference) = reference { + reference.clone() + } else { + continue; + }; + let skip_components = !repo.components.contains(&reference.component); + + // TODO make arch filtering some proper thing + let skip = skip_components + || match &reference.file_type { + FileReferenceType::Contents(arch, _) + | FileReferenceType::ContentsUdeb(arch, _) + | FileReferenceType::Packages(arch, _) => { + !binary || !config.architectures.contains(arch) + } + FileReferenceType::Sources(_) => !source, + _ => false, + }; + if skip { + println!("Skipping {}", reference.path); + others.push(reference); + } else { + let list = per_component + .entry(reference.component) + .or_insert_with(Vec::new); + list.push(basename); + } + } + println!(); + + let mut indices_size = 0_usize; + let mut total_count = 0; + + for (component, references) in &per_component { + println!("Component '{component}'"); + + let mut component_indices_size = 0; + + for basename in references { + for reference in release.files.get(*basename).unwrap() { + println!("\t{:?}: {:?}", reference.path, reference.file_type); + component_indices_size += reference.size; + } + } + indices_size += component_indices_size; + + let component_count = references.len(); + total_count += component_count; + + println!("Component references count: {component_count}"); + println!("Component indices size: {component_indices_size}"); + if references.is_empty() { + println!("\tNo references found.."); + } + } + println!("Total indices count: {total_count}"); + println!("Total indices size: {indices_size}"); + + if !others.is_empty() { + println!("Skipped {} references", others.len()); + } + println!(); + + let mut packages_size = 0_usize; + for (component, references) in per_component { + println!("Fetching indices for component '{component}'"); + let mut component_deb_size = 0; + for basename in &references { + let mut wrote_decompressed = false; + for reference in release.files.get(*basename).unwrap() { + match fetch_referenced_file(repo, Some(&output_dir), reference) { + Ok(data) => { + if !wrote_decompressed { + let mut path = output_dir.clone(); + path.push(basename); + replace_file(path, &data[..], CreateOptions::default(), true)?; + wrote_decompressed = true; + } + if matches!( + reference.file_type, + FileReferenceType::Packages(_, Some(CompressionType::Gzip)) + ) { + let packages: PackagesFile = data[..].try_into()?; + let size: usize = packages.files.iter().map(|p| p.size).sum(); + println!("\t{} packages totalling {size}", packages.files.len()); + component_deb_size += size; + } + } + Err(err) => { + eprintln!("Failed to fetch {} - {}", reference.path, err); + } + }; + } + if !wrote_decompressed { + bail!("Failed to write raw file.."); + } + } + println!("Total deb size for component: {component_deb_size}"); + packages_size += component_deb_size; + } + println!("Total deb size: {packages_size}"); + + Ok(()) +} diff --git a/src/verifier.rs b/src/verifier.rs new file mode 100644 index 0000000..ad1de09 --- /dev/null +++ b/src/verifier.rs @@ -0,0 +1,94 @@ +use anyhow::{bail, Error}; + +use sequoia_openpgp::{ + parse::{ + stream::{ + DetachedVerifierBuilder, MessageLayer, MessageStructure, VerificationHelper, + VerifierBuilder, + }, + Parse, + }, + policy::StandardPolicy, + Cert, KeyHandle, +}; +use std::io; + +struct Helper<'a> { + cert: &'a Cert, +} + +impl<'a> VerificationHelper for Helper<'a> { + fn get_certs(&mut self, _ids: &[KeyHandle]) -> sequoia_openpgp::Result> { + // Return public keys for signature verification here. + Ok(vec![self.cert.clone()]) + } + + fn check(&mut self, structure: MessageStructure) -> sequoia_openpgp::Result<()> { + // In this function, we implement our signature verification policy. + + let mut good = false; + + // we don't want compression and/or encryption + if structure.len() > 1 || structure.is_empty() { + bail!( + "unexpected GPG message structure - expected plain signed data, got {} layers!", + structure.len() + ); + } + let layer = &structure[0]; + let mut errors = Vec::new(); + match layer { + MessageLayer::SignatureGroup { results } => { + // We possibly have multiple signatures, but not all keys, so `or` all the individual results. + for result in results { + match result { + Ok(_) => good = true, + Err(e) => errors.push(e), + } + } + } + _ => return Err(anyhow::anyhow!("Unexpected message structure")), + } + + if good { + Ok(()) // Good signature. + } else if errors.len() > 1 { + Err(anyhow::anyhow!("encountered {} errors", errors.len())) + } else { + Err(anyhow::anyhow!("Signature verification failed")) + } + } +} +pub(crate) fn verify_signature<'msg>( + msg: &'msg [u8], + key: &[u8], + detached_sig: Option<&[u8]>, +) -> Result, Error> { + let cert = Cert::from_bytes(key)?; + + let policy = StandardPolicy::new(); + let helper = Helper { cert: &cert }; + + let verified = if let Some(sig) = detached_sig { + let mut verifier = + DetachedVerifierBuilder::from_bytes(sig)?.with_policy(&policy, None, helper)?; + verifier.verify_bytes(msg)?; + msg.to_vec() + } else { + let mut verified = Vec::new(); + let mut verifier = VerifierBuilder::from_bytes(msg)? + .with_policy(&policy, None, helper) + .map_err(|err| { + println!("{:#?}", err); + err + })?; + let bytes = io::copy(&mut verifier, &mut verified)?; + println!("{bytes} bytes verified"); + if !verifier.message_processed() { + bail!("Failed to verify message!"); + } + verified + }; + + Ok(verified) +}