use {
crate::{
content::{ContentProvider, IndexFile},
control::ParseError,
hash::Hash,
kvlist::KVList,
packages::PackageOrigin,
version::{Constraint, Dependency, Version, VersionSet},
Archive, Release, RepositoryFile,
},
async_compression::{
codecs::{DecodeV2, EncodeV2, ZstdDecoder, ZstdEncoder},
core::util::{PartialBuffer, WriteBuffer},
},
base64::{engine::general_purpose::STANDARD, write::EncoderWriter, Engine},
futures::{
stream::{self, LocalBoxStream},
StreamExt,
},
itertools::Itertools,
serde::{ser::SerializeStruct, Deserialize, Serialize},
std::{
io::{self, Write},
mem::MaybeUninit,
},
};
pub mod extends_serde {
use {
serde::{
de::{self, SeqAccess, Visitor},
Deserializer, Serializer,
},
std::fmt,
};
pub fn serialize<S>(parents: &[String], serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
match parents.len() {
0 => unreachable!("extends should be skipped when empty"),
1 => serializer.serialize_str(&parents[0]),
_ => {
use serde::ser::SerializeSeq;
let mut seq = serializer.serialize_seq(Some(parents.len()))?;
for parent in parents {
seq.serialize_element(parent)?;
}
seq.end()
}
}
}
pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
struct ExtendsVisitor;
impl<'de> Visitor<'de> for ExtendsVisitor {
type Value = Vec<String>;
fn expecting(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("a parent spec name or an array of parent spec names")
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(vec![v.to_string()])
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: de::Error,
{
Ok(vec![v])
}
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
where
A: SeqAccess<'de>,
{
let mut parents = Vec::with_capacity(seq.size_hint().unwrap_or(0));
while let Some(parent) = seq.next_element::<String>()? {
if parents.iter().any(|p: &String| p == &parent) {
return Err(de::Error::custom(format!(
"duplicate parent spec in extends: {}",
parent
)));
}
parents.push(parent);
}
Ok(parents)
}
}
deserializer.deserialize_any(ExtendsVisitor)
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(deny_unknown_fields)]
pub struct Spec {
#[serde(
default,
rename = "extends",
with = "extends_serde",
skip_serializing_if = "Vec::is_empty"
)]
pub extends: Vec<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub include: Vec<Dependency<String>>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub exclude: Vec<Constraint<String>>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub stage: Vec<String>,
#[serde(
default,
rename = "build-env",
skip_serializing_if = "KVList::is_empty"
)]
pub build_env: KVList<String>,
#[serde(
default,
rename = "build-script",
skip_serializing_if = "Option::is_none"
)]
pub build_script: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub meta: Vec<String>,
}
impl Spec {
pub fn new() -> Self {
Self {
extends: Vec::new(),
include: Vec::new(),
exclude: Vec::new(),
stage: Vec::new(),
build_env: KVList::new(),
meta: Vec::new(),
build_script: None,
}
}
pub fn locked_spec(&self) -> LockedSpec {
LockedSpec {
hash: None,
installables: None,
}
}
}
impl Default for Spec {
fn default() -> Self {
Self::new()
}
}
pub(crate) const META_VALUE_MAX_BYTES: usize = 1024;
pub(crate) fn validate_meta_name(name: &str) -> Result<(), String> {
let mut chars = name.chars();
let first = chars
.next()
.ok_or_else(|| "meta name is empty".to_string())?;
if !first.is_ascii_alphabetic() {
return Err("meta name must start with an ASCII letter".to_string());
}
if !chars.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.') {
return Err(
"meta name may only contain ASCII alphanumerics, '_', '-', and '.'".to_string(),
);
}
Ok(())
}
pub(crate) fn validate_meta_value(value: &str) -> Result<(), String> {
if value.len() > META_VALUE_MAX_BYTES {
return Err(format!("meta value exceeds {} bytes", META_VALUE_MAX_BYTES));
}
if value.chars().any(|c| c.is_control()) {
return Err("meta value contains non-printable characters".to_string());
}
Ok(())
}
pub(crate) fn parse_meta_entry(entry: &str) -> Result<(&str, &str), String> {
let (name, value) = entry
.split_once(':')
.ok_or_else(|| "meta entry must be in \"name:value\" form".to_string())?;
validate_meta_name(name)?;
validate_meta_value(value)?;
Ok((name, value))
}
#[derive(Clone)]
pub struct LockedSuite {
pub path: String,
pub file: IndexFile,
pub rel: Release,
}
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LockedArchive {
pub suites: Vec<LockedSuite>,
}
impl LockedArchive {
pub(crate) fn fetch_update<'a, C: ContentProvider>(
locked: &'a Option<Self>,
archive: &'a Archive,
archive_idx: usize,
skip_verify: bool,
base_dir: std::path::PathBuf,
cache: &'a C,
) -> LocalBoxStream<'a, io::Result<(usize, usize, Option<LockedSuite>)>> {
tracing::debug!(
"Refreshing locked archive for {} {}",
archive.url,
archive.suites.iter().join(" "),
);
stream::iter(archive.suites.iter().enumerate())
.then(move |(suite_idx, suite)| {
let base_dir = base_dir.clone();
async move {
tracing::debug!("Refreshing locked archive for {} {}", archive.url, suite);
let path = archive.release_path(suite);
let file = cache.fetch_release_file(&archive.file_url(&path)).await?;
let rel = archive
.release_from_file(file.clone(), skip_verify, &base_dir)
.await?;
match locked.as_ref().and_then(|l| l.suites.get(suite_idx)) {
Some(suite) => {
if suite.path == path && suite.rel.as_bytes().eq(rel.as_bytes()) {
Ok((archive_idx, suite_idx, None))
} else {
Ok((
archive_idx,
suite_idx,
Some(LockedSuite { path, file, rel }),
))
}
}
None => Ok((
archive_idx,
suite_idx,
Some(LockedSuite { path, file, rel }),
)),
}
}
})
.boxed_local()
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(deny_unknown_fields)]
pub struct LockedPackage {
#[serde(
default = "PackageOrigin::legacy_local",
skip_serializing_if = "PackageOrigin::is_unknown"
)]
pub orig: PackageOrigin,
pub idx: u32,
pub name: String,
pub arch: String,
pub version: String,
pub order: u32,
#[serde(flatten)]
pub file: RepositoryFile,
}
impl TryFrom<&LockedPackage> for Constraint<String> {
type Error = ParseError;
fn try_from(pkg: &LockedPackage) -> Result<Self, Self::Error> {
Ok(Constraint::new(
Some(pkg.arch.clone()),
pkg.name.clone(),
VersionSet::Exactly(Version::try_from(pkg.version.as_str())?),
))
}
}
impl TryFrom<LockedPackage> for Constraint<String> {
type Error = ParseError;
fn try_from(pkg: LockedPackage) -> Result<Self, Self::Error> {
(&pkg).try_into()
}
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(deny_unknown_fields)]
pub struct LockedSpec {
#[serde(with = "crate::hash::serde::sri::opt")]
pub hash: Option<Hash>,
pub installables: Option<Vec<LockedPackage>>,
}
impl LockedSpec {
pub fn is_locked(&self) -> bool {
self.hash.is_some() && self.installables.is_some()
}
pub fn as_locked(&self) -> Option<&'_ Self> {
self.is_locked().then_some(self)
}
pub fn invalidate_solution(&mut self) {
self.hash = None;
self.installables = None;
}
pub fn installables(&self) -> impl Iterator<Item = &LockedPackage> {
self.installables.iter().flat_map(|v| v.iter())
}
}
#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::*;
#[derive(Debug, Deserialize, Serialize)]
struct WireLockedPackage {
#[serde(
default = "PackageOrigin::legacy_local",
skip_serializing_if = "PackageOrigin::is_unknown"
)]
orig: PackageOrigin,
}
#[test]
fn locked_package_origin_defaults_to_legacy_local_when_missing() {
let pkg: WireLockedPackage = toml_edit::de::from_str("").unwrap();
assert_eq!(pkg.orig, PackageOrigin::Local { manifest_id: 0 });
}
#[test]
fn locked_package_origin_serializes_unknown_by_omitting_field() {
let pkg = WireLockedPackage {
orig: PackageOrigin::Unknown,
};
assert_eq!(toml_edit::ser::to_string(&pkg).unwrap(), "");
}
#[test]
fn locked_package_origin_round_trips_new_archive_form() {
let pkg = WireLockedPackage {
orig: PackageOrigin::Archive {
manifest_id: 3,
archive_id: 11,
},
};
assert_eq!(
toml_edit::ser::to_string(&pkg).unwrap(),
"orig = \":3:11\"\n"
);
let parsed: WireLockedPackage = toml_edit::de::from_str("orig = \":3:11\"\n").unwrap();
assert_eq!(parsed.orig, pkg.orig);
}
#[test]
fn locked_package_origin_accepts_legacy_integer_form() {
let pkg: WireLockedPackage = toml_edit::de::from_str("orig = 9\n").unwrap();
assert_eq!(
pkg.orig,
PackageOrigin::Archive {
manifest_id: 0,
archive_id: 9
}
);
}
#[test]
fn locked_package_try_into_constraint_builds_exact_dependency() {
let pkg = LockedPackage {
orig: PackageOrigin::Local { manifest_id: 0 },
idx: 7,
name: "foo".to_string(),
arch: "amd64".to_string(),
version: "1.2.3-1".to_string(),
order: 0,
file: RepositoryFile {
path: "pool/main/f/foo.deb".to_string(),
fetch_path: None,
size: 1,
hash: Hash::default(),
},
};
let constraint: Constraint<String> = (&pkg).try_into().unwrap();
assert_eq!(constraint.to_string(), "foo:amd64 (= 1.2.3-1)");
}
#[test]
fn locked_package_try_into_constraint_rejects_invalid_version() {
let pkg = LockedPackage {
orig: PackageOrigin::Local { manifest_id: 0 },
idx: 7,
name: "foo".to_string(),
arch: "amd64".to_string(),
version: "bad version".to_string(),
order: 0,
file: RepositoryFile {
path: "pool/main/f/foo.deb".to_string(),
fetch_path: None,
size: 1,
hash: Hash::default(),
},
};
let err: Result<Constraint<String>, _> = pkg.try_into();
assert!(err.is_err());
}
}
impl serde::ser::Serialize for LockedSuite {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::ser::Serializer,
{
let mut struc = serializer.serialize_struct("LockedSuite", 2)?;
struc.serialize_field("path", self.path.as_str())?;
let mut comp = ZstdEncoder::new(9);
let mut buf = vec![MaybeUninit::<u8>::uninit(); 8 * 1024];
let mut inp = PartialBuffer::new(self.file.as_bytes());
let mut out = WriteBuffer::new_uninitialized(&mut buf);
struct LineWrapWriter {
out: Vec<u8>,
col: usize,
width: usize,
}
impl LineWrapWriter {
fn new(width: usize, capacity: usize) -> Self {
Self {
out: Vec::with_capacity(capacity),
col: 0,
width,
}
}
fn into_string(mut self) -> String {
if self.col == 0 && self.out.ends_with(b"\n") {
self.out.pop();
}
unsafe { String::from_utf8_unchecked(self.out) }
}
}
impl Write for LineWrapWriter {
fn write(&mut self, mut buf: &[u8]) -> io::Result<usize> {
let s = buf.len();
while !buf.is_empty() {
let to_write = std::cmp::min(buf.len(), self.width - self.col);
if to_write == 0 {
self.out.push(b'\n');
self.col = 0;
} else {
let (line, rest) = buf.split_at(to_write);
self.out.extend_from_slice(line);
self.col += to_write;
buf = rest;
}
}
Ok(s)
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
let mut enc = EncoderWriter::new(LineWrapWriter::new(80, self.file.len()), &STANDARD);
loop {
let done = inp.written_len() == self.file.len();
if !done {
comp.encode(&mut inp, &mut out)
.map_err(serde::ser::Error::custom)?;
enc.write_all(out.written())
.map_err(serde::ser::Error::custom)?;
out.reset();
} else if comp.finish(&mut out).map_err(serde::ser::Error::custom)? {
enc.write_all(out.written())
.map_err(serde::ser::Error::custom)?;
break;
} else {
enc.write_all(out.written())
.map_err(serde::ser::Error::custom)?;
out.reset();
}
}
let wrapped = enc
.finish()
.map_err(serde::ser::Error::custom)?
.into_string();
struc.serialize_field("text", wrapped.as_str())?;
struc.end()
}
}
impl<'de> serde::de::Deserialize<'de> for LockedSuite {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::de::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct EncodedReleaseFile {
path: String,
text: String,
}
let encoded = EncodedReleaseFile::deserialize(deserializer)?;
let normalized: String = encoded.text.split_whitespace().collect();
let compressed = STANDARD
.decode(normalized.as_bytes())
.map_err(serde::de::Error::custom)?;
let mut dec = ZstdDecoder::new();
let mut inp = PartialBuffer::new(compressed.as_slice());
let mut buf = vec![MaybeUninit::<u8>::uninit(); 8 * 1024];
let mut out = WriteBuffer::new_uninitialized(&mut buf);
let mut decoded = Vec::new();
loop {
let done = inp.written_len() == compressed.len();
if !done {
dec.decode(&mut inp, &mut out)
.map_err(serde::de::Error::custom)?;
decoded.extend_from_slice(out.written());
out.reset();
} else if dec.finish(&mut out).map_err(serde::de::Error::custom)? {
decoded.extend_from_slice(out.written());
break;
} else {
decoded.extend_from_slice(out.written());
out.reset();
}
}
let file =
IndexFile::from_string(String::from_utf8(decoded).map_err(serde::de::Error::custom)?);
let rel = Release::new(file.clear_text()).map_err(serde::de::Error::custom)?;
Ok(Self {
path: encoded.path,
file,
rel,
})
}
}