use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
use paperpaw_sdk::install::{self, Digest, MAX_FILES, Package, PackageFile};
use paperpaw_sdk::json::stream::{Event, read_file};
use paperpaw_sdk::native::Fingerprint;
use paperpaw_sdk::web::{encode, join_url};
use paperpaw_sdk::{Env, PluginKind, Plugins, Storage, Version};
use crate::PLUGIN;
use crate::needs::{Fit, Needs, Provides};
use crate::revoked::Revocations;
use crate::trust::{Named, Rotation};
mod runs;
pub(crate) use runs::RUN;
use runs::{Cursor, KEYED, Keyed, Output, before, order_of};
const FORMAT: u32 = 2;
pub(crate) const MAX_LIST_FILE: u64 = 4 * 1024 * 1024;
pub(crate) const MAX_PAGE_FILE: u64 = 512 * 1024;
pub(crate) const MAX_SHOT_FILE: u64 = 2 * 1024 * 1024;
const MAX_SUMMARY: usize = 300;
const MAX_NAME: usize = 80;
const MAX_LINE: usize = 1024;
const WRITE_AT: usize = 2048;
const MAX_RUNS: usize = 32;
pub(crate) const OFFICIAL_LIST: &str = "https://paperpaw.org/api/v1/plugins.json";
pub(crate) const FIRST_PARTY_KEY: [u8; 32] = [
0x14, 0x8c, 0xcf, 0x3e, 0xbc, 0x30, 0x50, 0x6b, 0x35, 0x5c, 0x6b, 0x5c, 0xb2, 0xba, 0xf8, 0x7d,
0x92, 0x65, 0x60, 0x6c, 0xc6, 0xa8, 0x1f, 0x08, 0x5b, 0xb3, 0xb9, 0x74, 0x0a, 0x95, 0xae, 0xbd,
];
pub(crate) const SECTIONS: [&str; 5] = ["recommended", "new", "trending", "updated", "discover"];
const MAX_SECTION: usize = 20;
pub(crate) const GROUPS: [&str; 13] = [
"reading",
"formats",
"library",
"sync",
"home",
"look",
"notes",
"learning",
"news",
"tools",
"games",
"languages",
"system",
];
pub(crate) fn group(name: &str) -> usize {
GROUPS
.iter()
.position(|g| *g == name)
.unwrap_or(GROUPS.len() - 1)
}
#[derive(Debug, Clone, Copy)]
pub(crate) struct Device<'a> {
pub name: &'a str,
pub fingerprint: Option<Fingerprint>,
pub code_file: &'a str,
pub plugins: Option<&'a Plugins>,
pub sdk: Version,
pub official: bool,
pub provides: Option<Provides>,
}
impl<'a> Device<'a> {
pub(crate) fn of(env: &Env<'a>) -> Option<Device<'a>> {
Some(Device {
name: env.catalog_device()?,
fingerprint: env.native().map(|native| native.fingerprint),
code_file: env.native().map_or("", |native| &native.file),
plugins: Some(env.plugins()),
sdk: env
.plugins()
.kernel()
.map_or(paperpaw_sdk::SDK_VERSION, |kernel| kernel.manifest.version),
official: env.settings().read(&crate::CATALOG).trim() == OFFICIAL_LIST,
provides: env.native().map(Provides::of),
})
}
}
pub(crate) fn may_install(plugins: &Plugins, id: &str, code: bool) -> bool {
plugins.get(id).is_none_or(|info| match info.kind {
PluginKind::BuiltIn => false,
PluginKind::Data => !code,
PluginKind::Code => code,
_ => false,
})
}
pub(crate) fn switched_off(plugins: &Plugins, id: &str) -> bool {
plugins
.get(id)
.is_some_and(|info| info.kind == PluginKind::BuiltIn && !info.enabled)
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct Entry {
pub id: String,
pub name: String,
pub summary: String,
pub group: usize,
pub author: String,
pub licence: String,
pub version: String,
pub sdk: String,
pub code: bool,
pub official: bool,
pub unreviewed: bool,
pub needs_newer: bool,
pub waits_author: bool,
pub size: u64,
pub added: u32,
pub updated: u32,
}
impl Entry {
pub(crate) fn installed(&self, env: &Env<'_>) -> Option<Version> {
env.plugins()
.get(&self.id)
.map(|info| info.update.unwrap_or(info.manifest.version))
}
pub(crate) fn switched_off(&self, env: &Env<'_>) -> bool {
switched_off(env.plugins(), &self.id)
}
fn line(&self) -> String {
let clean = |s: &str| s.replace(['\t', '\n', '\r'], " ");
format!(
"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n",
self.group,
self.size,
u8::from(self.code)
| (u8::from(self.official) << 1)
| (u8::from(self.unreviewed) << 2)
| (u8::from(self.needs_newer) << 3)
| (u8::from(self.waits_author) << 4),
self.added,
self.updated,
clean(&self.id),
clean(&self.version),
clean(&self.sdk),
clean(&self.author),
clean(&self.licence),
clean(&self.name),
clean(&self.summary)
)
}
fn parse(line: &str) -> Option<Entry> {
let mut f = line.splitn(12, '\t');
let group = f.next()?.parse().ok()?;
let size = f.next()?.parse().ok()?;
let flags: u8 = f.next()?.parse().ok()?;
Some(Entry {
group,
size,
code: flags & 1 != 0,
official: flags & 2 != 0,
unreviewed: flags & 4 != 0,
needs_newer: flags & 8 != 0,
waits_author: flags & 16 != 0,
added: f.next()?.parse().ok()?,
updated: f.next()?.parse().ok()?,
id: f.next()?.to_string(),
version: f.next()?.to_string(),
sdk: f.next()?.to_string(),
author: f.next()?.to_string(),
licence: f.next()?.to_string(),
name: f.next()?.to_string(),
summary: f.next()?.to_string(),
})
}
}
#[derive(Debug, Clone, Default)]
struct Build {
device: String,
abi: String,
url: String,
size: u64,
sha256: String,
image_len: Option<u64>,
needs: Needs,
families: Vec<u16>,
}
#[derive(Default)]
struct Item {
entry: Entry,
kind: String,
built_in: bool,
official: bool,
devices: bool,
device: bool,
formats: u32,
author_key: String,
recovery_key: String,
rotation_to: String,
rotation_signature: String,
rotations: Vec<Rotation>,
files: Vec<PackageFile>,
file: PackageFile,
build: Build,
code: Option<PackageFile>,
needs_newer: bool,
waits: String,
more: More,
shot: Shot,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct Shot {
pub url: String,
pub width: u32,
pub height: u32,
pub size: u64,
}
pub(crate) const MAX_SHOTS: usize = 4;
const MAX_SHOT_SIDE: u32 = 2560;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub(crate) struct More {
pub source: String,
pub repository: String,
pub author_key: String,
pub author_since: u32,
pub author_plugins: u32,
pub verified: String,
pub heap: Option<u64>,
pub image_len: Option<u64>,
pub shots: Vec<Shot>,
pub description: String,
}
const MAX_DESCRIPTION: usize = 4000;
pub(crate) fn date(text: &str) -> u32 {
let digits: String = text.chars().filter(char::is_ascii_digit).take(8).collect();
match digits.len() {
8 => digits.parse().unwrap_or(0),
4 => digits.parse::<u32>().map_or(0, |y| y * 10000 + 101),
_ => 0,
}
}
impl Item {
fn new() -> Self {
Self {
formats: 1,
..Self::default()
}
}
fn waits_for_newer(&self) -> bool {
self.code.is_none()
&& (self.needs_newer || self.waits == "update" || self.waits_for_author())
}
fn waits_for_author(&self) -> bool {
self.waits == "author"
}
fn fits(&self, device: &Device<'_>) -> bool {
let entry = &self.entry;
let off = device
.plugins
.is_some_and(|plugins| switched_off(plugins, &entry.id));
let kind = match self.kind.as_str() {
"data" => true,
"code" => self.code.is_some() || self.waits_for_newer() || off,
_ => false,
};
let code = self.code.is_some() || self.waits_for_newer();
kind && !self.built_in
&& (off
|| device
.plugins
.is_none_or(|plugins| may_install(plugins, &entry.id, code)))
&& (!self.devices || self.device)
&& install::is_plugin_name(&entry.id)
&& !entry.name.is_empty()
&& Version::parse(&entry.version).is_some()
&& install::fits_sdk(device.sdk, &entry.sdk, self.formats)
}
fn read(&mut self, key: &[String], value: &str, device: &Device<'_>) {
let entry = &mut self.entry;
match key {
[key] => match key.as_str() {
"id" => entry.id = cut(value, MAX_NAME),
"name" => entry.name = cut(value, MAX_NAME),
"summary" => entry.summary = cut(value, MAX_SUMMARY),
"author" => entry.author = cut(value, MAX_NAME),
"licence" => entry.licence = cut(value, MAX_NAME),
"group" => entry.group = group(value),
"kind" => self.kind = cut(value, MAX_NAME),
"waits" => self.waits = cut(value, MAX_NAME),
"built_in" => self.built_in = value == "true",
"official" => self.official = value == "true",
"unreviewed" => entry.unreviewed = value == "true",
"repository" if value.starts_with("https://") => {
self.more.repository = cut(value, 200);
}
"author_key" => self.author_key = cut(value, MAX_NAME),
"recovery_key" => self.recovery_key = cut(value, MAX_NAME),
"added" => entry.added = date(value),
"source" if value.starts_with("https://") && self.more.source.is_empty() => {
self.more.source = cut(value, 200);
}
"author_since" => self.more.author_since = date(value),
"description" => self.more.description = cut(value, MAX_DESCRIPTION),
"author_plugins" => self.more.author_plugins = value.parse().unwrap_or(0),
_ => {}
},
[rotations, at, key] if rotations == "rotations" && at == "[]" => match key.as_str() {
"to" => self.rotation_to = cut(value, MAX_NAME),
"signature" => self.rotation_signature = cut(value, 2 * MAX_NAME),
_ => {}
},
[latest, source, url] if latest == "latest" && source == "source" && url == "url" => {
if value.starts_with("https://") {
self.more.source = cut(value, 200);
}
}
[shots, at, key] if shots == "screenshots" && at == "[]" => match key.as_str() {
"url" => self.shot.url = cut(value, 300),
"width" => self.shot.width = value.parse().unwrap_or(0),
"height" => self.shot.height = value.parse().unwrap_or(0),
"size" => self.shot.size = value.parse().unwrap_or(0),
_ => {}
},
[devices, at] if devices == "devices" && at == "[]" => {
self.devices = true;
self.device |= value == device.name;
}
[latest, key] if latest == "latest" => match key.as_str() {
"version" => entry.version = cut(value, MAX_NAME),
"sdk" => entry.sdk = cut(value, MAX_NAME),
"formats" => self.formats = value.parse().unwrap_or(u32::MAX),
"size" => entry.size = value.parse().unwrap_or(0),
"date" => entry.updated = date(value),
"heap" => self.more.heap = value.parse().ok().filter(|&h| h > 0),
"verified" if self.more.verified.is_empty() => {
self.more.verified = if value == "true" {
"verified"
} else {
"unverified"
}
.into();
}
_ => {}
},
[latest, files, at, key] if latest == "latest" && files == "files" && at == "[]" => {
let file = &mut self.file;
match key.as_str() {
"path" => file.path = value.to_string(),
"url" => file.url = value.to_string(),
"size" => file.size = value.parse().unwrap_or(0),
key => {
file.digest.take(key, value);
}
}
}
[latest, builds, at, rest @ ..]
if latest == "latest" && builds == "builds" && at == "[]" =>
{
self.read_build(rest, value);
}
_ => {}
}
}
fn read_build(&mut self, key: &[String], value: &str) {
let build = &mut self.build;
match key {
[key] => match key.as_str() {
"device" => build.device = cut(value, MAX_NAME),
"abi" => build.abi = cut(value, MAX_NAME),
"url" => build.url = value.to_string(),
"size" => build.size = value.parse().unwrap_or(0),
"sha256" => build.sha256 = value.to_ascii_lowercase(),
"image_len" => build.image_len = value.parse().ok(),
"needs_api" => build.needs.api = value.parse().ok(),
"rebuilt" if value == "differs" => self.more.verified = "failed".into(),
_ => {}
},
[key, item]
if key == "families"
&& item == "[]"
&& build.families.len() <= crate::needs::FAMILIES =>
{
build.families.push(value.parse().unwrap_or(u16::MAX));
}
_ => {}
}
}
fn file_done(&mut self) {
let file = core::mem::take(&mut self.file);
if self.files.len() <= MAX_FILES {
self.files.push(file);
}
}
fn build_done(&mut self, device: &Device<'_>) {
let mut build = core::mem::take(&mut self.build);
let right = build.device == device.name
&& !device.code_file.is_empty()
&& device
.fingerprint
.is_some_and(|f| Fingerprint::parse(&build.abi) == Some(f));
if !right || self.code.is_some() {
return;
}
build.needs.interface = Fingerprint::parse(&build.abi).map(Fingerprint::interface);
build.needs.families = <[u16; crate::needs::FAMILIES]>::try_from(build.families.as_slice())
.ok()
.filter(|f| !f.contains(&u16::MAX));
match device.provides.map_or(Fit::Yes, |p| p.fit(&build.needs)) {
Fit::Yes => {
self.more.image_len = build.image_len;
self.code = Some(PackageFile::new(
device.code_file.to_string(),
build.url,
build.size,
Digest::Sha256(build.sha256),
));
}
Fit::Newer => self.needs_newer = true,
Fit::Older => {}
}
}
fn rotation_done(&mut self) {
let step = key(&self.rotation_to).zip(signature(&self.rotation_signature));
self.rotation_to.clear();
self.rotation_signature.clear();
if let Some((to, signature)) = step
&& self.rotations.len() < crate::trust::MAX_STEPS
{
self.rotations.push(Rotation { to, signature });
}
}
fn shot_done(&mut self) {
let shot = core::mem::take(&mut self.shot);
let path = shot
.url
.split(['?', '#'])
.next()
.unwrap_or("")
.to_ascii_lowercase();
let readable = [".png", ".jpg", ".jpeg"]
.iter()
.any(|end| path.ends_with(end));
let small = shot.width <= MAX_SHOT_SIDE && shot.height <= MAX_SHOT_SIDE;
if self.more.shots.len() < MAX_SHOTS && readable && small {
self.more.shots.push(shot);
}
}
fn finish(mut self, device: &Device<'_>) -> Entry {
if let Some(code) = &self.code {
self.entry.code = true;
self.entry.size = self.entry.size.saturating_add(code.size);
} else if self.needs_newer || self.waits == "update" {
self.entry.code = true;
self.entry.needs_newer = true;
} else if self.waits_for_author() {
self.entry.code = true;
self.entry.waits_author = true;
}
let signed = !self.entry.code || key(&self.author_key) == Some(FIRST_PARTY_KEY);
self.entry.official = self.official && device.official && signed;
self.entry.unreviewed &= !self.entry.official;
self.entry
}
}
fn signature(hex: &str) -> Option<[u8; 64]> {
paperpaw_sdk::digest::hex::decode(hex.trim())?
.try_into()
.ok()
}
fn key(hex: &str) -> Option<[u8; 32]> {
let hex = hex.trim().as_bytes();
if hex.len() != 64 {
return None;
}
let digit = |b: u8| char::from(b).to_digit(16);
let mut key = [0u8; 32];
for (byte, pair) in key.iter_mut().zip(hex.chunks(2)) {
*byte = u8::try_from(digit(pair[0])? * 16 + digit(pair[1])?).ok()?;
}
Some(key)
}
fn cut(text: &str, max: usize) -> String {
let mut end = text.len().min(max);
while !text.is_char_boundary(end) {
end -= 1;
}
text[..end].to_string()
}
fn is_file(path: &[String]) -> bool {
matches!(path, [latest, files, at] if latest == "latest" && files == "files" && at == "[]")
}
fn is_build(path: &[String]) -> bool {
matches!(path, [latest, builds, at] if latest == "latest" && builds == "builds" && at == "[]")
}
#[derive(Debug, Default)]
struct Picks {
fair: bool,
sections: [Vec<String>; SECTIONS.len()],
}
fn read_list(
storage: &dyn Storage,
path: &str,
device: &Device<'_>,
found: &mut dyn FnMut(Entry),
) -> Option<Picks> {
let mut format = None;
let mut item: Option<Item> = None;
let mut picks = Picks::default();
let mut revoked = Revocations::default();
read_file(storage, path, &mut |event| match event {
Event::Value([key], value) if key == "format" => format = value.parse::<u32>().ok(),
Event::Value([list, at, field], value) if list == "revoked_keys" && at == "[]" => {
revoked.field(field, value);
}
Event::Close([list, at])
if list == "revoked_keys" && at == "[]" && format == Some(FORMAT) =>
{
revoked.entry_done(storage, &|id| {
device
.plugins
.is_none_or(|plugins| plugins.get(id).is_some())
});
}
Event::Open([key, ..]) if key == "sections" => picks.fair = true,
Event::Value([key, section, at], value) if key == "sections" && at == "[]" => {
picks.fair = true;
let ids = SECTIONS
.iter()
.position(|s| s == section)
.map(|s| &mut picks.sections[s]);
if let Some(ids) = ids.filter(|ids| ids.len() < MAX_SECTION)
&& install::is_plugin_name(value)
{
ids.push(value.to_string());
}
}
Event::Open([list, at]) if list == "plugins" && at == "[]" => item = Some(Item::new()),
Event::Close([list, at]) if list == "plugins" && at == "[]" => {
if let Some(done) = item.take().filter(|item| item.fits(device)) {
found(done.finish(device));
}
}
Event::Close([list, at, rest @ ..])
if list == "plugins" && at == "[]" && is_build(rest) =>
{
if let Some(item) = item.as_mut() {
item.build_done(device);
}
}
Event::Value([list, at, rest @ ..], value) if list == "plugins" && at == "[]" => {
if let Some(item) = item.as_mut() {
item.read(rest, value, device);
}
}
_ => {}
});
if format == Some(FORMAT) {
revoked.hand_over(storage);
}
(format == Some(FORMAT)).then_some(picks)
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct Plugin {
pub entry: Entry,
pub package: Package,
pub author_key: Option<[u8; 32]>,
pub recovery_key: Option<[u8; 32]>,
pub rotations: Vec<Rotation>,
pub more: More,
}
impl Plugin {
pub(crate) fn named(&self) -> Option<Named> {
Some(Named {
author: self.author_key?,
recovery: self.recovery_key,
rotations: self.rotations.clone(),
})
}
}
fn read_plugin(
storage: &dyn Storage,
path: &str,
device: &Device<'_>,
base: &str,
) -> Option<Plugin> {
let mut format = None;
let mut item = Item::new();
read_file(storage, path, &mut |event| match event {
Event::Value([key], value) if key == "format" => format = value.parse::<u32>().ok(),
Event::Close([plugin, rest @ ..]) if plugin == "plugin" && is_file(rest) => {
item.file_done();
}
Event::Close([plugin, rest @ ..]) if plugin == "plugin" && is_build(rest) => {
item.build_done(device);
}
Event::Close([plugin, shots, at])
if plugin == "plugin" && shots == "screenshots" && at == "[]" =>
{
item.shot_done();
}
Event::Close([plugin, rotations, at])
if plugin == "plugin" && rotations == "rotations" && at == "[]" =>
{
item.rotation_done();
}
Event::Value([plugin, rest @ ..], value) if plugin == "plugin" => {
item.read(rest, value, device);
}
_ => {}
});
if format != Some(FORMAT) || !item.fits(device) {
return None;
}
let mut files = core::mem::take(&mut item.files);
files.extend(item.code.clone());
for file in &mut files {
file.url = join_url(base, &file.url);
}
let author_key = key(&item.author_key);
let recovery_key = key(&item.recovery_key);
let rotations = core::mem::take(&mut item.rotations);
let code = item.code.is_some();
let mut more = core::mem::take(&mut item.more);
if author_key.is_some() {
more.author_key = item.author_key.trim().to_lowercase();
}
for shot in &mut more.shots {
shot.url = join_url(base, &shot.url);
}
let entry = item.finish(device);
let package = Package::new(entry.id.clone(), entry.name.clone()).files(files);
let whole = package.is_valid() && (!code || author_key.is_some());
whole.then_some(Plugin {
entry,
package,
author_key,
recovery_key,
rotations,
more,
})
}
fn file(name: &str) -> String {
format!("{}/{name}", Env::plugin_dir(PLUGIN.as_str()))
}
pub(crate) fn list_part() -> String {
file("list.part")
}
pub(crate) fn plugin_part(id: &str) -> String {
file(&format!("plugin-{id}.part"))
}
fn index_path() -> String {
file("index.txt")
}
fn head_path() -> String {
file("index.head")
}
fn offsets_path() -> String {
file("index.bin")
}
fn hits_path() -> String {
file("search.bin")
}
fn sections_path() -> String {
file("sections.txt")
}
const HEADER: &str = "store-index 5";
pub(crate) fn list_url(
list: &str,
device: &str,
language: &str,
community: bool,
abi: Option<Fingerprint>,
) -> String {
let join = if list.contains('?') { '&' } else { '?' };
let community = if community { "&community=1" } else { "" };
let abi = abi.map_or_else(String::new, |abi| format!("&abi={abi}"));
format!(
"{list}{join}device={}&lang={}{community}{abi}",
encode(device),
encode(language)
)
}
pub(crate) fn plugin_url(
list: &str,
device: &str,
language: &str,
id: &str,
community: bool,
abi: Option<Fingerprint>,
) -> String {
let list = list.split('#').next().unwrap_or(list);
let (address, query) = list.split_once('?').unwrap_or((list, ""));
let mut page = join_url(address, &format!("plugins/{}.json", encode(id)));
if !query.is_empty() {
page.push('?');
page.push_str(query);
}
list_url(&page, device, language, community, abi)
}
fn build(
storage: &dyn Storage,
json: &str,
[index, offsets, head, sections]: [&str; 4],
device: &Device<'_>,
) -> bool {
let mut runs: Vec<(usize, u32)> = Vec::new();
let mut lines = String::new();
let mut starts: Vec<u8> = Vec::new();
let mut offset = 0u32;
let mut written = storage.write(index, b"").is_ok() && storage.write(offsets, b"").is_ok();
let read = read_list(storage, json, device, &mut |entry| {
let full = runs.len() >= MAX_RUNS;
match runs.last_mut() {
Some((group, count)) if *group == entry.group || full => *count += 1,
_ => runs.push((entry.group, 1)),
}
let line = entry.line();
starts.extend_from_slice(&offset.to_le_bytes());
offset = offset.saturating_add(line.len() as u32);
lines.push_str(&line);
if lines.len() > WRITE_AT {
written &= storage.append(index, lines.as_bytes()).is_ok();
written &= storage.append(offsets, &starts).is_ok();
lines.clear();
starts.clear();
}
});
written &= storage.append(index, lines.as_bytes()).is_ok();
written &= storage.append(offsets, &starts).is_ok();
let read_whole = read.is_some();
let picks = read.unwrap_or_default();
let mut text = String::new();
for (key, ids) in SECTIONS.iter().zip(&picks.sections) {
text.push_str(&format!("{key}\t{}\n", ids.join(",")));
}
written &= storage.write(sections, text.as_bytes()).is_ok();
let runs: Vec<String> = runs
.iter()
.map(|(group, count)| format!("{group}:{count}"))
.collect();
let counts: Vec<String> = picks.sections.iter().map(|s| s.len().to_string()).collect();
let fair = if picks.fair { "fair" } else { "name" };
let text = format!(
"{HEADER}\t{}\t{fair}\t{}\n",
runs.join(","),
counts.join(",")
);
written &= storage.write(head, text.as_bytes()).is_ok();
if !read_whole || !written {
for path in [index, offsets, head, sections] {
let _ = storage.remove(path);
}
return false;
}
true
}
fn line_at(storage: &dyn Storage, path: &str, offset: u32) -> Option<String> {
let mut buf = [0u8; 256];
let mut at = u64::from(offset);
let mut line = Vec::new();
loop {
let n = storage.read_at(path, at, &mut buf).ok()?;
if n == 0 {
break;
}
if let Some(end) = buf[..n].iter().position(|&b| b == b'\n') {
line.extend_from_slice(&buf[..end]);
break;
}
line.extend_from_slice(&buf[..n]);
if line.len() > MAX_LINE {
return None;
}
at += n as u64;
}
String::from_utf8(line).ok()
}
fn number_at(storage: &dyn Storage, path: &str, n: usize) -> Option<u32> {
let mut bytes = [0u8; 4];
let read = storage.read_at(path, (n as u64) * 4, &mut bytes).ok()?;
(read == 4).then(|| u32::from_le_bytes(bytes))
}
#[derive(Debug, Default)]
pub(crate) struct Index {
runs: Vec<(usize, usize)>,
fair: bool,
sections: [usize; SECTIONS.len()],
}
impl Index {
pub(crate) fn kept(storage: &dyn Storage) -> Option<Index> {
Self::open(storage, &head_path())
}
fn open(storage: &dyn Storage, head: &str) -> Option<Index> {
let text = line_at(storage, head, 0)?;
let mut fields = text.split('\t');
if fields.next() != Some(HEADER) {
return None;
}
let runs = fields.next().unwrap_or("");
let fair = fields.next() == Some("fair");
let mut sections = [0; SECTIONS.len()];
for (count, text) in sections
.iter_mut()
.zip(fields.next().unwrap_or("").split(','))
{
*count = text.parse().unwrap_or(0);
}
let runs = runs
.split(',')
.filter(|run| !run.is_empty())
.filter_map(|run| {
let (group, count) = run.split_once(':')?;
Some((group.parse().ok()?, count.parse().ok()?))
})
.filter(|&(group, _)| group < GROUPS.len())
.collect();
Some(Index {
runs,
fair,
sections,
})
}
pub(crate) fn fair(&self) -> bool {
self.fair
}
pub(crate) fn section_len(&self, section: usize) -> usize {
self.sections.get(section).copied().unwrap_or(0)
}
pub(crate) fn section(&self, storage: &dyn Storage, section: usize) -> Vec<String> {
let Some(key) = SECTIONS.get(section) else {
return Vec::new();
};
let mut ids = Vec::new();
let mut at = 0u32;
while let Some(line) = line_at(storage, §ions_path(), at) {
at += line.len() as u32 + 1;
if let Some(list) = line.strip_prefix(key).and_then(|l| l.strip_prefix('\t')) {
ids.extend(
list.split(',')
.filter(|id| !id.is_empty())
.map(String::from),
);
break;
}
if line.is_empty() {
break;
}
}
ids
}
pub(crate) fn orders(&self) -> &'static [Order] {
if self.fair {
&[Order::Fair, Order::Name, Order::Added, Order::Updated]
} else {
&[Order::Name, Order::Added, Order::Updated]
}
}
pub(crate) fn order(&self, key: &str) -> Order {
let orders = self.orders();
orders
.iter()
.copied()
.find(|o| o.key() == key)
.unwrap_or(orders[0])
}
pub(crate) fn runs(&self) -> &[(usize, usize)] {
&self.runs
}
#[cfg(test)]
pub(crate) fn len(&self) -> usize {
self.runs.iter().map(|(_, count)| count).sum()
}
pub(crate) fn entry(&self, storage: &dyn Storage, n: usize) -> Option<Entry> {
let offset = number_at(storage, &offsets_path(), n)?;
Entry::parse(&line_at(storage, &index_path(), offset)?)
}
pub(crate) fn search(&self, storage: &dyn Storage, query: &str) -> usize {
let query = query.trim().to_lowercase();
self.pick(
storage,
|_, entry| {
(entry.name.to_lowercase().contains(&query)
|| entry.summary.to_lowercase().contains(&query))
.then_some(0)
},
Order::Fair,
usize::MAX,
)
}
pub(crate) fn scan(&self, storage: &dyn Storage, found: &mut dyn FnMut(usize, Entry)) {
let path = index_path();
let mut buf = [0u8; 512];
let mut line: Vec<u8> = Vec::new();
let (mut at, mut n) = (0u64, 0usize);
while let Ok(read) = storage.read_at(&path, at, &mut buf) {
if read == 0 {
break;
}
at += read as u64;
for &byte in &buf[..read] {
if byte != b'\n' {
if line.len() <= MAX_LINE {
line.push(byte);
}
continue;
}
if let Some(entry) = core::str::from_utf8(&line).ok().and_then(Entry::parse) {
found(n, entry);
}
line.clear();
n += 1;
}
}
}
pub(crate) fn pick(
&self,
storage: &dyn Storage,
mut keep: impl FnMut(usize, &Entry) -> Option<u32>,
order: Order,
most: usize,
) -> usize {
let hits = hits_path();
let mut out = Output::new(storage, &hits);
if order == Order::Fair || (order == Order::Name && !self.fair) {
self.scan(storage, &mut |n, entry| {
if out.count < most && keep(n, &entry).is_some() {
out.push(n as u32);
}
});
return out.finish();
}
let key = |entry: &Entry, rank: u32| match order {
Order::Added => u64::from(entry.added),
Order::Updated => u64::from(entry.updated),
Order::Name => u64::MAX - name_key(&entry.name),
Order::Fair | Order::Rank => u64::from(rank),
};
if most <= RUN {
let mut best: Vec<Keyed> = Vec::new();
self.scan(storage, &mut |n, entry| {
if let Some(rank) = keep(n, &entry) {
let item = (key(&entry, rank), n as u32);
let at = best.partition_point(|b| before(*b, item));
if at < most {
if best.len() == most {
best.pop();
}
best.insert(at, item);
}
}
});
for (_, n) in best {
out.push(n);
}
return out.finish();
}
let runs_path = file("sort.bin");
let mut runs: Vec<u32> = Vec::new();
let mut run: Vec<Keyed> = Vec::new();
let mut kept = storage.write(&runs_path, b"").is_ok();
let mut flush = |run: &mut Vec<Keyed>, runs: &mut Vec<u32>| {
run.sort_by(|a, b| order_of(*a, *b));
let mut bytes = Vec::with_capacity(run.len() * KEYED);
for (key, n) in run.iter() {
bytes.extend_from_slice(&key.to_le_bytes());
bytes.extend_from_slice(&n.to_le_bytes());
}
kept &= storage.append(&runs_path, &bytes).is_ok();
runs.push(run.len() as u32);
run.clear();
};
self.scan(storage, &mut |n, entry| {
if let Some(rank) = keep(n, &entry) {
run.push((key(&entry, rank), n as u32));
if run.len() == RUN {
flush(&mut run, &mut runs);
}
}
});
if !run.is_empty() {
flush(&mut run, &mut runs);
}
drop(run);
if !kept {
let _ = storage.remove(&runs_path);
return 0;
}
let mut cursors: Vec<Cursor> = Vec::with_capacity(runs.len());
let mut at = 0u64;
for len in runs {
cursors.push(Cursor::new(at, len));
at += u64::from(len) * KEYED as u64;
}
while out.count < most {
let mut first: Option<(usize, Keyed)> = None;
for (i, cursor) in cursors.iter_mut().enumerate() {
if let Some(item) = cursor.head(storage, &runs_path)
&& first.is_none_or(|(_, best)| before(item, best))
{
first = Some((i, item));
}
}
let Some((i, (_, n))) = first else {
break;
};
cursors[i].next();
out.push(n);
}
let _ = storage.remove(&runs_path);
out.finish()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Order {
Fair,
Name,
Added,
Updated,
Rank,
}
impl Order {
pub(crate) fn key(self) -> &'static str {
match self {
Order::Fair => "fair",
Order::Name => "name",
Order::Added => "added",
Order::Updated => "updated",
Order::Rank => "rank",
}
}
}
const KEY_CHARS: u32 = 12;
pub(crate) fn name_key(name: &str) -> u64 {
let mut key = 0u64;
let mut taken = 0;
for value in name
.chars()
.flat_map(char::to_lowercase)
.filter_map(sort_value)
{
if taken == KEY_CHARS {
break;
}
key = key * 38 + value;
taken += 1;
}
key * 38u64.pow(KEY_CHARS - taken)
}
fn sort_value(c: char) -> Option<u64> {
let c = match c {
'à'..='å' | 'æ' | 'ā' | 'ą' => 'a',
'ç' | 'ć' | 'č' => 'c',
'è'..='ë' | 'ē' | 'ę' | 'ě' => 'e',
'ì'..='ï' => 'i',
'ł' => 'l',
'ñ' | 'ń' | 'ň' => 'n',
'ò'..='ö' | 'ø' | 'œ' => 'o',
'ř' => 'r',
'ß' | 'ś' | 'š' => 's',
'ù'..='ü' | 'ů' => 'u',
'ý' | 'ÿ' => 'y',
'ź' | 'ż' | 'ž' => 'z',
c => c,
};
match c {
'0'..='9' => Some(1 + u64::from(c) - u64::from('0')),
'a'..='z' => Some(11 + u64::from(c) - u64::from('a')),
c if c.is_alphanumeric() => Some(37),
_ => None,
}
}
pub(crate) fn hit(storage: &dyn Storage, n: usize) -> Option<usize> {
number_at(storage, &hits_path(), n).map(|n| n as usize)
}
pub(crate) fn take(storage: &dyn Storage, fetched: &str, device: &Device<'_>) -> Option<Index> {
let fresh = [index_path(), offsets_path(), sections_path(), head_path()]
.map(|path| format!("{path}.part"));
let built = build(
storage,
fetched,
[&fresh[0], &fresh[1], &fresh[3], &fresh[2]],
device,
);
let _ = storage.remove(fetched);
if !built {
return None;
}
let _ = storage.remove(&head_path());
for (from, to) in fresh
.iter()
.zip([index_path(), offsets_path(), sections_path(), head_path()])
{
if paperpaw_sdk::files::move_to(storage, from, &to).is_err() {
return None;
}
}
let _ = storage.remove(&hits_path());
Index::kept(storage)
}
pub(crate) fn plugin(
storage: &dyn Storage,
fetched: &str,
address: &str,
device: &Device<'_>,
) -> Option<Plugin> {
let read = read_plugin(storage, fetched, device, address);
let _ = storage.remove(fetched);
read
}
#[cfg(test)]
mod tests;