1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
//! `cavs` — CLI for the CAVS-1 content-addressable video packaging format.
//!
//! Converts videos into `.cavs` (via ffmpeg CMAF/fMP4 segmentation),
//! reconstructs them back to playable MP4/HLS, inspects, verifies and plays.
mod ffmpeg;
mod pack;
mod report;
mod store;
mod unpack;
use anyhow::Result;
use clap::{Parser, Subcommand, ValueEnum};
use std::path::PathBuf;
#[derive(Parser)]
#[command(
name = "cavs",
version,
about = "CAVS — content-addressable, deduplicated packaging",
long_about = "CAVS packages files, game builds or video into .cavs: deduplicated \
FastCDC chunks, zstd-compressed and verifiable (BLAKE3 + Merkle + \
optional Ed25519 signature). Served by cavs-server, a client with a \
cache downloads only the bytes it doesn't already have.",
after_help = "EXAMPLES:\n \
cavs pack --raw build_v42.pck -o v42.cavs # a game release\n \
cavs pack --raw --sign-key pub.key data/* -o r.cavs # signed\n \
cavs pack movie.mp4 -o movie.cavs # video (segmented via ffmpeg)\n \
cavs info v42.cavs # structure and dedupe\n \
cavs verify v42.cavs --pubkey pub.key.pub # integrity + signature\n \
cavs unpack v42.cavs -o restored/ # exact reconstruction\n\n\
To serve and update clients: cavs-server / cavs-client --help"
)]
struct Cli {
#[command(subcommand)]
command: Command,
}
#[derive(Clone, Copy, Debug, ValueEnum)]
pub enum ChunkModeArg {
/// Fixed 256 KiB chunks (default for packaged media segments).
Fixed,
/// FastCDC 64/256/1024 KiB (default for raw assets).
Cdc,
/// Aggressive FastCDC 16/64/256 KiB (screen content, very repetitive data).
Screen,
}
impl ChunkModeArg {
pub fn to_mode(self, chunk_size: Option<usize>) -> cavs_chunker::ChunkMode {
use cavs_chunker::ChunkMode;
match self {
ChunkModeArg::Fixed => ChunkMode::Fixed {
size: chunk_size.unwrap_or(256 * 1024),
},
ChunkModeArg::Cdc => match chunk_size {
Some(avg) => ChunkMode::Cdc {
min: (avg / 4).max(1024),
avg,
max: avg * 4,
},
None => ChunkMode::asset_default(),
},
ChunkModeArg::Screen => ChunkMode::screen_default(),
}
}
}
#[derive(Subcommand)]
enum Command {
/// Package files (--raw) or videos into a deduplicated .cavs.
///
/// With --raw it accepts any file (PCKs, bundles, binaries) and uses
/// FastCDC 64 KiB + zstd 3 (the configuration validated in benchmarks).
/// Without --raw it treats inputs as video: ffmpeg segments them into
/// CMAF/fMP4 and CAVS packages the segments.
Pack {
/// Input video files (or arbitrary files with --raw).
#[arg(required = true)]
inputs: Vec<PathBuf>,
/// Output .cavs path.
#[arg(short, long)]
output: PathBuf,
/// Pack raw file bytes without ffmpeg segmentation (any file type).
#[arg(long)]
raw: bool,
/// Target media segment duration in seconds (video mode).
#[arg(long, default_value_t = 4.0)]
segment_time: f64,
/// Chunking strategy for media/asset payloads.
#[arg(long, value_enum)]
mode: Option<ChunkModeArg>,
/// Chunk size in bytes (fixed size, or CDC average).
#[arg(long)]
chunk_size: Option<usize>,
/// Disable zstd compression of stored chunks.
#[arg(long)]
no_compress: bool,
/// zstd level for chunk storage/wire compression.
#[arg(long, default_value_t = 3)]
zstd_level: i32,
/// Force re-encode (H.264/AAC) instead of trying stream copy first.
#[arg(long)]
transcode: bool,
/// Sign the packed content with this Ed25519 secret key file
/// (as produced by `cavs keygen`).
#[arg(long)]
sign_key: Option<PathBuf>,
},
/// Reconstruct the original media from a .cavs file.
Unpack {
input: PathBuf,
/// Output directory.
#[arg(short, long)]
output: PathBuf,
/// Skip writing the combined progressive .mp4 per video track.
#[arg(long)]
no_mp4: bool,
},
/// Show structure, dedup and compression statistics of a .cavs file.
Info {
input: PathBuf,
/// Also list every segment.
#[arg(long)]
segments: bool,
/// Also list every chunk.
#[arg(long)]
chunks: bool,
},
/// Verify every chunk hash, the Merkle root and all section hashes.
Verify {
input: PathBuf,
/// Additionally require a valid content signature from this Ed25519
/// public key (64 hex chars, or a path to a .pub file).
#[arg(long)]
pubkey: Option<String>,
},
/// Generate an Ed25519 signing keypair: <output> (secret, hex) and
/// <output>.pub (public key, hex).
Keygen {
#[arg(short, long)]
output: PathBuf,
},
/// Reconstruct to a temp dir and play with ffplay.
Play { input: PathBuf },
/// Manage a global content-addressable store: ingest releases dedup'd
/// across all versions/titles, unpublish, garbage collect.
Store {
/// Store directory (created if missing).
dir: PathBuf,
#[command(subcommand)]
action: StoreAction,
},
}
#[derive(Subcommand)]
enum StoreAction {
/// Ingest a .cavs into the store, deduplicating its chunks.
Add {
/// Asset name (e.g. game_v42).
name: String,
/// The .cavs file to ingest.
cavs: PathBuf,
},
/// Unpublish an asset (chunks it uniquely held become reclaimable by gc).
Rm { name: String },
/// Remove zero-ref chunks that have been unreferenced for --grace seconds.
Gc {
#[arg(long, default_value_t = 0)]
grace: u64,
},
/// Show assets and storage savings.
Stat,
}
fn main() -> Result<()> {
let cli = Cli::parse();
match cli.command {
Command::Pack {
inputs,
output,
raw,
segment_time,
mode,
chunk_size,
no_compress,
zstd_level,
transcode,
sign_key,
} => {
let opts = pack::PackOptions {
segment_time,
mode,
chunk_size,
compress: !no_compress,
zstd_level,
force_transcode: transcode,
sign_key,
};
if raw {
pack::pack_raw(&inputs, &output, &opts)
} else {
pack::pack_video(&inputs, &output, &opts)
}
}
Command::Unpack {
input,
output,
no_mp4,
} => unpack::unpack(&input, &output, !no_mp4).map(|_| ()),
Command::Info {
input,
segments,
chunks,
} => report::info(&input, segments, chunks),
Command::Verify { input, pubkey } => report::verify(&input, pubkey.as_deref()),
Command::Keygen { output } => keygen(&output),
Command::Play { input } => unpack::play(&input),
Command::Store { dir, action } => match action {
StoreAction::Add { name, cavs } => store::add(&dir, &name, &cavs),
StoreAction::Rm { name } => store::remove(&dir, &name),
StoreAction::Gc { grace } => store::gc(&dir, grace),
StoreAction::Stat => store::stat(&dir),
},
}
}
fn keygen(output: &std::path::Path) -> Result<()> {
use rand_core::OsRng;
let key = ed25519_dalek::SigningKey::generate(&mut OsRng);
let secret_hex: String = key.to_bytes().iter().map(|b| format!("{b:02x}")).collect();
let public_hex: String = key
.verifying_key()
.to_bytes()
.iter()
.map(|b| format!("{b:02x}"))
.collect();
std::fs::write(output, format!("{secret_hex}\n"))?;
std::fs::write(output.with_extension("pub"), format!("{public_hex}\n"))?;
println!("secret : {} (keep private)", output.display());
println!("public : {}", output.with_extension("pub").display());
println!("pubkey : {public_hex}");
Ok(())
}