s4_codec/
lib.rs

1//! S4 圧縮 codec layer。バックエンドを差し替え可能にする中立 trait を提供する。
2//!
3//! ## 採用 backend (2026-05 検討)
4//!
5//! - **nvCOMP** (NVIDIA proprietary、要 license 確認): Bitcomp / gANS / zstd-GPU
6//! - **DietGPU** (Meta, MIT): ANS-only、license clean な fallback
7//! - **CPU zstd**: GPU 無し環境向け究極の fallback / test bed
8
9use std::str::FromStr;
10
11use bytes::Bytes;
12use serde::{Deserialize, Serialize};
13use thiserror::Error;
14
15pub mod cpu_zstd;
16pub mod dietgpu;
17pub mod dispatcher;
18#[cfg(feature = "nvcomp-gpu")]
19mod ferro_compress;
20pub mod index;
21pub mod multipart;
22pub mod nvcomp;
23pub mod passthrough;
24pub mod registry;
25
26pub use registry::CodecRegistry;
27
28/// 圧縮 codec の種類 (manifest に記録、後段の decompress で codec を確定するために使う)
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
30#[serde(rename_all = "kebab-case")]
31pub enum CodecKind {
32    Passthrough,
33    NvcompBitcomp,
34    NvcompGans,
35    NvcompZstd,
36    DietGpuAns,
37    CpuZstd,
38    /// nvCOMP GDeflate (v0.2 #9). DEFLATE-family GPU codec; output bytes are
39    /// NOT gzip-compatible at the wire level (different framing) but the
40    /// algorithm-level format aligns with stock DEFLATE/zlib decoders given
41    /// the right wrapper.
42    NvcompGDeflate,
43}
44
45impl CodecKind {
46    pub fn as_str(self) -> &'static str {
47        match self {
48            Self::Passthrough => "passthrough",
49            Self::NvcompBitcomp => "nvcomp-bitcomp",
50            Self::NvcompGans => "nvcomp-gans",
51            Self::NvcompZstd => "nvcomp-zstd",
52            Self::DietGpuAns => "dietgpu-ans",
53            Self::CpuZstd => "cpu-zstd",
54            Self::NvcompGDeflate => "nvcomp-gdeflate",
55        }
56    }
57
58    /// 安定 numeric ID。`s4-codec/multipart.rs` の frame header に書き込む際に使う。
59    /// ⚠️ **この値は wire format の一部** — 既存値の変更禁止 (新 codec は新 ID を割当)。
60    pub fn id(self) -> u32 {
61        match self {
62            Self::Passthrough => 0,
63            Self::CpuZstd => 1,
64            Self::NvcompZstd => 2,
65            Self::NvcompBitcomp => 3,
66            Self::NvcompGans => 4,
67            Self::DietGpuAns => 5,
68            Self::NvcompGDeflate => 6,
69        }
70    }
71
72    pub fn from_id(id: u32) -> Option<Self> {
73        Some(match id {
74            0 => Self::Passthrough,
75            1 => Self::CpuZstd,
76            2 => Self::NvcompZstd,
77            3 => Self::NvcompBitcomp,
78            4 => Self::NvcompGans,
79            5 => Self::DietGpuAns,
80            6 => Self::NvcompGDeflate,
81            _ => return None,
82        })
83    }
84}
85
86#[derive(Debug, thiserror::Error)]
87#[error("unknown codec kind: {0}")]
88pub struct ParseCodecKindError(String);
89
90impl FromStr for CodecKind {
91    type Err = ParseCodecKindError;
92    fn from_str(s: &str) -> Result<Self, Self::Err> {
93        Ok(match s {
94            "passthrough" => Self::Passthrough,
95            "nvcomp-bitcomp" => Self::NvcompBitcomp,
96            "nvcomp-gans" => Self::NvcompGans,
97            "nvcomp-zstd" => Self::NvcompZstd,
98            "dietgpu-ans" => Self::DietGpuAns,
99            "cpu-zstd" => Self::CpuZstd,
100            "nvcomp-gdeflate" => Self::NvcompGDeflate,
101            other => return Err(ParseCodecKindError(other.into())),
102        })
103    }
104}
105
106/// 圧縮済 chunk のメタ情報。S3 オブジェクトの metadata に格納される。
107#[derive(Debug, Clone, Serialize, Deserialize)]
108pub struct ChunkManifest {
109    pub codec: CodecKind,
110    pub original_size: u64,
111    pub compressed_size: u64,
112    pub crc32c: u32,
113}
114
115/// codec 操作のエラー型。`anyhow::Error` ではなく専用型にすることで、上位 (S4Service) が
116/// HTTP エラーコードを意味的に出し分けやすくする。
117#[derive(Debug, Error)]
118pub enum CodecError {
119    #[error("codec mismatch: expected {expected:?}, got {got:?}")]
120    CodecMismatch { expected: CodecKind, got: CodecKind },
121
122    #[error("crc32c mismatch (chunk corruption?): expected {expected:#010x}, got {got:#010x}")]
123    CrcMismatch { expected: u32, got: u32 },
124
125    #[error("compressed size mismatch: manifest says {expected} bytes, payload is {got} bytes")]
126    SizeMismatch { expected: u64, got: u64 },
127
128    #[error("compression backend error: {0}")]
129    Backend(#[from] anyhow::Error),
130
131    #[error("io error: {0}")]
132    Io(#[from] std::io::Error),
133
134    #[error("blocking-task join error: {0}")]
135    Join(#[from] tokio::task::JoinError),
136
137    #[error("codec {0:?} is not registered in this CodecRegistry")]
138    UnregisteredCodec(CodecKind),
139}
140
141/// pluggable な圧縮 backend trait。
142///
143/// すべて async — GPU codec は CUDA stream に await でき、CPU codec は
144/// `spawn_blocking` で別スレッドへ逃がす。
145#[async_trait::async_trait]
146pub trait Codec: Send + Sync {
147    /// この実装が提供する codec の種類
148    fn kind(&self) -> CodecKind;
149
150    /// 圧縮: 入力 bytes → 圧縮済 bytes + manifest
151    async fn compress(&self, input: Bytes) -> Result<(Bytes, ChunkManifest), CodecError>;
152
153    /// 解凍: 圧縮済 bytes + manifest → 元の bytes
154    async fn decompress(&self, input: Bytes, manifest: &ChunkManifest)
155    -> Result<Bytes, CodecError>;
156}
157
158pub use dispatcher::CodecDispatcher;
s4_codec/lib.rs

s4_codec/
lib.rs
