s4_codec/
lib.rs

1//! S4 圧縮 codec layer。バックエンドを差し替え可能にする中立 trait を提供する。
2//!
3//! ## 採用 backend (2026-05 検討)
4//!
5//! - **nvCOMP** (NVIDIA proprietary、要 license 確認): Bitcomp / gANS / zstd-GPU
6//! - **DietGPU** (Meta, MIT): ANS-only、license clean な fallback
7//! - **CPU zstd**: GPU 無し環境向け究極の fallback / test bed
8
9use std::str::FromStr;
10
11use bytes::Bytes;
12use serde::{Deserialize, Serialize};
13use thiserror::Error;
14
15pub mod cpu_zstd;
16pub mod dietgpu;
17pub mod dispatcher;
18#[cfg(feature = "nvcomp-gpu")]
19mod ferro_compress;
20pub mod index;
21pub mod multipart;
22pub mod nvcomp;
23pub mod passthrough;
24pub mod registry;
25
26pub use registry::CodecRegistry;
27
28/// 圧縮 codec の種類 (manifest に記録、後段の decompress で codec を確定するために使う)
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
30#[serde(rename_all = "kebab-case")]
31pub enum CodecKind {
32    Passthrough,
33    NvcompBitcomp,
34    NvcompGans,
35    NvcompZstd,
36    DietGpuAns,
37    CpuZstd,
38}
39
40impl CodecKind {
41    pub fn as_str(self) -> &'static str {
42        match self {
43            Self::Passthrough => "passthrough",
44            Self::NvcompBitcomp => "nvcomp-bitcomp",
45            Self::NvcompGans => "nvcomp-gans",
46            Self::NvcompZstd => "nvcomp-zstd",
47            Self::DietGpuAns => "dietgpu-ans",
48            Self::CpuZstd => "cpu-zstd",
49        }
50    }
51
52    /// 安定 numeric ID。`s4-codec/multipart.rs` の frame header に書き込む際に使う。
53    /// ⚠️ **この値は wire format の一部** — 既存値の変更禁止 (新 codec は新 ID を割当)。
54    pub fn id(self) -> u32 {
55        match self {
56            Self::Passthrough => 0,
57            Self::CpuZstd => 1,
58            Self::NvcompZstd => 2,
59            Self::NvcompBitcomp => 3,
60            Self::NvcompGans => 4,
61            Self::DietGpuAns => 5,
62        }
63    }
64
65    pub fn from_id(id: u32) -> Option<Self> {
66        Some(match id {
67            0 => Self::Passthrough,
68            1 => Self::CpuZstd,
69            2 => Self::NvcompZstd,
70            3 => Self::NvcompBitcomp,
71            4 => Self::NvcompGans,
72            5 => Self::DietGpuAns,
73            _ => return None,
74        })
75    }
76}
77
78#[derive(Debug, thiserror::Error)]
79#[error("unknown codec kind: {0}")]
80pub struct ParseCodecKindError(String);
81
82impl FromStr for CodecKind {
83    type Err = ParseCodecKindError;
84    fn from_str(s: &str) -> Result<Self, Self::Err> {
85        Ok(match s {
86            "passthrough" => Self::Passthrough,
87            "nvcomp-bitcomp" => Self::NvcompBitcomp,
88            "nvcomp-gans" => Self::NvcompGans,
89            "nvcomp-zstd" => Self::NvcompZstd,
90            "dietgpu-ans" => Self::DietGpuAns,
91            "cpu-zstd" => Self::CpuZstd,
92            other => return Err(ParseCodecKindError(other.into())),
93        })
94    }
95}
96
97/// 圧縮済 chunk のメタ情報。S3 オブジェクトの metadata に格納される。
98#[derive(Debug, Clone, Serialize, Deserialize)]
99pub struct ChunkManifest {
100    pub codec: CodecKind,
101    pub original_size: u64,
102    pub compressed_size: u64,
103    pub crc32c: u32,
104}
105
106/// codec 操作のエラー型。`anyhow::Error` ではなく専用型にすることで、上位 (S4Service) が
107/// HTTP エラーコードを意味的に出し分けやすくする。
108#[derive(Debug, Error)]
109pub enum CodecError {
110    #[error("codec mismatch: expected {expected:?}, got {got:?}")]
111    CodecMismatch { expected: CodecKind, got: CodecKind },
112
113    #[error("crc32c mismatch (chunk corruption?): expected {expected:#010x}, got {got:#010x}")]
114    CrcMismatch { expected: u32, got: u32 },
115
116    #[error("compressed size mismatch: manifest says {expected} bytes, payload is {got} bytes")]
117    SizeMismatch { expected: u64, got: u64 },
118
119    #[error("compression backend error: {0}")]
120    Backend(#[from] anyhow::Error),
121
122    #[error("io error: {0}")]
123    Io(#[from] std::io::Error),
124
125    #[error("blocking-task join error: {0}")]
126    Join(#[from] tokio::task::JoinError),
127
128    #[error("codec {0:?} is not registered in this CodecRegistry")]
129    UnregisteredCodec(CodecKind),
130}
131
132/// pluggable な圧縮 backend trait。
133///
134/// すべて async — GPU codec は CUDA stream に await でき、CPU codec は
135/// `spawn_blocking` で別スレッドへ逃がす。
136#[async_trait::async_trait]
137pub trait Codec: Send + Sync {
138    /// この実装が提供する codec の種類
139    fn kind(&self) -> CodecKind;
140
141    /// 圧縮: 入力 bytes → 圧縮済 bytes + manifest
142    async fn compress(&self, input: Bytes) -> Result<(Bytes, ChunkManifest), CodecError>;
143
144    /// 解凍: 圧縮済 bytes + manifest → 元の bytes
145    async fn decompress(&self, input: Bytes, manifest: &ChunkManifest)
146    -> Result<Bytes, CodecError>;
147}
148
149pub use dispatcher::CodecDispatcher;
s4_codec/lib.rs

s4_codec/
lib.rs
