u64 encode_mode5(const u8* rgba, u8* out, bool use_pca) {
RGBA min_color, max_color;
if (use_pca) {
std::array<u8, 16> mask{};
Vec4 axis = pca_axis_rgb(rgba, mask.data(), 0, 16);
pca_endpoints_rgb_subset(rgba, mask.data(), 0, axis, min_color, max_color);
} else {
bbox_endpoints_rgba(rgba, min_color, max_color);
}
i32 alo = 255, ahi = 0;
for (u32 i = 0; i < 16; ++i) {
i32 a = rgba[i * 4 + 3];
alo = std::min(alo, a);
ahi = std::max(ahi, a);
}
u32 r0 = quantize(min_color.r, 7), g0 = quantize(min_color.g, 7), b0 = quantize(min_color.b, 7);
u32 r1 = quantize(max_color.r, 7), g1 = quantize(max_color.g, 7), b1 = quantize(max_color.b, 7);
u32 a0 = static_cast<u32>(clamp_i(alo, 255));
u32 a1 = static_cast<u32>(clamp_i(ahi, 255));
RGBA c0{static_cast<i32>(bc7_unquantize(r0, 7)), static_cast<i32>(bc7_unquantize(g0, 7)),
static_cast<i32>(bc7_unquantize(b0, 7)), static_cast<i32>(a0)};
RGBA c1{static_cast<i32>(bc7_unquantize(r1, 7)), static_cast<i32>(bc7_unquantize(g1, 7)),
static_cast<i32>(bc7_unquantize(b1, 7)), static_cast<i32>(a1)};
std::array<RGBA, 4> color_palette;
for (u32 idx = 0; idx < 4; ++idx) {
u32 w = BCN_WEIGHT_2[idx];
color_palette[idx] = {static_cast<i32>(bcn_interpolate(c0.r, c1.r, w)),
static_cast<i32>(bcn_interpolate(c0.g, c1.g, w)),
static_cast<i32>(bcn_interpolate(c0.b, c1.b, w)), 0};
}
std::array<u32, 4> alpha_palette{};
for (u32 idx = 0; idx < 4; ++idx) {
u32 w = BCN_WEIGHT_2[idx];
alpha_palette[idx] = bcn_interpolate(a0, a1, w);
}
std::array<u8, 16> color_indices{};
std::array<u8, 16> alpha_indices{};
u64 total_err = 0;
for (u32 i = 0; i < 16; ++i) {
auto px = pixel_at(rgba, i);
u64 best_color_err = std::numeric_limits<u64>::max();
u32 best_color_idx = 0;
for (u32 j = 0; j < 4; ++j) {
u64 e = colour_error_rgb(px, color_palette[j]);
if (e < best_color_err) {
best_color_err = e;
best_color_idx = j;
}
}
color_indices[i] = static_cast<u8>(best_color_idx);
u64 best_alpha_err = std::numeric_limits<u64>::max();
u32 best_alpha_idx = 0;
for (u32 j = 0; j < 4; ++j) {
u64 e = bcn_sq(px.a - static_cast<i32>(alpha_palette[j]));
if (e < best_alpha_err) {
best_alpha_err = e;
best_alpha_idx = j;
}
}
alpha_indices[i] = static_cast<u8>(best_alpha_idx);
total_err += best_color_err + best_alpha_err;
}
if (color_indices[0] >= 2) {
std::swap(r0, r1);
std::swap(g0, g1);
std::swap(b0, b1);
for (u32 i = 0; i < 16; ++i)
color_indices[i] = static_cast<u8>(3 - color_indices[i]);
}
if (alpha_indices[0] >= 2) {
std::swap(a0, a1);
for (u32 i = 0; i < 16; ++i)
alpha_indices[i] = static_cast<u8>(3 - alpha_indices[i]);
}
BitWriter writer;
writer.write(0b100000, 6);
writer.write(0, 2);
writer.write(r0, 7);
writer.write(r1, 7);
writer.write(g0, 7);
writer.write(g1, 7);
writer.write(b0, 7);
writer.write(b1, 7);
writer.write(a0, 8);
writer.write(a1, 8);
writer.write(color_indices[0], 1);
for (u32 i = 1; i < 16; ++i)
writer.write(color_indices[i], 2);
writer.write(alpha_indices[0], 1);
for (u32 i = 1; i < 16; ++i)
writer.write(alpha_indices[i], 2);
std::memcpy(out, writer.data.data(), 16);
return total_err;
}