rdf_fusion_encoding/encoding.rs
1use crate::EncodingName;
2use datafusion::arrow::array::{Array, ArrayRef};
3use datafusion::arrow::datatypes::DataType;
4use datafusion::common::{ScalarValue, exec_err};
5use datafusion::logical_expr::ColumnarValue;
6use rdf_fusion_common::DFResult;
7use rdf_fusion_common::ThinResult;
8use std::fmt::Debug;
9use std::sync::Arc;
10
11/// Represents an Arrow [Array] with a specific [TermEncoding].
12///
13/// The constructors of types that implement [EncodingArray] are meant to ensure that the
14/// [ArrayRef] upholds all invariants of the encoding.
15pub trait EncodingArray: Clone {
16 /// The encoding used by this array.
17 type Encoding: TermEncoding;
18
19 /// Obtains the encoding instance for this array.
20 fn encoding(&self) -> &Arc<Self::Encoding>;
21
22 /// Returns a reference to the inner array.
23 fn inner(&self) -> &ArrayRef;
24
25 /// Consumes `self` and returns the inner array.
26 fn into_array_ref(self) -> ArrayRef;
27
28 /// Extracts a scalar from this array at `quad_table`.
29 ///
30 /// Returns an error if the `quad_table` is out of bounds.
31 fn try_as_scalar(
32 &self,
33 index: usize,
34 ) -> DFResult<<Self::Encoding as TermEncoding>::Scalar> {
35 let scalar = ScalarValue::try_from_array(self.inner(), index)?;
36 self.encoding().try_new_scalar(scalar)
37 }
38}
39
40/// Represents an Arrow [ScalarValue] with a specific [TermEncoding].
41///
42/// The constructors of types that implement [EncodingScalar] are meant to ensure that the
43/// [ScalarValue] upholds all invariants of the encoding.
44pub trait EncodingScalar: Clone {
45 /// The encoding used by this scalar.
46 type Encoding: TermEncoding;
47
48 /// Obtains the encoding instance for this scalar.
49 fn encoding(&self) -> &Arc<Self::Encoding>;
50
51 /// Returns a reference to the inner scalar value.
52 fn scalar_value(&self) -> &ScalarValue;
53
54 /// Consumes `self` and returns the inner scalar value.
55 fn into_scalar_value(self) -> ScalarValue;
56
57 /// Produces a new array with `number_of_rows`.
58 fn to_array(
59 &self,
60 number_of_rows: usize,
61 ) -> DFResult<<Self::Encoding as TermEncoding>::Array> {
62 let array = self.scalar_value().to_array_of_size(number_of_rows)?;
63 self.encoding().try_new_array(array)
64 }
65}
66
67/// A term encoding defines how RDF terms are represented in Arrow arrays.
68///
69/// Each encoding defines a [DataType] that is uses for encoding RDF terms, while also having a
70/// wrapper [Self::Array] and [Self::Scalar] for Arrow arrays and scalars.
71///
72/// Different term encodings usually have different purposes and may only be valid for certain
73/// operations. For example, the [`TypedFamilyEncoding`] cannot be used to perform arbitrary join
74/// operations as it does not retain the lexical value of the RDF literals. On the other hand,
75/// the [`TypedFamilyEncoding`] will outperform the [`PlainTermEncoding`] for nested numerical
76/// operations as the parsing and validation of numeric literals is only done once. It is up to the
77/// user to ensure the correct use.
78///
79/// # Invariants
80///
81/// RDF Fusion assumes some invariants on any encoding:
82/// - Regular equality (with null being equal to nothing) implements [sameTerm](https://www.w3.org/TR/sparql11-query/#func-sameTerm).
83///
84/// [`TypedFamilyEncoding`]: crate::typed_family::TypedFamilyEncoding
85/// [`PlainTermEncoding`]: crate::plain_term::PlainTermEncoding
86pub trait TermEncoding: Debug + Send + Sync {
87 /// Represents a wrapper for Arrow arrays of this encoding. This can be used in
88 /// conjunction with [TermDecoder] to obtain the values from an Arrow array.
89 type Array: EncodingArray<Encoding = Self>;
90 /// Represents a wrapper for Arrow scalars of this encoding. This can be used in
91 /// conjunction with [TermDecoder] to obtain the values from an Arrow scalar.
92 type Scalar: EncodingScalar<Encoding = Self>;
93
94 /// Returns the name of the encoding.
95 fn name(&self) -> EncodingName;
96
97 /// Returns the [DataType] that is used for this encoding.
98 ///
99 /// This function depends on the instance of an encoding, as some encodings can be configured
100 /// such that the data type changes (at least in the future). Some encodings also expose a
101 /// statically known data type (e.g., [PlainTermEncoding::data_type](crate::plain_term::PlainTermEncoding::data_type)).
102 fn data_type(&self) -> &DataType;
103
104 /// Checks whether `array` contains a value with the correct encoding (i.e., type and possibly
105 /// metadata checks). If yes, returns an instance of [Self::Array]. Otherwise, an error is
106 /// returned.
107 fn try_new_array(self: &Arc<Self>, array: ArrayRef) -> DFResult<Self::Array>;
108
109 /// Checks whether `scalar` contains a value with the correct encoding (i.e., type and possibly
110 /// metadata checks). If yes, returns an instance of [Self::Scalar]. Otherwise, an error is
111 /// returned.
112 fn try_new_scalar(self: &Arc<Self>, scalar: ScalarValue) -> DFResult<Self::Scalar>;
113
114 /// Checks whether `value` contains a value with the correct encoding (i.e., type and possibly
115 /// metadata checks). If yes, returns a datum that either wraps an array or a scalar. Otherwise,
116 /// an error is returned.
117 fn try_new_datum(
118 self: &Arc<Self>,
119 value: ColumnarValue,
120 number_rows: usize,
121 ) -> DFResult<EncodingDatum<Self>> {
122 let datum = match value {
123 ColumnarValue::Array(array) => {
124 if array.len() != number_rows {
125 return exec_err!(
126 "Unexpected array size. Expected {number_rows}, Actual: {}",
127 array.len()
128 );
129 }
130 EncodingDatum::Array(self.try_new_array(array)?)
131 }
132 ColumnarValue::Scalar(scalar) => {
133 EncodingDatum::Scalar(self.try_new_scalar(scalar)?)
134 }
135 };
136 Ok(datum)
137 }
138}
139
140/// Allows extracting an iterator of a type from an [EncodingArray].
141///
142/// This allows uesrs to access the inner values of an RDF term array. It allows one to
143/// obtain a typed iterator over the RDF terms in the array. A decoder is specialized for one
144/// encoding and one value type ([Self::Term]).
145///
146/// ### Compatibility
147///
148/// Decoders are allowed to only support a subset of the encoded RDF terms. For example, a decoder
149/// for boolean values may produce an error if it encounters a literal with a different type.
150/// However, it is recommended that there is one decoder per [TermEncoding] that allows users to
151/// extract all RDF terms.
152///
153/// ### Performance
154///
155/// Using a [TermDecoder] for accessing the array, performing an operation on [Self::Term], and then
156/// re-encoding the resulting value using a [TermEncoder] may incur a performance penalty. However,
157/// we hope that this impact can be mitigated by compiler optimizations. We have yet to benchmark
158/// this impact to make a founded recommendation of when to use decoders and encoders. Users are
159/// free to directly work on the Arrow arrays to side-step the typed Encoding/Decoding machinery.
160pub trait TermDecoder<TEncoding: TermEncoding + ?Sized>: Debug + Sync + Send {
161 /// The resulting value type of decoding an RDF term.
162 type Term<'data>;
163
164 /// Allows extracting an iterator over all RDF terms in `array` that are _compatible_ with this
165 /// decoder (see [TermDecoder] for more information).
166 ///
167 /// The creation of the iterator cannot fail by itself, as the invariants of the encodings
168 /// should have been checked while creating `array`. However, the iterator may return an error
169 /// on every new value. This could be due to the value being incompatible with the decoder.
170 fn decode_terms(
171 array: &TEncoding::Array,
172 ) -> impl Iterator<Item = ThinResult<Self::Term<'_>>>;
173
174 /// Allows extracting an iterator over all RDF terms in `array` that are _compatible_ with this
175 /// decoder (see [TermDecoder] for more information).
176 ///
177 /// The creation of the value can fail if the value stored in the `scalar` is incompatible with
178 /// this decoder.
179 fn decode_term(scalar: &TEncoding::Scalar) -> ThinResult<Self::Term<'_>>;
180}
181
182/// Allows encoding an iterator of a type into an [EncodingArray].
183///
184/// This allows users to encode values in an RDF term array. An encoder is specialized for
185/// one encoding and one value type ([Self::Term]). The value type may only represent a subset of
186/// all valid RDF terms (e.g., only Boolean values). However, it is recommended that there is
187/// one decoder per [TermEncoding] that allows users to encode all RDF terms.
188///
189/// ### Performance
190///
191/// Using a [TermDecoder] for accessing the array, performing an operation on [Self::Term], and then
192/// re-encoding the resulting value using a [TermEncoder] may incur a performance penalty. However,
193/// we hope that this impact can be mitigated by compiler optimizations. We have yet to benchmark
194/// this impact to make a founded recommendation of when to use decoders and encoders. Users are
195/// free to directly work on the Arrow arrays to side-step the typed Encoding/Decoding machinery.
196pub trait TermEncoder<TEncoding: TermEncoding + ?Sized>: Debug + Sync + Send {
197 /// The value type that is being encoded.
198 type Term<'data>;
199
200 /// Allows encoding an iterator over RDF terms in an Arrow array.
201 fn encode_terms<'data>(
202 &self,
203 terms: impl IntoIterator<Item = ThinResult<Self::Term<'data>>>,
204 ) -> DFResult<TEncoding::Array>;
205
206 /// Allows encoding a scalar RDF term in an Arrow scalar.
207 fn encode_term(
208 &self,
209 term: ThinResult<Self::Term<'_>>,
210 ) -> DFResult<TEncoding::Scalar>;
211}
212
213/// Represents either an array or a scalar for a given encoding.
214#[derive(Clone)]
215pub enum EncodingDatum<TEncoding: TermEncoding + ?Sized> {
216 /// An array underlies this datum.
217 Array(TEncoding::Array),
218 /// A scalar underlies this datum.
219 Scalar(TEncoding::Scalar),
220}
221
222impl<TEncoding: TermEncoding + ?Sized> EncodingDatum<TEncoding> {
223 /// Returns the encoding of this datum.
224 pub fn encoding(&self) -> &Arc<TEncoding> {
225 match self {
226 EncodingDatum::Array(array) => array.encoding(),
227 EncodingDatum::Scalar(scalar) => scalar.encoding(),
228 }
229 }
230
231 /// Creates an array for this datum.
232 pub fn to_array(&self, number_rows: usize) -> TEncoding::Array {
233 match self {
234 EncodingDatum::Array(array) => array.clone(),
235 EncodingDatum::Scalar(scalar) => scalar.to_array(number_rows).unwrap(),
236 }
237 }
238}