bittensor_core/codec/
mod.rs

1//! SCALE encode/decode against a [`crate::runtime::Runtime`].
2//!
3//! The decoded-value shapes are cyscale's, pinned by the shape corpus; the
4//! encoder accepts the same lenient inputs the SDK has always fed the codec
5//! (ss58 strings for account ids, 0x-hex for byte arrays, `{"Variant": ...}`
6//! dicts for enums, `None` for Option).
7
8pub mod batch;
9pub mod decode;
10pub mod encode;
11pub mod extrinsic;
12pub mod storage;
13pub mod value;
14
15pub use value::Value;
16
17#[cfg(test)]
18mod corpus_tests {
19    #![allow(clippy::unwrap_used, clippy::expect_used, clippy::indexing_slicing)]
20
21    use codec::Decode;
22
23    use crate::codec::value::Value;
24    use crate::runtime::type_string::TypeSpec;
25    use crate::runtime::Runtime;
26    use scale_info::{form::PortableForm, TypeDef};
27
28    fn json_u64(v: &serde_json::Value) -> u64 {
29        match v {
30            serde_json::Value::Number(n) => n.to_string().parse().unwrap(),
31            _ => panic!("fixture value {v} is not an unsigned integer"),
32        }
33    }
34
35    fn json_i64(v: &serde_json::Value) -> i64 {
36        match v {
37            serde_json::Value::Number(n) => n.to_string().parse().unwrap(),
38            _ => panic!("fixture value {v} is not a signed integer"),
39        }
40    }
41
42    fn as_u32(value: u64) -> u32 {
43        u32::try_from(value).expect("fixture integer fits u32")
44    }
45
46    fn as_u16(value: u64) -> u16 {
47        u16::try_from(value).expect("fixture integer fits u16")
48    }
49
50    fn as_u8(value: u64) -> u8 {
51        u8::try_from(value).expect("fixture integer fits u8")
52    }
53
54    /// JSON fixture data as codec input values (objects become string-keyed
55    /// dicts, matching the Python seam's params).
56    fn value_from_json(v: &serde_json::Value) -> Value {
57        match v {
58            serde_json::Value::Null => Value::Null,
59            serde_json::Value::Bool(b) => Value::Bool(*b),
60            serde_json::Value::Number(n) => {
61                let text = n.to_string();
62                if let Ok(i) = text.parse::<i128>() {
63                    Value::Int(i128::from(i))
64                } else if let Ok(u) = text.parse::<u128>() {
65                    Value::Uint(u)
66                } else {
67                    panic!("fixture number {n} is not an integer")
68                }
69            }
70            serde_json::Value::String(s) => Value::Str(s.clone()),
71            serde_json::Value::Array(items) => {
72                Value::List(items.iter().map(value_from_json).collect())
73            }
74            serde_json::Value::Object(map) => Value::Dict(
75                map.iter()
76                    .map(|(k, v)| (Value::Str(k.clone()), value_from_json(v)))
77                    .collect(),
78            ),
79        }
80    }
81
82    fn golden() -> serde_json::Value {
83        let path = concat!(
84            env!("CARGO_MANIFEST_DIR"),
85            "/../python/tests/fixtures/golden.json"
86        );
87        let raw = std::fs::read_to_string(path).expect("golden.json fixture exists");
88        serde_json::from_str(&raw).unwrap()
89    }
90
91    fn golden_metadata_v15() -> Vec<u8> {
92        let golden = golden();
93        let hex_str = golden["metadata"]["v15_hex"]
94            .as_str()
95            .expect("golden.json has metadata.v15_hex");
96        let raw = hex::decode(hex_str.trim_start_matches("0x")).unwrap();
97        Option::<Vec<u8>>::decode(&mut &raw[..])
98            .unwrap()
99            .expect("fixture metadata is Some")
100    }
101
102    fn runtime() -> Runtime {
103        let g = golden();
104        Runtime::parse(
105            &golden_metadata_v15(),
106            as_u32(json_u64(&g["network"]["spec_version"])),
107            as_u32(json_u64(&g["network"]["transaction_version"])),
108            as_u16(json_u64(&g["network"]["ss58_format"])),
109        )
110        .expect("golden metadata parses")
111    }
112
113    #[test]
114    fn singleton_tuple_sequence_accepts_flat_and_legacy_wrapped_values() {
115        #[derive(codec::Encode, scale_info::TypeInfo)]
116        struct Legs(Vec<(u16, u16, u64, u64)>);
117
118        let mut registry = scale_info::Registry::new();
119        let id = registry.register_type(&scale_info::meta_type::<Legs>()).id;
120        let mut rt = runtime();
121        rt.types = registry.into();
122        let flat = value_from_json(&serde_json::json!([[0, 2, 1_000_000_000u64, 0]]));
123        let wrapped = Value::List(vec![flat.clone()]);
124        let expected = codec::Encode::encode(&Legs(vec![(0, 2, 1_000_000_000, 0)]));
125        for value in [flat, wrapped] {
126            // A failed first attempt must not leave its sequence prefix behind.
127            let mut out = vec![99];
128            rt.encode_value(&TypeSpec::Id(id), &value, &mut out)
129                .unwrap();
130            assert_eq!(out, [vec![99], expected.clone()].concat());
131        }
132    }
133
134    #[test]
135    fn singleton_nested_sequence_preserves_its_inner_sequence() {
136        #[derive(codec::Encode, scale_info::TypeInfo)]
137        struct Rows(Vec<Vec<u16>>);
138
139        let mut registry = scale_info::Registry::new();
140        let id = registry.register_type(&scale_info::meta_type::<Rows>()).id;
141        let mut rt = runtime();
142        rt.types = registry.into();
143        for row in [vec![], vec![1], vec![1, 2]] {
144            let value = value_from_json(&serde_json::json!([row]));
145            assert_eq!(
146                rt.encode_spec(&TypeSpec::Id(id), &value).unwrap(),
147                codec::Encode::encode(&Rows(vec![row]))
148            );
149        }
150    }
151
152    /// The definition of done for the decoder: every recorded
153    /// `(type id, SCALE bytes, cyscale shape)` triple reproduces exactly.
154    #[test]
155    fn decoder_reproduces_the_shape_corpus() {
156        let path = concat!(
157            env!("CARGO_MANIFEST_DIR"),
158            "/../python/tests/fixtures/shape_corpus/corpus.json"
159        );
160        let raw = std::fs::read_to_string(path).expect("shape corpus recorded");
161        let corpus: serde_json::Value = serde_json::from_str(&raw).unwrap();
162        let rt = runtime();
163        assert_eq!(
164            as_u32(json_u64(&corpus["spec_version"])),
165            rt.spec_version,
166            "corpus and golden fixture must be recorded from the same runtime"
167        );
168
169        let mut failures: Vec<String> = Vec::new();
170        let mut checked = 0usize;
171        for ty in corpus["types"].as_array().unwrap() {
172            let id = as_u32(json_u64(&ty["id"]));
173            let name = ty["name"].as_str().unwrap();
174            for (i, sample) in ty["samples"].as_array().unwrap().iter().enumerate() {
175                let scale_hex = sample["scale_hex"].as_str().unwrap();
176                let data = hex::decode(scale_hex.trim_start_matches("0x")).unwrap();
177                let expected = &sample["decoded"];
178                checked += 1;
179                match rt.decode_spec(&TypeSpec::Id(id), &data, true) {
180                    Ok(value) => {
181                        let got = crate::codec::value::to_corpus_json(&value);
182                        if &got != expected {
183                            failures.push(format!(
184                                "type {id} {name} sample {i}:\n  bytes    {scale_hex}\n  expected {expected}\n  got      {got}"
185                            ));
186                        }
187                    }
188                    Err(e) => failures.push(format!(
189                        "type {id} {name} sample {i}: decode error {e} (bytes {scale_hex})"
190                    )),
191                }
192            }
193        }
194        assert!(
195            failures.is_empty(),
196            "{} of {} corpus samples diverged (first 20):\n{}",
197            failures.len(),
198            checked,
199            failures
200                .iter()
201                .take(20)
202                .cloned()
203                .collect::<Vec<_>>()
204                .join("\n")
205        );
206    }
207
208    /// Pathological nesting must surface as a codec error, never a stack
209    /// overflow: type specs derive from untrusted node metadata, and an
210    /// overflow aborts the process (it is not a catchable panic).
211    #[test]
212    fn pathological_nesting_errors_instead_of_overflowing() {
213        let rt = runtime();
214
215        // A 1-tuple recurses into its element without consuming bytes, so
216        // this spec models a self-referential metadata type. 1000 levels is
217        // far past the cap but shallow enough that the spec's own recursive
218        // Drop stays within the stack.
219        let mut spec = TypeSpec::Primitive(crate::runtime::type_string::Primitive::U8);
220        for _ in 0..1000 {
221            spec = TypeSpec::Tuple(vec![spec]);
222        }
223
224        let err = rt
225            .decode_spec(&spec, &[0u8], true)
226            .expect_err("deep nesting must not decode");
227        assert!(
228            err.to_string().contains("recursion"),
229            "unexpected error: {err}"
230        );
231
232        let mut value = Value::Int(0);
233        for _ in 0..1000 {
234            value = Value::Tuple(vec![value]);
235        }
236        let err = rt
237            .encode_spec(&spec, &value)
238            .expect_err("deep nesting must not encode");
239        assert!(
240            err.to_string().contains("recursion"),
241            "unexpected error: {err}"
242        );
243    }
244
245    /// The definition of done for compose_call: every golden call vector
246    /// (including the nested Sudo / Utility.batch / Proxy ones) reproduces
247    /// byte-identically.
248    #[test]
249    fn composed_calls_match_the_golden_vectors() {
250        let g = golden();
251        let rt = runtime();
252        let addr = |i: usize| g["ss58"][i]["address"].as_str().unwrap().to_string();
253        let (alice, bob, charlie) = (addr(0), addr(1), addr(2));
254
255        let transfer = |dest: &str, value: i128| {
256            rt.compose_call(
257                "Balances",
258                "transfer_keep_alive",
259                &Value::record(vec![
260                    ("dest".into(), Value::str(dest)),
261                    ("value".into(), Value::Int(value)),
262                ]),
263            )
264            .unwrap()
265        };
266
267        for case in g["calls"].as_array().unwrap() {
268            let module = case["module"].as_str().unwrap();
269            let function = case["function"].as_str().unwrap();
270            let composed = match module {
271                "Sudo" => {
272                    let inner = rt
273                        .compose_call(
274                            "System",
275                            "remark",
276                            &Value::record(vec![("remark".into(), Value::str("0xdeadbeef"))]),
277                        )
278                        .unwrap();
279                    rt.compose_call(
280                        "Sudo",
281                        "sudo",
282                        &Value::record(vec![("call".into(), Value::Bytes(inner))]),
283                    )
284                }
285                "Utility" => {
286                    let t1 = transfer(&bob, 1);
287                    let t2 = transfer(&charlie, 2);
288                    rt.compose_call(
289                        "Utility",
290                        "batch",
291                        &Value::record(vec![(
292                            "calls".into(),
293                            Value::List(vec![Value::Bytes(t1), Value::Bytes(t2)]),
294                        )]),
295                    )
296                }
297                "Proxy" => {
298                    let t1 = transfer(&bob, 1);
299                    rt.compose_call(
300                        "Proxy",
301                        "proxy",
302                        &Value::record(vec![
303                            ("real".into(), Value::str(alice.clone())),
304                            ("force_proxy_type".into(), Value::str("Transfer")),
305                            ("call".into(), Value::Bytes(t1)),
306                        ]),
307                    )
308                }
309                _ => rt.compose_call(module, function, &value_from_json(&case["params"])),
310            }
311            .unwrap_or_else(|e| panic!("{module}.{function} failed to compose: {e}"));
312            let expected = case["data_hex"].as_str().unwrap();
313            assert_eq!(
314                format!("0x{}", hex::encode(&composed)),
315                expected,
316                "{module}.{function} encoding diverged"
317            );
318        }
319    }
320
321    /// Every golden storage-key vector (plain, single map, double map,
322    /// account-keyed) reproduces byte-identically.
323    #[test]
324    fn storage_keys_match_the_golden_vectors() {
325        let g = golden();
326        let rt = runtime();
327        for case in g["storage_keys"].as_array().unwrap() {
328            let pallet = case["pallet"].as_str().unwrap();
329            let name = case["storage_function"].as_str().unwrap();
330            let entry = rt
331                .storage_entry(pallet, name)
332                .unwrap_or_else(|| panic!("{pallet}.{name} not found"));
333            let params: Vec<Value> = case["params"]
334                .as_array()
335                .unwrap()
336                .iter()
337                .map(value_from_json)
338                .collect();
339            let key = rt
340                .storage_key(entry, &params)
341                .unwrap_or_else(|e| panic!("{pallet}.{name} key failed: {e}"));
342            assert_eq!(
343                format!("0x{}", hex::encode(&key)),
344                case["key_hex"].as_str().unwrap(),
345                "{pallet}.{name} storage key diverged"
346            );
347        }
348    }
349
350    /// Map keys recover from raw storage keys the way the query_map fixtures
351    /// recorded them.
352    #[test]
353    fn map_keys_recover_from_the_golden_query_maps() {
354        let g = golden();
355        let rt = runtime();
356        for case in g["query_maps"].as_array().unwrap() {
357            let pallet = case["pallet"].as_str().unwrap();
358            let name = case["storage_function"].as_str().unwrap();
359            let entry = rt.storage_entry(pallet, name).unwrap();
360            let fixed = case["params"].as_array().unwrap().len();
361            let free = entry.key_types.len() - fixed;
362            for (raw_key, expected_pair) in case["raw_keys"]
363                .as_array()
364                .unwrap()
365                .iter()
366                .zip(case["pairs"].as_array().unwrap())
367            {
368                let key = hex::decode(raw_key.as_str().unwrap().trim_start_matches("0x")).unwrap();
369                let recovered = rt
370                    .decode_storage_key_params(entry, &key, fixed)
371                    .unwrap_or_else(|e| panic!("{pallet}.{name} key recovery failed: {e}"));
372                assert_eq!(recovered.len(), free);
373                // pairs are [key, value]; multi-component keys record a list.
374                let expected_key = &expected_pair[0];
375                let got = if free == 1 {
376                    crate::codec::value::to_corpus_json(&recovered[0])
377                } else {
378                    crate::codec::value::to_corpus_json(&Value::List(recovered))
379                };
380                assert_eq!(&got, expected_key, "{pallet}.{name} recovered key diverged");
381            }
382        }
383    }
384
385    fn h256(hex_str: &str) -> [u8; 32] {
386        hex::decode(hex_str.trim_start_matches("0x"))
387            .unwrap()
388            .try_into()
389            .unwrap()
390    }
391
392    fn golden_transfer(rt: &Runtime, g: &serde_json::Value) -> Vec<u8> {
393        rt.compose_call(
394            "Balances",
395            "transfer_keep_alive",
396            &Value::record(vec![
397                (
398                    "dest".into(),
399                    Value::str(g["ss58"][1]["address"].as_str().unwrap()),
400                ),
401                ("value".into(), Value::Int(12345)),
402            ]),
403        )
404        .unwrap()
405    }
406
407    /// Both golden signature payloads (immortal and mortal) reproduce
408    /// byte-identically, era phase computed from the current block.
409    #[test]
410    fn signature_payloads_match_the_golden_vectors() {
411        let g = golden();
412        let rt = runtime();
413        let call = golden_transfer(&rt, &g);
414        let genesis = h256(g["network"]["genesis_hash"].as_str().unwrap());
415
416        let immortal = &g["signature_payloads"][0];
417        let payload = rt
418            .signature_payload(
419                &call,
420                &crate::codec::extrinsic::TxParams {
421                    era: Value::str("00"),
422                    nonce: json_u64(&immortal["nonce"]),
423                    tip: 0,
424                    tip_asset_id: None,
425                    genesis_hash: genesis,
426                    era_block_hash: genesis,
427                    metadata_hash: None,
428                },
429            )
430            .unwrap();
431        assert_eq!(
432            format!("0x{}", hex::encode(&payload)),
433            immortal["payload_hex"].as_str().unwrap()
434        );
435
436        let mortal = &g["signature_payloads"][1];
437        let payload = rt
438            .signature_payload(
439                &call,
440                &crate::codec::extrinsic::TxParams {
441                    era: Value::record(vec![
442                        (
443                            "period".into(),
444                            Value::Int(i128::from(json_i64(&mortal["era"]["period"]))),
445                        ),
446                        (
447                            "current".into(),
448                            Value::Int(i128::from(json_i64(&mortal["era"]["current"]))),
449                        ),
450                    ]),
451                    nonce: json_u64(&mortal["nonce"]),
452                    tip: 0,
453                    tip_asset_id: None,
454                    genesis_hash: genesis,
455                    era_block_hash: h256(mortal["era_birth_block_hash"].as_str().unwrap()),
456                    metadata_hash: None,
457                },
458            )
459            .unwrap();
460        assert_eq!(
461            format!("0x{}", hex::encode(&payload)),
462            mortal["payload_hex"].as_str().unwrap()
463        );
464    }
465
466    /// Signed extrinsic assembly reproduces the golden extrinsic bytes and
467    /// hashes exactly.
468    #[test]
469    fn signed_extrinsics_match_the_golden_vectors() {
470        let g = golden();
471        let rt = runtime();
472        let call = golden_transfer(&rt, &g);
473        for case in g["extrinsics"].as_array().unwrap() {
474            let era = if case["era"] == "00" {
475                Value::str("00")
476            } else {
477                value_from_json(&case["era"])
478            };
479            let signature = hex::decode(
480                case["signature_hex"]
481                    .as_str()
482                    .unwrap()
483                    .trim_start_matches("0x"),
484            )
485            .unwrap();
486            let (data, hash) = rt
487                .encode_signed_extrinsic(
488                    &call,
489                    h256(case["public_key_hex"].as_str().unwrap()),
490                    &signature,
491                    as_u8(json_u64(&case["signature_version"])),
492                    &crate::codec::extrinsic::TxParams {
493                        era,
494                        nonce: json_u64(&case["nonce"]),
495                        tip: json_u64(&case["tip"]).into(),
496                        tip_asset_id: None,
497                        genesis_hash: [0; 32],
498                        era_block_hash: [0; 32],
499                        metadata_hash: None,
500                    },
501                )
502                .unwrap();
503            assert_eq!(
504                format!("0x{}", hex::encode(&data)),
505                case["extrinsic_hex"].as_str().unwrap()
506            );
507            assert_eq!(
508                format!("0x{}", hex::encode(hash)),
509                case["extrinsic_hash"].as_str().unwrap()
510            );
511        }
512    }
513
514    /// Raw block extrinsics decode into the exact dicts cyscale produced.
515    #[test]
516    fn extrinsics_decode_like_the_golden_block() {
517        let g = golden();
518        let rt = runtime();
519        let raws = g["block"]["raw"]["block"]["extrinsics"].as_array().unwrap();
520        let olds = g["block"]["decoded_extrinsics"].as_array().unwrap();
521        assert_eq!(raws.len(), olds.len());
522        for (raw, old) in raws.iter().zip(olds) {
523            let data = hex::decode(raw.as_str().unwrap().trim_start_matches("0x")).unwrap();
524            let decoded = rt.decode_extrinsic(&data, true).unwrap();
525            assert_eq!(&crate::codec::value::to_corpus_json(&decoded), old);
526        }
527    }
528
529    /// Multisig account derivation matches the golden fixtures (including the
530    /// unsorted-signatories case the derivation must normalize).
531    #[test]
532    fn multisig_accounts_match_the_golden_vectors() {
533        let g = golden();
534        let ss58_format = as_u16(json_u64(&g["network"]["ss58_format"]));
535        for case in g["multisig"].as_array().unwrap() {
536            let signatories: Vec<[u8; 32]> = case["signatories"]
537                .as_array()
538                .unwrap()
539                .iter()
540                .map(|s| crate::keys::public_key_from_ss58(s.as_str().unwrap()).unwrap())
541                .collect();
542            let threshold = as_u16(json_u64(&case["threshold"]));
543            let (account, sorted) =
544                crate::codec::extrinsic::multisig_account_id(&signatories, threshold).unwrap();
545            assert_eq!(
546                crate::keys::ss58_from_public(account, ss58_format),
547                case["ss58_address"].as_str().unwrap()
548            );
549            assert_eq!(sorted.len(), signatories.len());
550        }
551    }
552
553    /// Composed calls decode back into cyscale's call dict shape.
554    #[test]
555    fn composed_calls_decode_back() {
556        let g = golden();
557        let rt = runtime();
558        let case = &g["calls"].as_array().unwrap()[0];
559        let data =
560            hex::decode(case["data_hex"].as_str().unwrap().trim_start_matches("0x")).unwrap();
561        let decoded = rt.decode_spec(&TypeSpec::Call, &data, true).unwrap();
562        let json = crate::codec::value::to_corpus_json(&decoded);
563        assert_eq!(json["call_module"], "Balances");
564        assert_eq!(json["call_function"], "transfer_keep_alive");
565        assert_eq!(json["call_args"][0]["value"], g["ss58"][1]["address"]);
566    }
567
568    /// A `BoundedVec` whose element type matches.
569    fn bounded_vec_of(rt: &Runtime, elem: impl Fn(&scale_info::Type<PortableForm>) -> bool) -> u32 {
570        rt.types
571            .types
572            .iter()
573            .find(|t| {
574                let TypeDef::Composite(c) = &t.ty.type_def else {
575                    return false;
576                };
577                if t.ty.path.segments.last().map(String::as_str) != Some("BoundedVec") {
578                    return false;
579                };
580                let [only] = c.fields.as_slice() else {
581                    return false;
582                };
583                let Ok(TypeDef::Sequence(s)) = rt.resolve(only.ty.id).map(|t| &t.type_def) else {
584                    return false;
585                };
586                only.name.is_none() && rt.resolve(s.type_param.id).is_ok_and(&elem)
587            })
588            .map(|t| t.id)
589            .expect("metadata has a matching BoundedVec")
590    }
591
592    /// One nesting level of a single-field sequence newtype is legacy
593    /// double-wrapping only when the payload reads as such; a flat one-entry
594    /// list of tuples stays flat.
595    #[test]
596    fn newtype_sequences_unwrap_only_double_wrapped_payloads() {
597        let g = golden();
598        let rt = runtime();
599        let addr = |i: usize| Value::str(g["ss58"][i]["address"].as_str().unwrap());
600        let enc = |id: u32, v: &Value| rt.encode_spec(&TypeSpec::Id(id), v);
601        let pair = |a: i128, b: i128| Value::List(vec![Value::Int(a), Value::Int(b)]);
602        let list = |items: Vec<Value>| Value::List(items);
603
604        let pairs = bounded_vec_of(&rt, |t| match &t.type_def {
605            TypeDef::Tuple(tup) => {
606                tup.fields.len() == 2
607                    && tup.fields.iter().all(|f| {
608                        rt.resolve(f.id)
609                            .is_ok_and(|p| matches!(p.type_def, TypeDef::Primitive(_)))
610                    })
611            }
612            _ => false,
613        });
614        let one = enc(pairs, &list(vec![pair(1, 5)])).unwrap();
615        assert_eq!(one[0], 1 << 2, "flat one-entry list encodes one pair");
616        let two = enc(pairs, &list(vec![pair(1, 5), pair(1, 7)])).unwrap();
617        assert_eq!(two[0], 2 << 2);
618        assert_eq!(
619            enc(pairs, &list(vec![list(vec![pair(1, 5)])])).unwrap(),
620            one
621        );
622        assert_eq!(
623            enc(pairs, &list(vec![list(vec![pair(1, 5), pair(1, 7)])])).unwrap(),
624            two
625        );
626
627        let accounts = bounded_vec_of(&rt, |t| {
628            t.path.segments.last().map(String::as_str) == Some("AccountId32")
629        });
630        let flat = enc(accounts, &list(vec![addr(0), addr(1)])).unwrap();
631        assert_eq!(
632            enc(accounts, &list(vec![list(vec![addr(0), addr(1)])])).unwrap(),
633            flat
634        );
635
636        let data = bounded_vec_of(&rt, |t| {
637            t.path.segments.last().map(String::as_str) == Some("Data")
638                && matches!(t.type_def, TypeDef::Variant(_))
639        });
640        let raw = Value::record(vec![("Raw5".into(), Value::str("0x0102030405"))]);
641        let flat = enc(data, &list(vec![raw.clone()])).unwrap();
642        assert_eq!(enc(data, &list(vec![list(vec![raw])])).unwrap(), flat);
643    }
644}
bittensor_core/codec/mod.rs

bittensor_core/codec/
mod.rs
