From 8d30367bcda9b9fdcc4338dda86197a17a03fa60 Mon Sep 17 00:00:00 2001 From: DaPorkchop_ Date: Fri, 17 May 2024 23:20:50 +0200 Subject: [PATCH 1/6] utils: add dedicated modules for serializing and deserializing arrays --- src/utils/mod.rs | 1 + src/utils/serialize_utils.rs | 353 +++++++++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) create mode 100644 src/utils/serialize_utils.rs diff --git a/src/utils/mod.rs b/src/utils/mod.rs index adb6987..d7b7f3e 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -8,6 +8,7 @@ use crate::primitives::asset::TokenAmount; pub mod druid_utils; pub mod error_utils; pub mod script_utils; +pub mod serialize_utils; pub mod test_utils; pub mod transaction_utils; diff --git a/src/utils/serialize_utils.rs b/src/utils/serialize_utils.rs new file mode 100644 index 0000000..5d211e2 --- /dev/null +++ b/src/utils/serialize_utils.rs @@ -0,0 +1,353 @@ +use std::any::TypeId; +use std::convert::TryFrom; +use std::fmt::Formatter; +use std::marker::PhantomData; + +use bincode::config::*; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::de::{SeqAccess, Visitor}; +use serde::ser::{SerializeTuple}; + +pub fn bincode_default() -> WithOtherTrailing, RejectTrailing> { + DefaultOptions::new() + .with_fixint_encoding() + .reject_trailing_bytes() +} + +pub fn bincode_compact() -> WithOtherTrailing, RejectTrailing> { + DefaultOptions::new() + .with_varint_encoding() + .reject_trailing_bytes() +} + +/// A codec for fixed-size arrays. +pub mod fixed_array_codec { + use super::*; + + pub fn serialize( + values: &[T; N], + serializer: S, + ) -> Result { + if TypeId::of::() == TypeId::of::() && serializer.is_human_readable() { + // We're serializing a byte array for a human-readable format, make it a hex string + vec_codec::serialize(values, serializer) + } else { + // Serialize the array as a tuple, to avoid adding a length prefix + let mut tuple = serializer.serialize_tuple(N)?; + for e in values { + tuple.serialize_element(e)?; + } + tuple.end() + } + } + + pub fn deserialize<'de, T: Deserialize<'de> + 'static, D: Deserializer<'de>, const N: usize>( + deserializer: D, + ) -> Result<[T; N], D::Error> { + if TypeId::of::() == TypeId::of::() && deserializer.is_human_readable() { + // We're deserializing a byte array for a human-readable format, we'll accept two different + // representations: + // - A hexadecimal string + // - An array of byte literals (this format should never be produced by the serializer + // for human-readable formats, but it was in the past, so we'll still support reading + // it for backwards-compatibility). + vec_to_fixed_array(vec_codec::deserialize(deserializer)?) + } else { + // We're deserializing a binary format, read the array as a tuple + // (to avoid adding a length prefix) + + struct FixedArrayVisitor(PhantomData); + impl<'de, T: Deserialize<'de>, const N: usize> Visitor<'de> for FixedArrayVisitor { + type Value = [T; N]; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + write!(formatter, "a sequence") + } + + fn visit_seq>(self, mut seq: A) -> Result { + let mut vec = Vec::with_capacity(N); + while let Some(val) = seq.next_element::()? { + vec.push(val) + } + vec_to_fixed_array(vec) + } + } + + deserializer.deserialize_tuple(N, FixedArrayVisitor(Default::default())) + } + } +} + +/// A codec for variable-length `Vec`s. +pub mod vec_codec { + use super::*; + + pub fn serialize( + values: &[T], + serializer: S, + ) -> Result { + if TypeId::of::() == TypeId::of::() && serializer.is_human_readable() { + // We're serializing a byte array for a human-readable format, make it a hex string + let bytes = unsafe { std::slice::from_raw_parts(values.as_ptr() as *const u8, values.len()) }; + serializer.serialize_str(&hex::encode(bytes)) + } else { + // Serialize the array as a length-prefixed sequence + values.serialize(serializer) + } + } + + pub fn deserialize<'de, T: Deserialize<'de> + 'static, D: Deserializer<'de>>( + deserializer: D, + ) -> Result, D::Error> { + if TypeId::of::() == TypeId::of::() && deserializer.is_human_readable() { + // We're deserializing a byte array for a human-readable format, we'll accept two different + // representations: + // - A hexadecimal string + // - An array of byte literals (this format should never be produced by the serializer + // for human-readable formats, but it was in the past, so we'll still support reading + // it for backwards-compatibility). + + struct HexStringOrBytesVisitor(); + impl<'de> Visitor<'de> for HexStringOrBytesVisitor { + type Value = Vec; + + fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + formatter.write_str("hex string or byte array") + } + + fn visit_str(self, value: &str) -> Result { + hex::decode(value).map_err(E::custom) + } + + fn visit_seq(self, mut seq: A) -> Result where A: SeqAccess<'de> { + let mut vec = Vec::new(); + while let Some(elt) = seq.next_element::()? { + vec.push(elt); + } + Ok(vec) + } + } + + Ok(deserializer.deserialize_any(HexStringOrBytesVisitor())?.into_iter() + // This is a hack to convert the Vec into a Vec, even though we already know + // that T = u8. This could be done in a much nicer way if trait specialization were + // a thing, but unfortunately it's still only available on nightly :( + .map(|b| unsafe { std::mem::transmute_copy::(&b) }) + .collect::>()) + } else { + // Read a length-prefixed sequence as a Vec + >::deserialize(deserializer) + } + } +} + +fn vec_to_fixed_array( + vec: Vec, +) -> Result<[T; N], E> { + <[T; N]>::try_from(vec) + .map_err(|vec| E::custom(format!("expected exactly {} elements, but read {}", N, vec.len()))) +} + +/*---- TESTS ----*/ + +#[cfg(test)] +mod tests { + use std::fmt::Debug; + use bincode::Options; + + use serde::{Deserialize, Serialize}; + use serde::de::DeserializeOwned; + use super::*; + + fn repeat(orig: &str, n: usize) -> String { + let mut res = String::with_capacity(orig.len() * n); + for _ in 0..n { + res.push_str(orig) + } + res + } + + fn test_bin_codec( + options: fn() -> O, + obj: T, + expect: &str, + ) { + let bytes = options().serialize(&obj).unwrap(); + assert_eq!(hex::encode(&bytes), expect); + assert_eq!(options().deserialize::(&bytes).unwrap(), obj); + } + + fn test_json_codec( + obj: T, + expect: &str, + ) { + let json = serde_json::to_string(&obj).unwrap(); + assert_eq!(json, expect); + assert_eq!(serde_json::from_str::(&json).unwrap(), obj); + } + + fn test_json_deserialize( + obj: T, + json: &str, + ) { + assert_eq!(serde_json::from_str::(&json).unwrap(), obj); + } + + macro_rules! test_fixed_array { + ($n:literal) => { + test_bin_codec(bincode_default, [VAL; $n], &repeat(HEX, $n)); + test_json_codec([VAL; $n], &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + macro_rules! test_fixed_array_wrapper { + ($e:ty, $t:ident, $n:literal) => { + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct $t([$e; $n]); + test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_json_codec($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + #[test] + fn test_fixed_u32_arrays() { + const VAL : u32 = 0xDEADBEEF; + const HEX : &str = "efbeadde"; + + test_fixed_array!(0); + test_fixed_array!(1); + test_fixed_array!(32); + + test_fixed_array_wrapper!(u32, FixedArrayWrapper0, 0); + test_fixed_array_wrapper!(u32, FixedArrayWrapper1, 1); + test_fixed_array_wrapper!(u32, FixedArrayWrapper32, 32); + + macro_rules! test_fixed_array_wrapper_codec { + ($t:ident, $n:literal) => { + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct $t(#[serde(with = "fixed_array_codec")] [u32; $n]); + test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_json_codec($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper0, 0); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper1, 1); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper32, 32); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper33, 33); + } + + #[test] + fn test_fixed_u8_arrays() { + const VAL : u8 = 123; + const HEX : &str = "7b"; + + test_fixed_array!(0); + test_fixed_array!(1); + test_fixed_array!(32); + + test_fixed_array_wrapper!(u8, FixedArrayWrapper0, 0); + test_fixed_array_wrapper!(u8, FixedArrayWrapper1, 1); + test_fixed_array_wrapper!(u8, FixedArrayWrapper32, 32); + + macro_rules! test_fixed_array_wrapper_codec { + ($t:ident, $n:literal) => { + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct $t(#[serde(with = "fixed_array_codec")] [u8; $n]); + test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_json_codec($t([VAL; $n]), &format!("\"{}\"", hex::encode(&[VAL; $n].to_vec()))); + test_json_deserialize($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper0, 0); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper1, 1); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper32, 32); + test_fixed_array_wrapper_codec!(CodecFixedArrayWrapper33, 33); + } + + fn size_to_hex_default(n: usize) -> String { + hex::encode(bincode_default().serialize(&n).unwrap()) + } + + macro_rules! test_vec { + ($n:literal) => { + test_bin_codec(bincode_default, [VAL; $n].to_vec(), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec([VAL; $n].to_vec(), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + macro_rules! test_vec_wrapper { + ($n:literal) => { + test_bin_codec(bincode_default, VecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec(VecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + #[test] + fn test_u32_vecs() { + const VAL : u32 = 0xDEADBEEF; + const HEX : &str = "efbeadde"; + + test_vec!(0); + test_vec!(1); + test_vec!(32); + test_vec!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct VecWrapper(Vec); + + test_vec_wrapper!(0); + test_vec_wrapper!(1); + test_vec_wrapper!(32); + test_vec_wrapper!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct CodecVecWrapper(#[serde(with = "vec_codec")] Vec); + macro_rules! test_vec_wrapper_codec { + ($n:literal) => { + test_bin_codec(bincode_default, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec(CodecVecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_vec_wrapper_codec!(0); + test_vec_wrapper_codec!(1); + test_vec_wrapper_codec!(32); + test_vec_wrapper_codec!(33); + } + + #[test] + fn test_u8_vecs() { + const VAL : u8 = 123; + const HEX : &str = "7b"; + + test_vec!(0); + test_vec!(1); + test_vec!(32); + test_vec!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct VecWrapper(Vec); + + test_vec_wrapper!(0); + test_vec_wrapper!(1); + test_vec_wrapper!(32); + test_vec_wrapper!(33); + + #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] + struct CodecVecWrapper(#[serde(with = "vec_codec")] Vec); + macro_rules! test_vec_wrapper_codec { + ($n:literal) => { + test_bin_codec(bincode_default, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_json_codec(CodecVecWrapper([VAL; $n].to_vec()), &format!("\"{}\"", hex::encode(&[VAL; $n].to_vec()))); + test_json_deserialize(CodecVecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + }; + } + + test_vec_wrapper_codec!(0); + test_vec_wrapper_codec!(1); + test_vec_wrapper_codec!(32); + test_vec_wrapper_codec!(33); + } +} From 7097e54d4342bcdc1123769240ea227b1bbfdb51 Mon Sep 17 00:00:00 2001 From: DaPorkchop_ Date: Wed, 29 May 2024 12:56:17 +0200 Subject: [PATCH 2/6] Add a struct for representing fixed-length byte arrays This allows other structs which wrap a fixed-length byte array to simply use it as a field, and not have to deal with any of the details of formatting or JSON hex (de)serialization. --- src/utils/serialize_utils.rs | 146 +++++++++++++++++++++++++++++++++-- 1 file changed, 141 insertions(+), 5 deletions(-) diff --git a/src/utils/serialize_utils.rs b/src/utils/serialize_utils.rs index 5d211e2..bd199fc 100644 --- a/src/utils/serialize_utils.rs +++ b/src/utils/serialize_utils.rs @@ -1,7 +1,9 @@ use std::any::TypeId; -use std::convert::TryFrom; -use std::fmt::Formatter; +use std::convert::{TryFrom, TryInto}; +use std::fmt; use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; +use std::str::FromStr; use bincode::config::*; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -20,6 +22,104 @@ pub fn bincode_compact() -> WithOtherTrailing( + #[serde(with = "fixed_array_codec")] + [u8; N], +); + +impl FixedByteArray { + pub fn new(arr: [u8; N]) -> Self { + Self(arr) + } +} + +impl AsRef<[u8]> for FixedByteArray { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl AsMut<[u8]> for FixedByteArray { + fn as_mut(&mut self) -> &mut [u8] { + &mut self.0 + } +} + +impl Deref for FixedByteArray { + type Target = [u8; N]; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for FixedByteArray { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From<[u8; N]> for FixedByteArray { + fn from(value: [u8; N]) -> Self { + Self(value) + } +} + +impl TryFrom<&[u8]> for FixedByteArray { + type Error = std::array::TryFromSliceError; + + fn try_from(value: &[u8]) -> Result { + value.try_into().map(Self) + } +} + +impl TryFrom<&Vec> for FixedByteArray { + type Error = std::array::TryFromSliceError; + + fn try_from(value: &Vec) -> Result { + value.as_slice().try_into().map(Self) + } +} + +impl fmt::LowerHex for FixedByteArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // This is hacky because we can't make an array of type [u8; {N * 2}] due to + // generic parameters not being allowed in constant expressions on stable rust + assert_eq!(std::mem::size_of::<[u16; N]>(), std::mem::size_of::<[u8; N]>() * 2); + let mut buf = [0u16; N]; + let slice = unsafe { std::slice::from_raw_parts_mut(buf.as_mut_ptr() as *mut u8, N * 2) }; + hex::encode_to_slice(&self.0, slice).unwrap(); + f.write_str(std::str::from_utf8(slice).unwrap()) + } +} + +impl fmt::Display for FixedByteArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fmt::LowerHex::fmt(self, f) + } +} + +impl fmt::Debug for FixedByteArray { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "FixedByteArray<{N}>({self:x})") + } +} + +impl FromStr for FixedByteArray { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + let mut buf = [0u8; N]; + hex::decode_to_slice(s, &mut buf)?; + Ok(Self(buf)) + } +} + /// A codec for fixed-size arrays. pub mod fixed_array_codec { use super::*; @@ -60,7 +160,7 @@ pub mod fixed_array_codec { impl<'de, T: Deserialize<'de>, const N: usize> Visitor<'de> for FixedArrayVisitor { type Value = [T; N]; - fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { write!(formatter, "a sequence") } @@ -111,7 +211,7 @@ pub mod vec_codec { impl<'de> Visitor<'de> for HexStringOrBytesVisitor { type Value = Vec; - fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result { + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str("hex string or byte array") } @@ -152,7 +252,7 @@ fn vec_to_fixed_array( #[cfg(test)] mod tests { - use std::fmt::Debug; + use std::fmt::{Debug, Display}; use bincode::Options; use serde::{Deserialize, Serialize}; @@ -193,6 +293,15 @@ mod tests { assert_eq!(serde_json::from_str::(&json).unwrap(), obj); } + fn test_display_fromstr( + obj: T, + expect: &str, + ) { + let string = obj.to_string(); + assert_eq!(string, expect); + assert_eq!(::from_str(&string).ok().unwrap(), obj); + } + macro_rules! test_fixed_array { ($n:literal) => { test_bin_codec(bincode_default, [VAL; $n], &repeat(HEX, $n)); @@ -350,4 +459,31 @@ mod tests { test_vec_wrapper_codec!(32); test_vec_wrapper_codec!(33); } + + #[test] + fn test_fixed_byte_array() { + const VAL : u8 = 123; + const HEX : &str = "7b"; + + macro_rules! test_fixed_byte_array { + ($n:literal) => { + test_bin_codec(bincode_default, FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); + test_json_codec(FixedByteArray::<$n>([VAL; $n]), &format!("\"{}\"", repeat(HEX, $n))); + test_json_deserialize(FixedByteArray::<$n>([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); + test_display_fromstr(FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); + assert_eq!(format!("{:x}", FixedByteArray::<$n>([VAL; $n])), repeat(HEX, $n)); + assert_eq!( + format!("{:?}", FixedByteArray::<$n>([VAL; $n])), + format!("FixedByteArray<{}>({})", $n, repeat(HEX, $n))); + assert_eq!( + format!("{:x?}", FixedByteArray::<$n>([VAL; $n])), + format!("FixedByteArray<{}>({})", $n, repeat(HEX, $n))); + }; + } + + test_fixed_byte_array!(0); + test_fixed_byte_array!(1); + test_fixed_byte_array!(32); + test_fixed_byte_array!(33); + } } From 4be53f5a1ed14aac6a73ed340b2ee826799fd81f Mon Sep 17 00:00:00 2001 From: DaPorkchop_ Date: Wed, 29 May 2024 12:56:17 +0200 Subject: [PATCH 3/6] serialize_utils: implement From<&[u8; N]> for FixedByteArray --- src/utils/serialize_utils.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/utils/serialize_utils.rs b/src/utils/serialize_utils.rs index bd199fc..af72618 100644 --- a/src/utils/serialize_utils.rs +++ b/src/utils/serialize_utils.rs @@ -70,6 +70,12 @@ impl From<[u8; N]> for FixedByteArray { } } +impl From<&[u8; N]> for FixedByteArray { + fn from(value: &[u8; N]) -> Self { + Self(*value) + } +} + impl TryFrom<&[u8]> for FixedByteArray { type Error = std::array::TryFromSliceError; From 87f48d6d3e5512b35bdbc498f29cdeffaab08294 Mon Sep 17 00:00:00 2001 From: DaPorkchop_ Date: Mon, 3 Jun 2024 09:38:17 +0200 Subject: [PATCH 4/6] always use fully-qualified name for bincode (de)serialize functions this makes it clear which serialization format is being used where, as we are going to be changing this up soon --- src/primitives/block.rs | 7 +++---- src/primitives/transaction.rs | 11 ----------- src/script/interface_ops.rs | 2 -- src/script/lang.rs | 2 -- src/utils/script_utils.rs | 2 -- src/utils/transaction_utils.rs | 10 +++------- 6 files changed, 6 insertions(+), 28 deletions(-) diff --git a/src/primitives/block.rs b/src/primitives/block.rs index 72e07e9..3397424 100644 --- a/src/primitives/block.rs +++ b/src/primitives/block.rs @@ -4,7 +4,6 @@ use crate::crypto::sha3_256::{self, Sha3_256}; use crate::crypto::sign_ed25519::PublicKey; use crate::primitives::asset::Asset; use crate::primitives::transaction::{Transaction, TxIn, TxOut}; -use bincode::{deserialize, serialize}; use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::convert::TryInto; @@ -82,7 +81,7 @@ impl Block { /// Sets the internal number of bits based on length pub fn set_bits(&mut self) { - let bytes = Bytes::from(match serialize(&self) { + let bytes = Bytes::from(match bincode::serialize(&self) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize block: {:?}", e); @@ -94,7 +93,7 @@ impl Block { /// Checks whether a block has hit its maximum size pub fn is_full(&self) -> bool { - let bytes = Bytes::from(match serialize(&self) { + let bytes = Bytes::from(match bincode::serialize(&self) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize block: {:?}", e); @@ -143,7 +142,7 @@ pub fn gen_random_hash() -> String { /// /// * `transactions` - Transactions to construct a merkle tree for pub fn build_hex_txs_hash(transactions: &[String]) -> String { - let txs = match serialize(transactions) { + let txs = match bincode::serialize(transactions) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize transactions: {:?}", e); diff --git a/src/primitives/transaction.rs b/src/primitives/transaction.rs index 08435fd..e1d7236 100644 --- a/src/primitives/transaction.rs +++ b/src/primitives/transaction.rs @@ -8,8 +8,6 @@ use crate::primitives::{ use crate::script::lang::Script; use crate::script::{OpCodes, StackEntry}; use crate::utils::is_valid_amount; -use bincode::serialize; -use bytes::Bytes; use serde::{Deserialize, Serialize}; use std::fmt; @@ -205,15 +203,6 @@ impl Transaction { } } - /// Get the total transaction size in bytes - pub fn get_total_size(&self) -> usize { - let bytes = match serialize(self) { - Ok(bytes) => bytes, - Err(_) => vec![], - }; - bytes.len() - } - /// Gets the create asset assigned to this transaction, if it exists fn get_create_asset(&self) -> Option<&Asset> { let is_create = self.inputs.len() == 1 diff --git a/src/script/interface_ops.rs b/src/script/interface_ops.rs index 7b9875b..ba2a63c 100644 --- a/src/script/interface_ops.rs +++ b/src/script/interface_ops.rs @@ -11,8 +11,6 @@ use crate::utils::error_utils::*; use crate::utils::transaction_utils::{ construct_address, construct_address_temp, construct_address_v0, }; -use bincode::de; -use bincode::serialize; use bytes::Bytes; use hex::encode; use std::collections::BTreeMap; diff --git a/src/script/lang.rs b/src/script/lang.rs index 259465e..2498e5c 100644 --- a/src/script/lang.rs +++ b/src/script/lang.rs @@ -8,8 +8,6 @@ use crate::script::interface_ops::*; use crate::script::{OpCodes, StackEntry}; use crate::utils::error_utils::*; use crate::utils::transaction_utils::{construct_address, construct_address_for}; -use bincode::serialize; -use bytes::Bytes; use hex::encode; use serde::{Deserialize, Serialize}; use tracing::{error, warn}; diff --git a/src/utils/script_utils.rs b/src/utils/script_utils.rs index a2cd085..fca79da 100644 --- a/src/utils/script_utils.rs +++ b/src/utils/script_utils.rs @@ -15,8 +15,6 @@ use crate::utils::transaction_utils::{ construct_address, construct_tx_hash, construct_tx_in_out_signable_hash, construct_tx_in_signable_asset_hash, construct_tx_in_signable_hash, }; -use bincode::serialize; -use bytes::Bytes; use hex::encode; use ring::error; use std::collections::{BTreeMap, BTreeSet}; diff --git a/src/utils/transaction_utils.rs b/src/utils/transaction_utils.rs index 8e9fd99..9a96af1 100644 --- a/src/utils/transaction_utils.rs +++ b/src/utils/transaction_utils.rs @@ -6,7 +6,6 @@ use crate::primitives::druid::{DdeValues, DruidExpectation}; use crate::primitives::transaction::*; use crate::script::lang::Script; use crate::script::{OpCodes, StackEntry}; -use bincode::serialize; use std::collections::BTreeMap; use tracing::debug; @@ -21,10 +20,7 @@ pub struct ReceiverInfo { /// /// * `script` - Script to build address for pub fn construct_p2sh_address(script: &Script) -> String { - let bytes = match serialize(script) { - Ok(bytes) => bytes, - Err(_) => vec![], - }; + let bytes = bincode::serialize(script).unwrap(); let mut addr = hex::encode(sha3_256::digest(&bytes)); addr.insert(ZERO, P2SH_PREPEND as char); addr.truncate(STANDARD_ADDRESS_LENGTH); @@ -344,7 +340,7 @@ pub fn update_utxo_set(current_utxo: &mut BTreeMap) { /// /// * `tx` - Transaction to hash pub fn construct_tx_hash(tx: &Transaction) -> String { - let bytes = match serialize(tx) { + let bytes = match bincode::serialize(tx) { Ok(bytes) => bytes, Err(_) => vec![], }; @@ -1179,7 +1175,7 @@ mod tests { ..Default::default() }]; - let bytes = match serialize(&tx_ins) { + let bytes = match bincode::serialize(&tx_ins) { Ok(bytes) => bytes, Err(_) => vec![], }; From feef369594a8cc6d7c4f6ca673999f6de17b3f8d Mon Sep 17 00:00:00 2001 From: DaPorkchop_ Date: Thu, 6 Jun 2024 14:17:56 +0200 Subject: [PATCH 5/6] Upgrade to bincode 2 This will enable us to keep JSON serialization separate from the binary encoding, and also be much more explicit about the exact binary representation of objects. --- Cargo.lock | 20 ++++++++++++-- Cargo.toml | 2 +- src/primitives/block.rs | 11 ++++---- src/utils/serialize_utils.rs | 50 ++++++++++++++++++---------------- src/utils/transaction_utils.rs | 6 ++-- 5 files changed, 54 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b6575dd..009c39b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,13 +108,23 @@ checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5" [[package]] name = "bincode" -version = "1.3.3" +version = "2.0.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +checksum = "f11ea1a0346b94ef188834a65c068a03aec181c94896d481d7a0a40d85b0ce95" dependencies = [ + "bincode_derive", "serde", ] +[[package]] +name = "bincode_derive" +version = "2.0.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e30759b3b99a1b802a7a3aa21c85c3ded5c28e1c83170d82d70f08bbf7f3e4c" +dependencies = [ + "virtue", +] + [[package]] name = "bindgen" version = "0.65.1" @@ -1068,6 +1078,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtue" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 5c6fd2e..ee03c21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ exclude = ["**/tests/**", "**/examples/**", "**/benchmarks/**", "docs/**", ".hoo [dependencies] actix-rt = "2.8.0" base64 = "0.20.0" -bincode = "1.3.3" +bincode = { version = "2.0.0-rc.3", features = ["serde"] } bytes = "1.4.0" colored = { version = "2.1.0", optional = true } hex = "0.4.3" diff --git a/src/primitives/block.rs b/src/primitives/block.rs index 3397424..18cb611 100644 --- a/src/primitives/block.rs +++ b/src/primitives/block.rs @@ -81,25 +81,25 @@ impl Block { /// Sets the internal number of bits based on length pub fn set_bits(&mut self) { - let bytes = Bytes::from(match bincode::serialize(&self) { + let bytes = match bincode::serde::encode_to_vec(&self, bincode::config::legacy()) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize block: {:?}", e); return; } - }); + }; self.header.bits = bytes.len(); } /// Checks whether a block has hit its maximum size pub fn is_full(&self) -> bool { - let bytes = Bytes::from(match bincode::serialize(&self) { + let bytes = match bincode::serde::encode_to_vec(&self, bincode::config::legacy()) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize block: {:?}", e); return false; } - }); + }; bytes.len() >= MAX_BLOCK_SIZE } @@ -142,7 +142,8 @@ pub fn gen_random_hash() -> String { /// /// * `transactions` - Transactions to construct a merkle tree for pub fn build_hex_txs_hash(transactions: &[String]) -> String { - let txs = match bincode::serialize(transactions) { + // TODO: This is bad, it won't produce the same result on 32-bit systems + let txs = match bincode::serde::encode_to_vec(transactions, bincode::config::legacy()) { Ok(bytes) => bytes, Err(e) => { warn!("Failed to serialize transactions: {:?}", e); diff --git a/src/utils/serialize_utils.rs b/src/utils/serialize_utils.rs index af72618..39eb7ea 100644 --- a/src/utils/serialize_utils.rs +++ b/src/utils/serialize_utils.rs @@ -5,21 +5,24 @@ use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::str::FromStr; -use bincode::config::*; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::de::{SeqAccess, Visitor}; use serde::ser::{SerializeTuple}; -pub fn bincode_default() -> WithOtherTrailing, RejectTrailing> { - DefaultOptions::new() - .with_fixint_encoding() - .reject_trailing_bytes() +/// Implements `Write` by simply counting the number of bytes written to it. +pub struct ByteCountingWriter { + pub count: usize, } -pub fn bincode_compact() -> WithOtherTrailing, RejectTrailing> { - DefaultOptions::new() - .with_varint_encoding() - .reject_trailing_bytes() +impl std::io::Write for ByteCountingWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.count += buf.len(); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } /// Simple wrapper around a fixed-length byte array. @@ -259,7 +262,6 @@ fn vec_to_fixed_array( #[cfg(test)] mod tests { use std::fmt::{Debug, Display}; - use bincode::Options; use serde::{Deserialize, Serialize}; use serde::de::DeserializeOwned; @@ -273,14 +275,14 @@ mod tests { res } - fn test_bin_codec( - options: fn() -> O, + fn test_bin_codec( + config: fn() -> C, obj: T, expect: &str, ) { - let bytes = options().serialize(&obj).unwrap(); + let bytes = bincode::serde::encode_to_vec(&obj, config()).unwrap(); assert_eq!(hex::encode(&bytes), expect); - assert_eq!(options().deserialize::(&bytes).unwrap(), obj); + assert_eq!(bincode::serde::decode_from_slice::(&bytes, config()).unwrap().0, obj); } fn test_json_codec( @@ -310,7 +312,7 @@ mod tests { macro_rules! test_fixed_array { ($n:literal) => { - test_bin_codec(bincode_default, [VAL; $n], &repeat(HEX, $n)); + test_bin_codec(bincode::config::legacy, [VAL; $n], &repeat(HEX, $n)); test_json_codec([VAL; $n], &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; } @@ -319,7 +321,7 @@ mod tests { ($e:ty, $t:ident, $n:literal) => { #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] struct $t([$e; $n]); - test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_bin_codec(bincode::config::legacy, $t([VAL; $n]), &repeat(HEX, $n)); test_json_codec($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; } @@ -341,7 +343,7 @@ mod tests { ($t:ident, $n:literal) => { #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] struct $t(#[serde(with = "fixed_array_codec")] [u32; $n]); - test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_bin_codec(bincode::config::legacy, $t([VAL; $n]), &repeat(HEX, $n)); test_json_codec($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; } @@ -369,7 +371,7 @@ mod tests { ($t:ident, $n:literal) => { #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)] struct $t(#[serde(with = "fixed_array_codec")] [u8; $n]); - test_bin_codec(bincode_default, $t([VAL; $n]), &repeat(HEX, $n)); + test_bin_codec(bincode::config::legacy, $t([VAL; $n]), &repeat(HEX, $n)); test_json_codec($t([VAL; $n]), &format!("\"{}\"", hex::encode(&[VAL; $n].to_vec()))); test_json_deserialize($t([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; @@ -382,19 +384,19 @@ mod tests { } fn size_to_hex_default(n: usize) -> String { - hex::encode(bincode_default().serialize(&n).unwrap()) + hex::encode(&(n as u64).to_le_bytes()) } macro_rules! test_vec { ($n:literal) => { - test_bin_codec(bincode_default, [VAL; $n].to_vec(), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_bin_codec(bincode::config::legacy, [VAL; $n].to_vec(), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); test_json_codec([VAL; $n].to_vec(), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; } macro_rules! test_vec_wrapper { ($n:literal) => { - test_bin_codec(bincode_default, VecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_bin_codec(bincode::config::legacy, VecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); test_json_codec(VecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; } @@ -421,7 +423,7 @@ mod tests { struct CodecVecWrapper(#[serde(with = "vec_codec")] Vec); macro_rules! test_vec_wrapper_codec { ($n:literal) => { - test_bin_codec(bincode_default, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_bin_codec(bincode::config::legacy, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); test_json_codec(CodecVecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; } @@ -454,7 +456,7 @@ mod tests { struct CodecVecWrapper(#[serde(with = "vec_codec")] Vec); macro_rules! test_vec_wrapper_codec { ($n:literal) => { - test_bin_codec(bincode_default, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); + test_bin_codec(bincode::config::legacy, CodecVecWrapper([VAL; $n].to_vec()), &format!("{}{}", size_to_hex_default($n), repeat(HEX, $n))); test_json_codec(CodecVecWrapper([VAL; $n].to_vec()), &format!("\"{}\"", hex::encode(&[VAL; $n].to_vec()))); test_json_deserialize(CodecVecWrapper([VAL; $n].to_vec()), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); }; @@ -473,7 +475,7 @@ mod tests { macro_rules! test_fixed_byte_array { ($n:literal) => { - test_bin_codec(bincode_default, FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); + test_bin_codec(bincode::config::legacy, FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); test_json_codec(FixedByteArray::<$n>([VAL; $n]), &format!("\"{}\"", repeat(HEX, $n))); test_json_deserialize(FixedByteArray::<$n>([VAL; $n]), &serde_json::to_string(&[VAL; $n].to_vec()).unwrap()); test_display_fromstr(FixedByteArray::<$n>([VAL; $n]), &repeat(HEX, $n)); diff --git a/src/utils/transaction_utils.rs b/src/utils/transaction_utils.rs index 9a96af1..fbc89e3 100644 --- a/src/utils/transaction_utils.rs +++ b/src/utils/transaction_utils.rs @@ -20,7 +20,7 @@ pub struct ReceiverInfo { /// /// * `script` - Script to build address for pub fn construct_p2sh_address(script: &Script) -> String { - let bytes = bincode::serialize(script).unwrap(); + let bytes = bincode::serde::encode_to_vec(script, bincode::config::legacy()).unwrap(); let mut addr = hex::encode(sha3_256::digest(&bytes)); addr.insert(ZERO, P2SH_PREPEND as char); addr.truncate(STANDARD_ADDRESS_LENGTH); @@ -340,7 +340,7 @@ pub fn update_utxo_set(current_utxo: &mut BTreeMap) { /// /// * `tx` - Transaction to hash pub fn construct_tx_hash(tx: &Transaction) -> String { - let bytes = match bincode::serialize(tx) { + let bytes = match bincode::serde::encode_to_vec(tx, bincode::config::legacy()) { Ok(bytes) => bytes, Err(_) => vec![], }; @@ -1175,7 +1175,7 @@ mod tests { ..Default::default() }]; - let bytes = match bincode::serialize(&tx_ins) { + let bytes = match bincode::serde::encode_to_vec(&tx_ins, bincode::config::legacy()) { Ok(bytes) => bytes, Err(_) => vec![], }; From 36409d524f2589c1777af3fb234b861c73c47d4f Mon Sep 17 00:00:00 2001 From: DaPorkchop_ Date: Thu, 11 Jul 2024 12:51:44 +0200 Subject: [PATCH 6/6] serialize_utils: add a bunch of helper methods --- src/utils/serialize_utils.rs | 116 +++++++++++++++++++++++++++++++++-- 1 file changed, 112 insertions(+), 4 deletions(-) diff --git a/src/utils/serialize_utils.rs b/src/utils/serialize_utils.rs index 39eb7ea..9053c02 100644 --- a/src/utils/serialize_utils.rs +++ b/src/utils/serialize_utils.rs @@ -1,20 +1,32 @@ use std::any::TypeId; use std::convert::{TryFrom, TryInto}; use std::fmt; +use std::io::Write; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::str::FromStr; +use bincode::{BorrowDecode, Decode, Encode}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde::de::{SeqAccess, Visitor}; use serde::ser::{SerializeTuple}; /// Implements `Write` by simply counting the number of bytes written to it. +#[derive(Copy, Clone, Debug)] pub struct ByteCountingWriter { pub count: usize, } -impl std::io::Write for ByteCountingWriter { +impl ByteCountingWriter { + /// Creates a new `ByteCountingWriter` with a `count` of 0. + pub fn new() -> Self { + Self { + count: 0, + } + } +} + +impl Write for ByteCountingWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { self.count += buf.len(); Ok(buf.len()) @@ -29,7 +41,7 @@ impl std::io::Write for ByteCountingWriter { /// /// This can be formatted to and parsed from a hexadecimal string using `Display` and `FromStr`. /// When serialized as JSON, it is also represented as a hexadecimal string. -#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Encode, Decode)] pub struct FixedByteArray( #[serde(with = "fixed_array_codec")] [u8; N], @@ -129,7 +141,7 @@ impl FromStr for FixedByteArray { } } -/// A codec for fixed-size arrays. +/// A serde codec for fixed-size arrays. pub mod fixed_array_codec { use super::*; @@ -187,7 +199,7 @@ pub mod fixed_array_codec { } } -/// A codec for variable-length `Vec`s. +/// A serde codec for variable-length `Vec`s. pub mod vec_codec { use super::*; @@ -257,6 +269,102 @@ fn vec_to_fixed_array( .map_err(|vec| E::custom(format!("expected exactly {} elements, but read {}", N, vec.len()))) } +/// Encodes an object into a `Vec` using bincode 2's standard configuration. +/// +/// This allows using the turbofish operator to explicitly specify the encode type without also +/// having to specify the config type. +/// +/// ### Arguments +/// +/// * `value` - the value to encode +#[inline(always)] +pub fn bincode_encode_to_vec_standard( + value: &T, +) -> Result, bincode::error::EncodeError> { + bincode::encode_to_vec(value, bincode::config::standard()) +} + +/// Encodes an object into the given `Write` using bincode 2's standard configuration. +/// +/// This allows using the turbofish operator to explicitly specify the encode type without also +/// having to specify the config type. +/// +/// ### Arguments +/// +/// * `value` - the value to encode +/// * `write` - the `Write` to encode into +#[inline(always)] +pub fn bincode_encode_to_write_standard( + value: &T, + write: &mut impl Write, +) -> Result { + bincode::encode_into_std_write(value, write, bincode::config::standard()) +} + +/// Calculates the encoded size of the given object using bincode 2's standard configuration. +/// +/// ### Arguments +/// +/// * `value` - the value to encode +#[inline(always)] +pub fn bincode_encoded_size_standard( + value: &T, +) -> Result { + let mut writer = ByteCountingWriter::new(); + bincode::encode_into_std_write(value, &mut writer, bincode::config::standard())?; + Ok(writer.count) +} + +/// Decodes an object from a slice using bincode 2's standard configuration. +/// +/// This allows using the turbofish operator to explicitly specify the decode type without also +/// having to specify the config type. +/// +/// ### Arguments +/// +/// * `slice` - the slice to decode from +#[inline(always)] +pub fn bincode_decode_from_slice_standard( + slice: &[u8], +) -> Result<(T, usize), bincode::error::DecodeError> { + bincode::decode_from_slice(slice, bincode::config::standard()) +} + +/// Decodes an object from a slice using bincode 2's standard configuration. +/// +/// This allows using the turbofish operator to explicitly specify the decode type without also +/// having to specify the config type. +/// +/// ### Arguments +/// +/// * `slice` - the slice to decode from +pub fn bincode_decode_from_slice_standard_full( + slice: &[u8], +) -> Result { + let (result, read_bytes) = bincode_decode_from_slice_standard::(slice)?; + if read_bytes == slice.len() { + Ok(result) + } else { + Err(bincode::error::DecodeError::OtherString( + format!("{} bytes left over after decoding", slice.len() - read_bytes))) + } +} + +/// Decodes an object from a slice using bincode 2's standard configuration. +/// +/// This allows using the turbofish operator to explicitly specify the decode type without also +/// having to specify the config type. +/// +/// ### Arguments +/// +/// * `slice` - the slice to decode from +#[inline(always)] +pub fn bincode_borrow_decode_from_slice_standard<'a, T: BorrowDecode<'a>>( + slice: &'a [u8], +) -> Result<(T, usize), bincode::error::DecodeError> { + bincode::borrow_decode_from_slice(slice, bincode::config::standard()) +} + /*---- TESTS ----*/ #[cfg(test)]