From 880bfa5c2a015fc70fc813713915e2d4a18eed65 Mon Sep 17 00:00:00 2001 From: Bryan Burgers Date: Wed, 8 Feb 2023 14:23:35 -0600 Subject: [PATCH] Implement serde::Serialize, serde::Deserialize Make it possible for `Item` and `AttributeValue` to be deserialized from and serialized to DynamoDB's native JSON format. While this is typically not necessary, for some use cases it can be valuable. --- Cargo.toml | 1 + src/attribute_value.rs | 491 ++++++++++++++++++++++++++++++++++++++++- src/lib.rs | 41 +++- 3 files changed, 531 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1bba388..b57e8d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,7 @@ __rusoto_dynamodb_0_48 = { package = "rusoto_dynamodb", version = "0.48", defaul __rusoto_dynamodbstreams_0_46 = { package = "rusoto_dynamodbstreams", version = "0.46", default-features = false, optional = true } __rusoto_dynamodbstreams_0_47 = { package = "rusoto_dynamodbstreams", version = "0.47", default-features = false, optional = true } __rusoto_dynamodbstreams_0_48 = { package = "rusoto_dynamodbstreams", version = "0.48", default-features = false, optional = true } +base64 = "0.21.0" serde = "1" __rusoto_core_0_46_crate = { package = "rusoto_core", version = "0.46", default-features = false, features = ["rustls"], optional = true } diff --git a/src/attribute_value.rs b/src/attribute_value.rs index 773c1ac..98d07d1 100644 --- a/src/attribute_value.rs +++ b/src/attribute_value.rs @@ -1,5 +1,8 @@ +use base64::Engine; use std::collections::HashMap; +const BASE64_ENGINE: base64::engine::GeneralPurpose = base64::engine::general_purpose::STANDARD; + /// The value for an attribute that comes from DynamoDb. #[derive(Debug, Clone, Eq, PartialEq)] pub enum AttributeValue { @@ -79,10 +82,216 @@ pub enum AttributeValue { Bs(Vec>), } +impl serde::Serialize for AttributeValue { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + use serde::ser::SerializeMap; + + match self { + AttributeValue::N(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("N", inner)?; + map.end() + } + AttributeValue::S(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("S", inner)?; + map.end() + } + AttributeValue::Bool(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("BOOL", inner)?; + map.end() + } + AttributeValue::B(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("B", &BASE64_ENGINE.encode(inner))?; + map.end() + } + AttributeValue::Null(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("NULL", inner)?; + map.end() + } + AttributeValue::M(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("M", inner)?; + map.end() + } + AttributeValue::L(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("L", inner)?; + map.end() + } + AttributeValue::Ss(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("SS", inner)?; + map.end() + } + AttributeValue::Ns(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + map.serialize_entry("NS", inner)?; + map.end() + } + AttributeValue::Bs(inner) => { + let mut map = serializer.serialize_map(Some(1))?; + let items: Vec = inner + .iter() + .map(|item| BASE64_ENGINE.encode(item)) + .collect(); + map.serialize_entry("BS", &items)?; + map.end() + } + } + } +} + +impl<'de> serde::Deserialize<'de> for AttributeValue { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = AttributeValue; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str(r#"an object with a single key "N", "S", "BOOL", "B", "NULL", "M", "L", "SS", "NS", or "BS""#) + } + + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + use serde::de::Error; + + let first_key: String = match map.next_key()? { + Some(key) => key, + None => { + return Err(A::Error::custom( + "Expected exactly one key in the object, found none", + )) + } + }; + + let attribute_value = match first_key.as_str() { + "N" => AttributeValue::N(map.next_value()?), + "S" => AttributeValue::S(map.next_value()?), + "BOOL" => AttributeValue::Bool(map.next_value()?), + "B" => { + let string: String = map.next_value()?; + let bytes = BASE64_ENGINE.decode(string).map_err(|err| { + A::Error::custom(format!("Failed to decode base64: {err}")) + })?; + AttributeValue::B(bytes) + } + "NULL" => AttributeValue::Null(map.next_value()?), + "M" => AttributeValue::M(map.next_value()?), + "L" => AttributeValue::L(map.next_value()?), + "SS" => AttributeValue::Ss(map.next_value()?), + "NS" => AttributeValue::Ns(map.next_value()?), + "BS" => { + let strings: Vec = map.next_value()?; + let mut byte_entries = Vec::with_capacity(strings.len()); + for string in strings { + let bytes = base64::engine::general_purpose::STANDARD + .decode(string) + .map_err(|err| { + A::Error::custom(format!("Failed to decode base64: {err}")) + })?; + byte_entries.push(bytes); + } + AttributeValue::Bs(byte_entries) + } + key => { + return Err(A::Error::custom(format!( + "The key '{key}' is not a known DynamoDB prefix" + ))) + } + }; + + if map.next_key::()?.is_some() { + return Err(A::Error::custom( + "Expected exactly one key in the object, found multiple keys", + )); + } + + Ok(attribute_value) + } + } + + let visitor = Visitor; + deserializer.deserialize_map(visitor) + } +} + +impl<'de> serde::Deserialize<'de> for Item { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + HashMap::deserialize(deserializer).map(Item) + } +} + +impl serde::Serialize for Item { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.0.serialize(serializer) + } +} + /// An item that comes from DynamoDb. -#[derive(Debug, Clone, Eq, PartialEq)] +#[derive(Debug, Clone, Eq, PartialEq, Default)] pub struct Item(HashMap); +impl Item { + /// Get a reference to the inner HashMap + pub fn inner(&self) -> &HashMap { + &self.0 + } + + /// Get a mutable reference to the inner HashMap + pub fn inner_mut(&mut self) -> &mut HashMap { + &mut self.0 + } + + /// Take the inner HashMap + pub fn into_inner(self) -> HashMap { + self.0 + } +} + +impl AsRef> for Item { + fn as_ref(&self) -> &HashMap { + self.inner() + } +} + +impl AsMut> for Item { + fn as_mut(&mut self) -> &mut HashMap { + self.inner_mut() + } +} + +impl std::ops::Deref for Item { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + self.inner() + } +} + +impl std::ops::DerefMut for Item { + fn deref_mut(&mut self) -> &mut Self::Target { + self.inner_mut() + } +} + impl From for HashMap where T: From, @@ -128,3 +337,283 @@ where Items(items.into_iter().map(Into::into).collect()) } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn deserialize_from_example() { + // Example from https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/S3DataExport.Output.html + let subject = json!({ + "Authors":{ + "SS":[ + "Author1", + "Author2" + ] + }, + "Dimensions":{ + "S":"8.5 x 11.0 x 1.5" + }, + "ISBN":{ + "S":"333-3333333333" + }, + "Id":{ + "N":"103" + }, + "InPublication":{ + "BOOL":false + }, + "PageCount":{ + "N":"600" + }, + "Price":{ + "N":"2000" + }, + "ProductCategory":{ + "S":"Book" + }, + "Title":{ + "S":"Book 103 Title" + } + }); + + let item: Item = + serde_json::from_value(subject).expect("expected successful deserialization"); + + assert_eq!( + item, + Item(HashMap::from([ + ( + String::from("Authors"), + AttributeValue::Ss(vec![String::from("Author1"), String::from("Author2")]) + ), + ( + String::from("Dimensions"), + AttributeValue::S(String::from("8.5 x 11.0 x 1.5")) + ), + ( + String::from("ISBN"), + AttributeValue::S(String::from("333-3333333333")) + ), + (String::from("Id"), AttributeValue::N(String::from("103"))), + (String::from("InPublication"), AttributeValue::Bool(false)), + ( + String::from("PageCount"), + AttributeValue::N(String::from("600")) + ), + ( + String::from("Price"), + AttributeValue::N(String::from("2000")) + ), + ( + String::from("ProductCategory"), + AttributeValue::S(String::from("Book")) + ), + ( + String::from("Title"), + AttributeValue::S(String::from("Book 103 Title")) + ), + ])) + ) + } + + #[test] + fn deserialize_exhaustive() { + let subject = json!({ + "n_example": { "N": "123.45" }, + "s_example": { "S": "Hello" }, + "bool_example": { "BOOL": true }, + "b_example": { "B": "dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk" }, + "null_example": { "NULL": true }, + "m_example": { "M": {"Name": {"S": "Joe"}, "Age": {"N": "35"}} }, + "l_example": { "L": [ {"S": "Cookies"} , {"S": "Coffee"}, {"N": "3.14159"}] }, + "ss_example": { "SS": ["Giraffe", "Hippo" ,"Zebra"] }, + "ns_example": { "NS": ["42.2", "-19", "7.5", "3.14"] }, + "bs_example": { "BS": ["U3Vubnk=", "UmFpbnk=", "U25vd3k="] }, + }); + + let item: Item = + serde_json::from_value(subject).expect("expected successful deserialization"); + + assert_eq!( + item, + Item(HashMap::from([ + ( + String::from("n_example"), + AttributeValue::N(String::from("123.45")) + ), + ( + String::from("s_example"), + AttributeValue::S(String::from("Hello")) + ), + (String::from("bool_example"), AttributeValue::Bool(true)), + ( + String::from("b_example"), + AttributeValue::B(Vec::from(b"this text is base64-encoded".as_slice())) + ), + (String::from("null_example"), AttributeValue::Null(true)), + ( + String::from("m_example"), + AttributeValue::M(HashMap::from([ + (String::from("Name"), AttributeValue::S(String::from("Joe"))), + (String::from("Age"), AttributeValue::N(String::from("35"))), + ])) + ), + ( + String::from("l_example"), + AttributeValue::L(vec![ + AttributeValue::S(String::from("Cookies")), + AttributeValue::S(String::from("Coffee")), + AttributeValue::N(String::from("3.14159")) + ]) + ), + ( + String::from("ss_example"), + AttributeValue::Ss(vec![ + String::from("Giraffe"), + String::from("Hippo"), + String::from("Zebra") + ]) + ), + ( + String::from("ns_example"), + AttributeValue::Ns(vec![ + String::from("42.2"), + String::from("-19"), + String::from("7.5"), + String::from("3.14") + ]) + ), + ( + String::from("bs_example"), + AttributeValue::Bs(vec![ + Vec::from(b"Sunny".as_slice()), + Vec::from(b"Rainy".as_slice()), + Vec::from(b"Snowy".as_slice()) + ]) + ), + ])) + ); + } + + #[test] + fn deserialize_error_invalid_key() { + let err = serde_json::from_str::(r#"{ "X": "1" }"#) + .expect_err("expected to fail"); + assert!(err.to_string().contains("'X'")) + } + + #[test] + fn deserialize_error_zero_keys() { + let err = serde_json::from_str::(r#"{}"#).expect_err("expected to fail"); + assert!(err.to_string().contains("none")) + } + + #[test] + fn deserialize_error_multiple_keys() { + let err = serde_json::from_str::(r#"{ "S": "1", "N": "1" }"#) + .expect_err("expected to fail"); + assert!(err.to_string().contains("multiple keys")) + } + + #[test] + fn deserialize_error_base64_b() { + let err = serde_json::from_str::(r#"{ "B": "X" }"#) + .expect_err("expected to fail"); + assert!(err.to_string().contains("base64")) + } + + #[test] + fn deserialize_error_base64_bs() { + let err = serde_json::from_str::(r#"{ "BS": ["X"] }"#) + .expect_err("expected to fail"); + assert!(err.to_string().contains("base64")) + } + + #[test] + fn deserialize_expecting() { + let err = serde_json::from_str::(r#"42"#).expect_err("expected to fail"); + assert!(err + .to_string() + .contains("expected an object with a single key")); + } + + #[test] + fn serialize_exhaustive() { + let subject = Item(HashMap::from([ + ( + String::from("n_example"), + AttributeValue::N(String::from("123.45")), + ), + ( + String::from("s_example"), + AttributeValue::S(String::from("Hello")), + ), + (String::from("bool_example"), AttributeValue::Bool(true)), + ( + String::from("b_example"), + AttributeValue::B(Vec::from(b"this text is base64-encoded".as_slice())), + ), + (String::from("null_example"), AttributeValue::Null(true)), + ( + String::from("m_example"), + AttributeValue::M(HashMap::from([ + (String::from("Name"), AttributeValue::S(String::from("Joe"))), + (String::from("Age"), AttributeValue::N(String::from("35"))), + ])), + ), + ( + String::from("l_example"), + AttributeValue::L(vec![ + AttributeValue::S(String::from("Cookies")), + AttributeValue::S(String::from("Coffee")), + AttributeValue::N(String::from("3.14159")), + ]), + ), + ( + String::from("ss_example"), + AttributeValue::Ss(vec![ + String::from("Giraffe"), + String::from("Hippo"), + String::from("Zebra"), + ]), + ), + ( + String::from("ns_example"), + AttributeValue::Ns(vec![ + String::from("42.2"), + String::from("-19"), + String::from("7.5"), + String::from("3.14"), + ]), + ), + ( + String::from("bs_example"), + AttributeValue::Bs(vec![ + Vec::from(b"Sunny".as_slice()), + Vec::from(b"Rainy".as_slice()), + Vec::from(b"Snowy".as_slice()), + ]), + ), + ])); + + let json = serde_json::to_value(subject).expect("expected successful deserialization"); + assert_eq!( + json, + json!({ + "n_example": { "N": "123.45" }, + "s_example": { "S": "Hello" }, + "bool_example": { "BOOL": true }, + "b_example": { "B": "dGhpcyB0ZXh0IGlzIGJhc2U2NC1lbmNvZGVk" }, + "null_example": { "NULL": true }, + "m_example": { "M": {"Name": {"S": "Joe"}, "Age": {"N": "35"}} }, + "l_example": { "L": [ {"S": "Cookies"} , {"S": "Coffee"}, {"N": "3.14159"}] }, + "ss_example": { "SS": ["Giraffe", "Hippo" ,"Zebra"] }, + "ns_example": { "NS": ["42.2", "-19", "7.5", "3.14"] }, + "bs_example": { "BS": ["U3Vubnk=", "UmFpbnk=", "U25vd3k="] }, + }) + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index 55b506f..6fb7ade 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -201,7 +201,7 @@ //! 1.0. //! //! To avoid doing a major version bump for every release of `aws-sdk-dynamodb` and -//! `rusoto_dynamodb`, **serde_dynamo** uses features to opt into the correct version of the +//! `aws_lambda_events`, **serde_dynamo** uses features to opt into the correct version of the //! dynamodb library. //! //! See the [modules](#modules) section for all possible features. Feature names are largely @@ -209,6 +209,45 @@ //! because crates.io doesn't support feature names with dots). For example, support for //! `aws-sdk-dynamodb` version `0.13` is enabled with the feature `aws-sdk-dynamodb+0_13`. //! +//! ## Converting to and from DynamoDB JSON +//! +//! In most cases, libraries already exist to handle the raw DynamoDB JSON and convert it into an +//! item. For example, [aws-sdk-dynamodb] deals with the raw JSON if you're making API calls, and +//! [aws_lambda_events] deals with the raw JSON if you're writing lambdas that react on DynamoDB +//! change streams. +//! +//! However, in very rare cases, you may need to convert the DynamoDB JSON yourself. In those cases, +//! both [Item] and [AttributeValue] implement [serde::Serialize] and [serde::Deserialize]. +//! +//! ``` +//! # use serde_dynamo::{AttributeValue, Item}; +//! let input = r#"{ +//! "Id":{ +//! "N":"103" +//! }, +//! "Title":{ +//! "S":"Book 103 Title" +//! }, +//! "Authors":{ +//! "SS":[ +//! "Author1", +//! "Author2" +//! ] +//! }, +//! "InPublication":{ +//! "BOOL":false +//! } +//! }"#; +//! +//! let item: Item = serde_json::from_str(input) +//! .expect("expected to deserialize DynamoDB JSON format"); +//! +//! assert_eq!( +//! item.get("Id").unwrap(), +//! &AttributeValue::N(String::from("103")), +//! ); +//! ``` +//! //! [DynamoDB]: https://aws.amazon.com/dynamodb/ //! [serde]: https://docs.rs/serde //! [serde_json]: https://docs.rs/serde_json