Merge pull request #384 from bouk/stream-literals

Support top-level primitives in StreamDeserializer

src/de.rs

@@ -1302,17 +1302,16 @@ where
 /// A stream deserializer can be created from any JSON deserializer using the
 /// `Deserializer::into_iter` method.
 ///
-/// The data must consist of JSON arrays and JSON objects optionally separated
-/// by whitespace. A null, boolean, number, or string at the top level are all
-/// errors.
+/// The data can consist of any JSON value. Values need to be a self-delineating value e.g.
+/// arrays, objects, or strings, or be followed by whitespace or a self-delineating value.
 ///
 /// ```rust
 /// extern crate serde_json;
 ///
 /// use serde_json::{Deserializer, Value};
 ///
 /// fn main() {
-///     let data = "{\"k\": 3}  {}  [0, 1, 2]";
+///     let data = "{\"k\": 3}1\"cool\"\"stuff\" 3{}  [0, 1, 2]";
 ///
 ///     let stream = Deserializer::from_str(data).into_iter::<Value>();
 ///
@@ -1385,6 +1384,18 @@ where
     pub fn byte_offset(&self) -> usize {
         self.offset
     }
+
+    fn peek_end_of_value(&mut self) -> Result<()> {
+        match try!(self.de.peek()) {
+            Some(b' ') | Some(b'\n') | Some(b'\t') | Some(b'\r') |
+            Some(b'"') | Some(b'[') | Some(b']') | Some(b'{') |
+            Some(b'}') | Some(b',') | Some(b':') | None => Ok(()),
+            Some(_) => {
+                let pos = self.de.read.peek_position();
+                Err(Error::syntax(ErrorCode::TrailingCharacters, pos.line, pos.column))
+            },
+        }
+    }
 }
 
 impl<'de, R, T> Iterator for StreamDeserializer<'de, R, T>
@@ -1403,16 +1414,30 @@ where
                 self.offset = self.de.read.byte_offset();
                 None
             }
-            Ok(Some(b'{')) | Ok(Some(b'[')) => {
+            Ok(Some(b)) => {
+                // If the value does not have a clear way to show the end of the value
+                // (like numbers, null, true etc.) we have to look for whitespace or
+                // the beginning of a self-delineated value.
+                let self_delineated_value = match b {
+                    b'[' | b'"' | b'{' => true,
+                    _ => false,
+                };
                 self.offset = self.de.read.byte_offset();
                 let result = de::Deserialize::deserialize(&mut self.de);
-                if result.is_ok() {
-                    self.offset = self.de.read.byte_offset();
-                }
-                Some(result)
+
+                Some(match result {
+                    Ok(value) => {
+                        self.offset = self.de.read.byte_offset();
+                        if self_delineated_value {
+                            Ok(value)
+                        } else {
+                            self.peek_end_of_value().map(|_| value)
+                        }
+                    }
+                    Err(e) => Err(e)
+                })
             }
-            Ok(Some(_)) => Some(Err(self.de.peek_error(ErrorCode::ExpectedObjectOrArray))),
-            Err(e) => Some(Err(e)),
+            Err(e) => Some(Err(e))
         }
     }
 }

tests/stream.rs

@@ -107,15 +107,60 @@ fn test_json_stream_empty() {
 
 #[test]
 fn test_json_stream_primitive() {
-    let data = "{} true";
+    let data = "{} true{}1[]\nfalse\"hey\"2 ";
 
     test_stream!(
         data, Value, |stream| {
             assert_eq!(stream.next().unwrap().unwrap(), json!({}));
             assert_eq!(stream.byte_offset(), 2);
 
+            assert_eq!(stream.next().unwrap().unwrap(), true);
+            assert_eq!(stream.byte_offset(), 7);
+
+            assert_eq!(stream.next().unwrap().unwrap(), json!({}));
+            assert_eq!(stream.byte_offset(), 9);
+
+            assert_eq!(stream.next().unwrap().unwrap(), 1);
+            assert_eq!(stream.byte_offset(), 10);
+
+            assert_eq!(stream.next().unwrap().unwrap(), json!([]));
+            assert_eq!(stream.byte_offset(), 12);
+
+            assert_eq!(stream.next().unwrap().unwrap(), false);
+            assert_eq!(stream.byte_offset(), 18);
+
+            assert_eq!(stream.next().unwrap().unwrap(), "hey");
+            assert_eq!(stream.byte_offset(), 23);
+
+            assert_eq!(stream.next().unwrap().unwrap(), 2);
+            assert_eq!(stream.byte_offset(), 24);
+
+            assert!(stream.next().is_none());
+            assert_eq!(stream.byte_offset(), 25);
+        }
+    );
+}
+
+#[test]
+fn test_json_stream_invalid_literal() {
+    let data = "truefalse";
+
+    test_stream!(
+        data, Value, |stream| {
+            let second = stream.next().unwrap().unwrap_err();
+            assert_eq!(second.to_string(), "trailing characters at line 1 column 5");
+        }
+    );
+}
+
+#[test]
+fn test_json_stream_invalid_number() {
+    let data = "1true";
+
+    test_stream!(
+        data, Value, |stream| {
             let second = stream.next().unwrap().unwrap_err();
-            assert_eq!(second.to_string(), "expected `{` or `[` at line 1 column 4");
+            assert_eq!(second.to_string(), "trailing characters at line 1 column 2");
         }
     );
 }