Properly normalize attribute values

closes tafia#371

Author: dralley

src/events/attributes.rs

@@ -331,6 +331,96 @@ impl<'a> From<(&'a str, &'a str)> for Attribute<'a> {
     }
 }
 
+// 1) All line breaks MUST have been normalized on input to #xA as described in 2.11 End-of-Line Handling, so the rest of this algorithm operates on text normalized in this way.
+// 2) Begin with a normalized value consisting of the empty string.
+// 3) For each character, entity reference, or character reference in the unnormalized attribute value, beginning with the first and continuing to the last, do the following:
+//   * For a character reference, append the referenced character to the normalized value.
+//   * For an entity reference, recursively apply step 3 of this algorithm to the replacement text of the entity.
+//   * For a white space character (#x20, #xD, #xA, #x9), append a space character (#x20) to the normalized value.
+//   * For another character, append the character to the normalized value.
+//
+// If the attribute type is not CDATA, then the XML processor MUST further process the normalized attribute value by discarding any leading and trailing space (#x20) characters,
+// and by replacing sequences of space (#x20) characters by a single space (#x20) character.
+//
+// Note that if the unnormalized attribute value contains a character reference to a white space character other than space (#x20), the normalized value contains the referenced
+// character itself (#xD, #xA or #x9). This contrasts with the case where the unnormalized value contains a white space character (not a reference), which is replaced with a
+// space character (#x20) in the normalized value and also contrasts with the case where the unnormalized value contains an entity reference whose replacement text contains a
+// white space character; being recursively processed, the white space character is replaced with a space character (#x20) in the normalized value.
+fn normalize_attribute_value(attr: Cow<[u8]>) -> Cow<[u8]> {
+    // TODO: character references, entity references
+    // TODO: don't allocated unless needed?
+
+    #[derive(PartialEq)]
+    enum ParseState {
+        SpaceOrStart,
+        CDATA,
+    }
+
+    let mut value: Vec<u8> = Vec::new();
+    // Starting in the state where we think we've added a space means we implicitly skip leading spaces
+    let mut current_state = ParseState::SpaceOrStart;
+    // Used for trimming trailing spaces
+    let mut last_cdata_idx = 0;
+
+    // In one pass, strip leading and trailing spaces and replace sequences of spaces with a single one
+    for ch in attr.as_ref() {
+        match current_state {
+            ParseState::SpaceOrStart => match ch {
+                b'\n' | b'\r' | b'\t' | b' ' => continue,
+                c @ _ => {
+                    current_state = ParseState::CDATA;
+                    last_cdata_idx = value.len();
+                    value.push(*c);
+                }
+            },
+            ParseState::CDATA => match ch {
+                b'\n' | b'\r' | b'\t' | b' ' => {
+                    current_state = ParseState::SpaceOrStart;
+                    value.push(b' ');
+                }
+                c @ _ => {
+                    last_cdata_idx = value.len();
+                    value.push(*c)
+                }
+            },
+        }
+    }
+
+    // Trim any trailing spaces
+    if current_state == ParseState::SpaceOrStart {
+        value.truncate(last_cdata_idx + 1);
+    }
+
+    Cow::Owned(value)
+
+
+    // let mut value: Vec<u8> = Vec::new();
+
+    // // TODO: replace sequences of spaces
+    // for i in 0..attr.len() {
+    //     let ch = attr[i];
+    //     match ch {
+    //         b'\n' => value.push(b' '),
+    //         b'\r' => value.push(b' '),
+    //         b'\t' => value.push(b' '),
+    //         c @ _ => value.push(c),
+    //     }
+    // }
+
+    // // Position where value starts after whitespace.
+    // let first_non_space_char = value
+    //     .iter()
+    //     .position(|c| !c.is_ascii_whitespace())
+    //     .unwrap_or(0);
+    // // Position where the trailing whitespace starts.
+    // let last_non_space_char = value
+    //     .iter()
+    //     .rposition(|c| !c.is_ascii_whitespace())
+    //     .and_then(|idx| Some(idx + 1))
+    //     .unwrap_or(0);
+    // Cow::Owned(value[first_non_space_char..last_non_space_char].to_vec())
+}
+
 impl<'a> Iterator for Attributes<'a> {
     type Item = Result<Attribute<'a>>;
     fn next(&mut self) -> Option<Self::Item> {
@@ -355,7 +445,7 @@ impl<'a> Iterator for Attributes<'a> {
             ($key:expr, $val:expr) => {
                 Some(Ok(Attribute {
                     key: &self.bytes[$key],
-                    value: Cow::Borrowed(&self.bytes[$val]),
+                    value: normalize_attribute_value(Cow::Borrowed(&self.bytes[$val])),
                 }))
             };
         }
@@ -460,6 +550,52 @@ mod tests {
         assert!(attributes.next().is_none());
     }
 
+    #[test]
+    fn attribute_value_normalization() {
+        let event = b"a attr=\"tab character\tor newline\nor return\rshould be replaced\"";
+        let mut attributes = Attributes::new(event, 0);
+        attributes.with_checks(true);
+        let a = attributes.next().unwrap().unwrap();
+        assert_eq!(
+            a.value.as_ref(),
+            b"tab character or newline or return should be replaced"
+        );
+    }
+
+    #[test]
+    fn test_normalize_attribute_value() {
+        // return, tab, and newline characters (0xD, 0x9, 0xA) must be replaced with a space character
+        assert_eq!(
+            normalize_attribute_value(Cow::Borrowed(b"\rfoo\rbar\tbaz\ndelta\n")).as_ref(),
+            b"foo bar baz delta"
+        );
+        // leading and trailing spaces must be stripped
+        assert_eq!(
+            normalize_attribute_value(Cow::Borrowed(b"  foo ")).as_ref(),
+            b"foo"
+        );
+        // leading space
+        assert_eq!(
+            normalize_attribute_value(Cow::Borrowed(b"  bar")).as_ref(),
+            b"bar"
+        );
+        // trailing space
+        assert_eq!(
+            normalize_attribute_value(Cow::Borrowed(b"baz ")).as_ref(),
+            b"baz"
+        );
+        // sequences of spaces must be replaced with a single space
+        assert_eq!(
+            normalize_attribute_value(Cow::Borrowed(b"   foo bar   baz ")).as_ref(),
+            b"foo bar baz"
+        );
+        // sequence replacement including whitespace alias characters
+        assert_eq!(
+            normalize_attribute_value(Cow::Borrowed(b" \tfoo\tbar \rbaz \n\ndelta\n")).as_ref(),
+            b"foo bar baz delta"
+        );
+    }
+
     #[test]
     fn mixed_quote() {
         let event = b"name a='a' b = \"b\" c='cc\"cc'";