percent-encoding: make sets be values of one type, instead of types that implement a trait

SimonSapin · SimonSapin · commit bc4503b4e9bd · 2019-07-17T15:12:39.000+02:00
diff --git a/Cargo.toml b/Cargo.toml
@@ -39,7 +39,7 @@ bencher = "0.1"
 [dependencies]
 idna = { version = "0.2.0", path = "./idna" }
 matches = "0.1"
-percent-encoding = { version = "1.0.0", path = "./percent_encoding" }
+percent-encoding = { version = "2.0.0", path = "./percent_encoding" }
 serde = {version = "1.0", optional = true}
 
 [[bench]]
diff --git a/percent_encoding/Cargo.toml b/percent_encoding/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "percent-encoding"
-version = "1.0.2"
+version = "2.0.0"
 authors = ["The rust-url developers"]
 description = "Percent encoding and decoding"
 repository = "https://github.com/servo/rust-url/"
diff --git a/percent_encoding/lib.rs b/percent_encoding/lib.rs
@@ -37,8 +37,9 @@ use std::fmt;
 use std::slice;
 use std::str;
 
-/// Represents a set of characters / bytes that should be percent-encoded.
+/// Represents a set of characters or bytes in the ASCII range.
 ///
+/// This is use to represent which characters should be pecent-encoded.
 /// See [encode sets specification](http://url.spec.whatwg.org/#simple-encode-set).
 ///
 /// Different characters need to be encoded in different parts of an URL.
@@ -50,116 +51,81 @@ use std::str;
 ///
 /// A few sets are defined in this module.
 /// Use the [`define_encode_set!`](../macro.define_encode_set!.html) macro to define different ones.
-pub trait EncodeSet: Clone {
-    /// Called with UTF-8 bytes rather than code points.
-    /// Should return true for all non-ASCII bytes.
-    fn contains(&self, byte: u8) -> bool;
+pub struct AsciiSet {
+    mask: [Chunk; 0x80 / BITS_PER_CHUNK],
 }
 
-/// Define a new struct
-/// that implements the [`EncodeSet`](percent_encoding/trait.EncodeSet.html) trait,
-/// for use in [`percent_decode()`](percent_encoding/fn.percent_encode.html)
-/// and related functions.
-///
-/// Parameters are characters to include in the set in addition to those of the base set.
-/// See [encode sets specification](http://url.spec.whatwg.org/#simple-encode-set).
-///
-/// Example
-/// =======
-///
-/// ```rust
-/// #[macro_use] extern crate percent_encoding;
-/// use percent_encoding::{utf8_percent_encode, SIMPLE_ENCODE_SET};
-/// define_encode_set! {
-///     /// This encode set is used in the URL parser for query strings.
-///     pub QUERY_ENCODE_SET = [SIMPLE_ENCODE_SET] | {' ', '"', '#', '<', '>'}
-/// }
-/// # fn main() {
-/// assert_eq!(utf8_percent_encode("foo bar", QUERY_ENCODE_SET).collect::<String>(), "foo%20bar");
-/// # }
-/// ```
-#[macro_export]
-macro_rules! define_encode_set {
-    ($(#[$attr: meta])* pub $name: ident = [$base_set: expr] | {$($ch: pat),*}) => {
-        $(#[$attr])*
-        #[derive(Copy, Clone, Debug)]
-        #[allow(non_camel_case_types)]
-        pub struct $name;
-
-        impl $crate::EncodeSet for $name {
-            #[inline]
-            fn contains(&self, byte: u8) -> bool {
-                match byte as char {
-                    $(
-                        $ch => true,
-                    )*
-                    _ => $base_set.contains(byte)
-                }
-            }
-        }
+type Chunk = u32;
+
+const BITS_PER_CHUNK: usize = 8 * std::mem::size_of::<Chunk>();
+
+impl AsciiSet {
+    /// Called with UTF-8 bytes rather than code points.
+    /// Not used for non-ASCII bytes.
+    const fn contains(&self, byte: u8) -> bool {
+        let chunk = self.mask[byte as usize / BITS_PER_CHUNK];
+        let mask = 1 << (byte as usize % BITS_PER_CHUNK);
+        (chunk & mask) != 0
     }
-}
 
-/// This encode set is used for the path of cannot-be-a-base URLs.
-///
-/// All ASCII charcters less than hexidecimal 20 and greater than 7E are encoded.  This includes
-/// special charcters such as line feed, carriage return, NULL, etc.
-#[derive(Copy, Clone, Debug)]
-#[allow(non_camel_case_types)]
-pub struct SIMPLE_ENCODE_SET;
-
-impl EncodeSet for SIMPLE_ENCODE_SET {
-    #[inline]
-    fn contains(&self, byte: u8) -> bool {
-        byte < 0x20 || byte > 0x7E
+    fn should_percent_encode(&self, byte: u8) -> bool {
+        !byte.is_ascii() || self.contains(byte)
     }
-}
 
-define_encode_set! {
-    /// This encode set is used in the URL parser for query strings.
-    ///
-    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
-    /// space, double quote ("), hash (#), and inequality qualifiers (<), (>) are encoded.
-    pub QUERY_ENCODE_SET = [SIMPLE_ENCODE_SET] | {' ', '"', '#', '<', '>'}
+    pub const fn add(&self, byte: u8) -> Self {
+        let mut mask = self.mask;
+        mask[byte as usize / BITS_PER_CHUNK] |= 1 << (byte as usize % BITS_PER_CHUNK);
+        AsciiSet { mask }
+    }
 }
 
-define_encode_set! {
-    /// This encode set is used for path components.
-    ///
-    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
-    /// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
-    /// question mark (?), and curly brackets ({), (}) are encoded.
-    pub DEFAULT_ENCODE_SET = [QUERY_ENCODE_SET] | {'`', '?', '{', '}'}
-}
+/// https://url.spec.whatwg.org/#c0-control-percent-encode-set
+pub const C0_CONTROL: &AsciiSet = &AsciiSet {
+    mask: [
+        !0_u32, // C0: 0x00 to 0x1F (32 bits set)
+        0,
+        0,
+        1 << (0x7F_u32 % 32), // DEL: 0x7F (one bit set)
+    ],
+};
 
-define_encode_set! {
-    /// This encode set is used for on '/'-separated path segment
-    ///
-    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
-    /// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
-    /// question mark (?), and curly brackets ({), (}), percent sign (%), forward slash (/) are
-    /// encoded.
-    ///
-    /// # Note
-    ///
-    /// For [special URLs](https://url.spec.whatwg.org/#is-special), the backslash (\) character should
-    /// additionally be escaped, but that is *not* included in this encode set.
-    pub PATH_SEGMENT_ENCODE_SET = [DEFAULT_ENCODE_SET] | {'%', '/'}
-}
+/// https://url.spec.whatwg.org/#fragment-percent-encode-set
+pub const FRAGMENT: &AsciiSet = &C0_CONTROL.add(b' ').add(b'"').add(b'<').add(b'>').add(b'`');
 
-define_encode_set! {
-    /// This encode set is used for username and password.
-    ///
-    /// Aside from special chacters defined in the [`SIMPLE_ENCODE_SET`](struct.SIMPLE_ENCODE_SET.html),
-    /// space, double quote ("), hash (#), inequality qualifiers (<), (>), backtick (`),
-    /// question mark (?), and curly brackets ({), (}), forward slash (/), colon (:), semi-colon (;),
-    /// equality (=), at (@), backslash (\\), square brackets ([), (]), caret (\^), and pipe (|) are
-    /// encoded.
-    pub USERINFO_ENCODE_SET = [DEFAULT_ENCODE_SET] | {
-        '/', ':', ';', '=', '@', '[', '\\', ']', '^', '|'
+/// https://url.spec.whatwg.org/#path-percent-encode-set
+pub const PATH: &AsciiSet = &FRAGMENT.add(b'#').add(b'?').add(b'{').add(b'}');
+
+/// https://url.spec.whatwg.org/#userinfo-percent-encode-set
+pub const USERINFO: &AsciiSet = &PATH
+    .add(b'/')
+    .add(b':')
+    .add(b';')
+    .add(b'=')
+    .add(b'@')
+    .add(b'[')
+    .add(b'\\')
+    .add(b']')
+    .add(b'^')
+    .add(b'|');
+
+macro_rules! static_assert {
+    ($( $bool: expr, )+) => {
+        fn _static_assert() {
+            $(
+                let _ = std::mem::transmute::<[u8; $bool as usize], u8>;
+            )+
+        }
     }
 }
 
+static_assert! {
+    C0_CONTROL.contains(0x00),
+    C0_CONTROL.contains(0x1F),
+    !C0_CONTROL.contains(0x20),
+    !C0_CONTROL.contains(0x7E),
+    C0_CONTROL.contains(0x7F),
+}
+
 /// Return the percent-encoding of the given bytes.
 ///
 /// This is unconditional, unlike `percent_encode()` which uses an encode set.
@@ -214,10 +180,10 @@ pub fn percent_encode_byte(byte: u8) -> &'static str {
 /// assert_eq!(percent_encode(b"foo bar?", DEFAULT_ENCODE_SET).to_string(), "foo%20bar%3F");
 /// ```
 #[inline]
-pub fn percent_encode<E: EncodeSet>(input: &[u8], encode_set: E) -> PercentEncode<E> {
+pub fn percent_encode<'a>(input: &'a [u8], encode_set: &'static AsciiSet) -> PercentEncode<'a> {
     PercentEncode {
         bytes: input,
-        encode_set: encode_set,
+        encode_set,
     }
 }
 
@@ -233,35 +199,32 @@ pub fn percent_encode<E: EncodeSet>(input: &[u8], encode_set: E) -> PercentEncod
 /// assert_eq!(utf8_percent_encode("foo bar?", DEFAULT_ENCODE_SET).to_string(), "foo%20bar%3F");
 /// ```
 #[inline]
-pub fn utf8_percent_encode<E: EncodeSet>(input: &str, encode_set: E) -> PercentEncode<E> {
+pub fn utf8_percent_encode<'a>(input: &'a str, encode_set: &'static AsciiSet) -> PercentEncode<'a> {
     percent_encode(input.as_bytes(), encode_set)
 }
 
 /// The return type of `percent_encode()` and `utf8_percent_encode()`.
-#[derive(Clone, Debug)]
-pub struct PercentEncode<'a, E: EncodeSet> {
+#[derive(Clone)]
+pub struct PercentEncode<'a> {
     bytes: &'a [u8],
-    encode_set: E,
+    encode_set: &'static AsciiSet,
 }
 
-impl<'a, E: EncodeSet> Iterator for PercentEncode<'a, E> {
+impl<'a> Iterator for PercentEncode<'a> {
     type Item = &'a str;
 
     fn next(&mut self) -> Option<&'a str> {
         if let Some((&first_byte, remaining)) = self.bytes.split_first() {
-            if self.encode_set.contains(first_byte) {
+            if self.encode_set.should_percent_encode(first_byte) {
                 self.bytes = remaining;
                 Some(percent_encode_byte(first_byte))
             } else {
-                assert!(first_byte.is_ascii());
                 for (i, &byte) in remaining.iter().enumerate() {
-                    if self.encode_set.contains(byte) {
+                    if self.encode_set.should_percent_encode(byte) {
                         // 1 for first_byte + i for previous iterations of this loop
                         let (unchanged_slice, remaining) = self.bytes.split_at(1 + i);
                         self.bytes = remaining;
                         return Some(unsafe { str::from_utf8_unchecked(unchanged_slice) });
-                    } else {
-                        assert!(byte.is_ascii());
                     }
                 }
                 let unchanged_slice = self.bytes;
@@ -282,7 +245,7 @@ impl<'a, E: EncodeSet> Iterator for PercentEncode<'a, E> {
     }
 }
 
-impl<'a, E: EncodeSet> fmt::Display for PercentEncode<'a, E> {
+impl<'a> fmt::Display for PercentEncode<'a> {
     fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
         for c in (*self).clone() {
             formatter.write_str(c)?
@@ -291,8 +254,8 @@ impl<'a, E: EncodeSet> fmt::Display for PercentEncode<'a, E> {
     }
 }
 
-impl<'a, E: EncodeSet> From<PercentEncode<'a, E>> for Cow<'a, str> {
-    fn from(mut iter: PercentEncode<'a, E>) -> Self {
+impl<'a> From<PercentEncode<'a>> for Cow<'a, str> {
+    fn from(mut iter: PercentEncode<'a>) -> Self {
         match iter.next() {
             None => "".into(),
             Some(first) => match iter.next() {
diff --git a/src/host.rs b/src/host.rs
@@ -8,7 +8,7 @@
 
 use idna;
 use parser::{ParseError, ParseResult};
-use percent_encoding::{percent_decode, utf8_percent_encode, SIMPLE_ENCODE_SET};
+use percent_encoding::{percent_decode, utf8_percent_encode, C0_CONTROL};
 use std::cmp;
 use std::fmt::{self, Formatter};
 use std::net::{Ipv4Addr, Ipv6Addr};
@@ -207,7 +207,7 @@ impl Host<String> {
         {
             return Err(ParseError::InvalidDomainCharacter);
         }
-        let s = utf8_percent_encode(input, SIMPLE_ENCODE_SET).to_string();
+        let s = utf8_percent_encode(input, C0_CONTROL).to_string();
         Ok(Host::Domain(s))
     }
 }
diff --git a/src/lib.rs b/src/lib.rs
@@ -110,17 +110,13 @@ assert_eq!(css_url.as_str(), "http://servo.github.io/rust-url/main.css");
 #[macro_use]
 extern crate matches;
 extern crate idna;
+extern crate percent_encoding;
 #[cfg(feature = "serde")]
 extern crate serde;
-#[macro_use]
-extern crate percent_encoding;
 
 use host::HostInternal;
 use parser::{to_u32, Context, Parser, SchemeType};
-use percent_encoding::{
-    percent_decode, percent_encode, utf8_percent_encode, PATH_SEGMENT_ENCODE_SET,
-    USERINFO_ENCODE_SET,
-};
+use percent_encoding::{percent_decode, percent_encode, utf8_percent_encode, USERINFO};
 use std::borrow::Borrow;
 use std::cmp;
 #[cfg(feature = "serde")]
@@ -1229,8 +1225,9 @@ impl Url {
         if let Some(input) = query {
             self.query_start = Some(to_u32(self.serialization.len()).unwrap());
             self.serialization.push('?');
+            let scheme_type = SchemeType::from(self.scheme());
             let scheme_end = self.scheme_end;
-            self.mutate(|parser| parser.parse_query(scheme_end, parser::Input::new(input)));
+            self.mutate(|parser| parser.parse_query(scheme_type, scheme_end, parser::Input::new(input)));
         }
 
         self.restore_already_parsed_fragment(fragment);
@@ -1729,7 +1726,7 @@ impl Url {
             self.serialization.truncate(self.username_end as usize);
             self.serialization.push(':');
             self.serialization
-                .extend(utf8_percent_encode(password, USERINFO_ENCODE_SET));
+                .extend(utf8_percent_encode(password, USERINFO));
             self.serialization.push('@');
 
             let old_host_start = self.host_start;
@@ -1824,7 +1821,7 @@ impl Url {
         let after_username = self.slice(self.username_end..).to_owned();
         self.serialization.truncate(username_start as usize);
         self.serialization
-            .extend(utf8_percent_encode(username, USERINFO_ENCODE_SET));
+            .extend(utf8_percent_encode(username, USERINFO));
 
         let mut removed_bytes = self.username_end;
         self.username_end = to_u32(self.serialization.len()).unwrap();
@@ -2290,6 +2287,8 @@ impl<'de> serde::Deserialize<'de> for Url {
     }
 }
 
+const PATH_SEGMENT: &percent_encoding::AsciiSet = &percent_encoding::PATH.add(b'/');
+
 #[cfg(any(unix, target_os = "redox"))]
 fn path_to_file_url_segments(
     path: &Path,
@@ -2307,7 +2306,7 @@ fn path_to_file_url_segments(
         serialization.push('/');
         serialization.extend(percent_encode(
             component.as_os_str().as_bytes(),
-            PATH_SEGMENT_ENCODE_SET,
+            PATH_SEGMENT,
         ));
     }
     if empty {
@@ -2355,7 +2354,7 @@ fn path_to_file_url_segments_windows(
                 host_internal = host.into();
                 serialization.push('/');
                 let share = share.to_str().ok_or(())?;
-                serialization.extend(percent_encode(share.as_bytes(), PATH_SEGMENT_ENCODE_SET));
+                serialization.extend(percent_encode(share.as_bytes(), PATH_SEGMENT));
             }
             _ => return Err(()),
         },
@@ -2370,10 +2369,7 @@ fn path_to_file_url_segments_windows(
         // FIXME: somehow work with non-unicode?
         let component = component.as_os_str().to_str().ok_or(())?;
         serialization.push('/');
-        serialization.extend(percent_encode(
-            component.as_bytes(),
-            PATH_SEGMENT_ENCODE_SET,
-        ));
+        serialization.extend(percent_encode(component.as_bytes(), PATH_SEGMENT));
     }
     Ok((host_end, host_internal))
 }
diff --git a/src/parser.rs b/src/parser.rs
diff --git a/tests/unit.rs b/tests/unit.rs

Original file line number	Diff line number	Diff line change
`@@ -8,7 +8,7 @@`
`8`	`8`
`9`	`9`	`use idna;`
`10`	`10`	`use parser::{ParseError, ParseResult};`
`11`		`-use percent_encoding::{percent_decode, utf8_percent_encode, SIMPLE_ENCODE_SET};`
	`11`	`+use percent_encoding::{percent_decode, utf8_percent_encode, C0_CONTROL};`
`12`	`12`	`use std::cmp;`
`13`	`13`	`use std::fmt::{self, Formatter};`
`14`	`14`	`use std::net::{Ipv4Addr, Ipv6Addr};`
`@@ -207,7 +207,7 @@ impl Host<String> {`
`207`	`207`	`{`
`208`	`208`	`return Err(ParseError::InvalidDomainCharacter);`
`209`	`209`	`}`
`210`		`- let s = utf8_percent_encode(input, SIMPLE_ENCODE_SET).to_string();`
	`210`	`+ let s = utf8_percent_encode(input, C0_CONTROL).to_string();`
`211`	`211`	`Ok(Host::Domain(s))`
`212`	`212`	`}`
`213`	`213`	`}`