auto merge of #14613 : schmee/rust/utf16-iterator, r=huonw

Closes #14358.

The tests are not yet moved to `utf16_iter`, so this probably won't compile. I'm submitting this PR anyway so it can be reviewed and since it was mentioned in #14611.

This deprecates `.to_utf16`. `x.to_utf16()` should be replaced by either `x.utf16_iter().collect::<Vec<u16>>()` (the type annotation may be optional), or just `x.utf16_iter()` directly, if it can be used in an iterator context.

[breaking-change]

cc @huonw

Author: bors

src/libcollections/str.rs

@@ -803,15 +803,9 @@ pub trait StrAllocating: Str {
     }
 
     /// Converts to a vector of `u16` encoded as UTF-16.
+    #[deprecated = "use `utf16_iter` instead"]
     fn to_utf16(&self) -> Vec<u16> {
-        let me = self.as_slice();
-        let mut u = Vec::new();
-        for ch in me.chars() {
-            let mut buf = [0u16, ..2];
-            let n = ch.encode_utf16(buf /* as mut slice! */);
-            u.push_all(buf.slice_to(n));
-        }
-        u
+        self.as_slice().utf16_iter().collect::<Vec<u16>>()
     }
 
     /// Given a string, make a new string with repeated copies of it.
@@ -1619,14 +1613,17 @@ mod tests {
 
         for p in pairs.iter() {
             let (s, u) = (*p).clone();
+            let s_as_utf16 = s.as_slice().utf16_iter().collect::<Vec<u16>>();
+            let u_as_string = from_utf16(u.as_slice()).unwrap();
+
             assert!(is_utf16(u.as_slice()));
-            assert_eq!(s.to_utf16(), u);
+            assert_eq!(s_as_utf16, u);
 
-            assert_eq!(from_utf16(u.as_slice()).unwrap(), s);
+            assert_eq!(u_as_string, s);
             assert_eq!(from_utf16_lossy(u.as_slice()), s);
 
-            assert_eq!(from_utf16(s.to_utf16().as_slice()).unwrap(), s);
-            assert_eq!(from_utf16(u.as_slice()).unwrap().to_utf16(), u);
+            assert_eq!(from_utf16(s_as_utf16.as_slice()).unwrap(), s);
+            assert_eq!(u_as_string.as_slice().utf16_iter().collect::<Vec<u16>>(), u);
         }
     }
 

src/libcore/str.rs

@@ -16,6 +16,7 @@
 
 use mem;
 use char;
+use char::Char;
 use clone::Clone;
 use cmp;
 use cmp::{PartialEq, Eq};
@@ -554,6 +555,40 @@ impl<'a> Iterator<&'a str> for StrSplits<'a> {
     }
 }
 
+/// External iterator for a string's UTF16 codeunits.
+/// Use with the `std::iter` module.
+#[deriving(Clone)]
+pub struct Utf16CodeUnits<'a> {
+    chars: Chars<'a>,
+    extra: u16
+}
+
+impl<'a> Iterator<u16> for Utf16CodeUnits<'a> {
+    #[inline]
+    fn next(&mut self) -> Option<u16> {
+        if self.extra != 0 {
+            let tmp = self.extra;
+            self.extra = 0;
+            return Some(tmp);
+        }
+
+        let mut buf = [0u16, ..2];
+        self.chars.next().map(|ch| {
+            let n = ch.encode_utf16(buf /* as mut slice! */);
+            if n == 2 { self.extra = buf[1]; }
+            buf[0]
+        })
+    }
+
+    #[inline]
+    fn size_hint(&self) -> (uint, Option<uint>) {
+        let (low, high) = self.chars.size_hint();
+        // we could be entirely valid surrogates (2 elements per
+        // char), or entirely non-surrogates (1 element per char)
+        (low / 2, high)
+    }
+}
+
 /*
 Section: Comparing strings
 */
@@ -1612,6 +1647,9 @@ pub trait StrSlice<'a> {
     /// and that it is not reallocated (e.g. by pushing to the
     /// string).
     fn as_ptr(&self) -> *u8;
+
+    /// Return an iterator of `u16` over the string encoded as UTF-16.
+    fn utf16_iter(&self) -> Utf16CodeUnits<'a>;
 }
 
 impl<'a> StrSlice<'a> for &'a str {
@@ -1960,6 +1998,11 @@ impl<'a> StrSlice<'a> for &'a str {
     fn as_ptr(&self) -> *u8 {
         self.repr().data
     }
+
+    #[inline]
+    fn utf16_iter(&self) -> Utf16CodeUnits<'a> {
+        Utf16CodeUnits{ chars: self.chars(), extra: 0}
+    }
 }
 
 impl<'a> Default for &'a str {

src/libnative/io/process.rs

@@ -763,7 +763,7 @@ fn with_envp<T>(env: Option<&[(CString, CString)]>, cb: |*mut c_void| -> T) -> T
                 let kv = format!("{}={}",
                                  pair.ref0().as_str().unwrap(),
                                  pair.ref1().as_str().unwrap());
-                blk.push_all(kv.to_utf16().as_slice());
+                blk.extend(kv.utf16_iter());
                 blk.push(0);
             }