Auto merge of rust-lang#34924 - cgswords:empty_delim, r=nrc

Added empty CloseDelim to tokens for future use.

Description says it all. I added a new DelimToken type, Empty, to indicate a Delimited tokenstream with no actual delimiters (which are variously useful for constructing macro output).

r? @nrc

Author: bors

src/librustdoc/html/highlight.rs

@@ -233,6 +233,7 @@ impl<'a> Classifier<'a> {
             token::Dot | token::DotDot | token::DotDotDot | token::Comma | token::Semi |
                 token::Colon | token::ModSep | token::LArrow | token::OpenDelim(_) |
                 token::CloseDelim(token::Brace) | token::CloseDelim(token::Paren) |
+                token::CloseDelim(token::NoDelim) |
                 token::Question => Class::None,
             token::Dollar => {
                 if self.lexer.peek().tok.is_ident() {

src/libsyntax/ext/quote.rs

@@ -581,9 +581,10 @@ fn mk_binop(cx: &ExtCtxt, sp: Span, bop: token::BinOpToken) -> P<ast::Expr> {
 
 fn mk_delim(cx: &ExtCtxt, sp: Span, delim: token::DelimToken) -> P<ast::Expr> {
     let name = match delim {
-        token::Paren     => "Paren",
-        token::Bracket   => "Bracket",
-        token::Brace     => "Brace",
+        token::Paren   => "Paren",
+        token::Bracket => "Bracket",
+        token::Brace   => "Brace",
+        token::NoDelim => "NoDelim",
     };
     mk_token_path(cx, sp, name)
 }

src/libsyntax/parse/token.rs

@@ -48,6 +48,8 @@ pub enum DelimToken {
     Bracket,
     /// A curly brace: `{` or `}`
     Brace,
+    /// An empty delimiter
+    NoDelim,
 }
 
 #[derive(Clone, RustcEncodable, RustcDecodable, PartialEq, Eq, Hash, Debug, Copy)]

src/libsyntax/print/pprust.rs

@@ -242,6 +242,8 @@ pub fn token_to_string(tok: &Token) -> String {
         token::CloseDelim(token::Bracket) => "]".to_string(),
         token::OpenDelim(token::Brace) => "{".to_string(),
         token::CloseDelim(token::Brace) => "}".to_string(),
+        token::OpenDelim(token::NoDelim) => " ".to_string(),
+        token::CloseDelim(token::NoDelim) => " ".to_string(),
         token::Pound                => "#".to_string(),
         token::Dollar               => "$".to_string(),
         token::Question             => "?".to_string(),
@@ -1777,12 +1779,14 @@ impl<'a> State<'a> {
                 try!(self.head(""));
                 try!(self.bopen());
             }
+            token::NoDelim => {}
         }
         try!(self.print_tts(&m.node.tts));
         match delim {
             token::Paren => self.pclose(),
             token::Bracket => word(&mut self.s, "]"),
             token::Brace => self.bclose(m.span),
+            token::NoDelim => Ok(()),
         }
     }