Add proper XID_Start and XID_Continue rules and use CharPos for span comparison, closes rust-lang#15679

Author: fhahn

src/grammar/RustLexer.g4

@@ -11,11 +11,7 @@ tokens {
     COMMENT
 }
 
-/* Note: due to antlr limitations, we can't represent XID_start and
- * XID_continue properly. ASCII-only substitute. */
-
-fragment XID_start : [_a-zA-Z] ;
-fragment XID_continue : [_a-zA-Z0-9] ;
+import xidstart , xidcontinue;
 
 
 /* Expression-operator symbols */
@@ -194,7 +190,7 @@ LIT_STR_RAW
   : 'r' LIT_STR_RAW_INNER SUFFIX?
   ;
 
-IDENT : XID_start XID_continue* ;
+IDENT : XID_Start XID_Continue* ;
 
 LIFETIME : '\'' IDENT ;
 

src/grammar/verify.rs

@@ -28,6 +28,7 @@ use rustc::session::{self, config};
 
 use syntax::ast;
 use syntax::ast::Name;
+use syntax::codemap::Pos;
 use syntax::parse::token;
 use syntax::parse::lexer::TokenAndSpan;
 
@@ -233,6 +234,13 @@ fn tok_cmp(a: &token::Token, b: &token::Token) -> bool {
     }
 }
 
+fn span_cmp(rust_sp: syntax::codemap::Span, antlr_sp: syntax::codemap::Span, cm: &syntax::codemap::CodeMap) -> bool {
+    println!("{} {}", cm.bytepos_to_file_charpos(rust_sp.lo).to_uint(), cm.bytepos_to_file_charpos(rust_sp.hi).to_uint());
+    antlr_sp.lo.to_uint() == cm.bytepos_to_file_charpos(rust_sp.lo).to_uint() &&
+    antlr_sp.hi.to_uint() == cm.bytepos_to_file_charpos(rust_sp.hi).to_uint() &&
+    antlr_sp.expn_id == rust_sp.expn_id
+}
+
 fn main() {
     fn next(r: &mut lexer::StringReader) -> TokenAndSpan {
         use syntax::parse::lexer::Reader;
@@ -258,14 +266,15 @@ fn main() {
                                            code,
                                            String::from_str("<n/a>"));
     let mut lexer = lexer::StringReader::new(session.diagnostic(), filemap);
+    let ref cm = lexer.span_diagnostic.cm;
 
     for antlr_tok in antlr_tokens {
         let rustc_tok = next(&mut lexer);
         if rustc_tok.tok == token::Eof && antlr_tok.tok == token::Eof {
             continue
         }
 
-        assert!(rustc_tok.sp == antlr_tok.sp, "{} and {} have different spans", rustc_tok,
+        assert!(span_cmp(rustc_tok.sp, antlr_tok.sp, cm), "{:?} and {:?} have different spans", rustc_tok,
                 antlr_tok);
 
         macro_rules! matches {