Auto merge of #60387 - Goirad:test-expansion, r=ollie27

Allow cross-compiling doctests

This PR allows doctest to receive a --runtool argument, as well as possibly many --runtool-arg arguments, which are then used to run cross compiled doctests.
Also, functionality has been added to rustdoc to allow it to skip testing doctests on a per-target basis, in the same way that compiletest does it. For example, tagging the doctest with "ignore-sgx" disables testing on any targets that contain "sgx". A plain "ignore" still skips testing on all targets.

See [here](rust-lang/cargo#6892) for the companion PR in the cargo project that extends functionality in Cargo so that it passes the appropriate parameters to rustdoc when cross compiling and testing doctests.

Part of [#6460](rust-lang/cargo#6460)

Author: bors

Diff for: src/doc/rustdoc/src/unstable-features.md

@@ -471,3 +471,53 @@ Some methodology notes about what rustdoc counts in this metric:
 
 Public items that are not documented can be seen with the built-in `missing_docs` lint. Private
 items that are not documented can be seen with Clippy's `missing_docs_in_private_items` lint.
+
+### `--enable-per-target-ignores`: allow `ignore-foo` style filters for doctests
+
+Using this flag looks like this:
+
+```bash
+$ rustdoc src/lib.rs -Z unstable-options --enable-per-target-ignores
+```
+
+This flag allows you to tag doctests with compiltest style `ignore-foo` filters that prevent
+rustdoc from running that test if the target triple string contains foo. For example:
+
+```rust
+///```ignore-foo,ignore-bar
+///assert!(2 == 2);
+///```
+struct Foo;
+```
+
+This will not be run when the build target is `super-awesome-foo` or `less-bar-awesome`.
+If the flag is not enabled, then rustdoc will consume the filter, but do nothing with it, and
+the above example will be run for all targets.
+If you want to preserve backwards compatibility for older versions of rustdoc, you can use
+
+```rust
+///```ignore,ignore-foo
+///assert!(2 == 2);
+///```
+struct Foo;
+```
+
+In older versions, this will be ignored on all targets, but on newer versions `ignore-gnu` will
+override `ignore`.
+
+### `--runtool`, `--runtool-arg`: program to run tests with; args to pass to it
+
+Using thses options looks like this:
+
+```bash
+$ rustdoc src/lib.rs -Z unstable-options --runtool runner --runtool-arg --do-thing --runtool-arg --do-other-thing
+```
+
+These options can be used to run the doctest under a program, and also pass arguments to
+that program. For example, if you want to run your doctests under valgrind you might run
+
+```bash
+$ rustdoc src/lib.rs -Z unstable-options --runtool valgrind
+```
+
+Another use case would be to run a test inside an emulator, or through a Virtual Machine.

Diff for: src/librustdoc/config.rs

@@ -9,7 +9,7 @@ use rustc::session;
 use rustc::session::config::{CrateType, parse_crate_types_from_list};
 use rustc::session::config::{CodegenOptions, DebuggingOptions, ErrorOutputType, Externs};
 use rustc::session::config::{nightly_options, build_codegen_options, build_debugging_options,
-                             get_cmd_lint_options, ExternEntry};
+                             get_cmd_lint_options, host_triple, ExternEntry};
 use rustc::session::search_paths::SearchPath;
 use rustc_driver;
 use rustc_target::spec::TargetTriple;
@@ -54,7 +54,7 @@ pub struct Options {
     /// Debugging (`-Z`) options to pass to the compiler.
     pub debugging_options: DebuggingOptions,
     /// The target used to compile the crate against.
-    pub target: Option<TargetTriple>,
+    pub target: TargetTriple,
     /// Edition used when reading the crate. Defaults to "2015". Also used by default when
     /// compiling doctests from the crate.
     pub edition: Edition,
@@ -77,6 +77,14 @@ pub struct Options {
     /// Optional path to persist the doctest executables to, defaults to a
     /// temporary directory if not set.
     pub persist_doctests: Option<PathBuf>,
+    /// Runtool to run doctests with
+    pub runtool: Option<String>,
+    /// Arguments to pass to the runtool
+    pub runtool_args: Vec<String>,
+    /// Whether to allow ignoring doctests on a per-target basis
+    /// For example, using ignore-foo to ignore running the doctest on any target that
+    /// contains "foo" as a substring
+    pub enable_per_target_ignores: bool,
 
     // Options that affect the documentation process
 
@@ -140,6 +148,9 @@ impl fmt::Debug for Options {
             .field("show_coverage", &self.show_coverage)
             .field("crate_version", &self.crate_version)
             .field("render_options", &self.render_options)
+            .field("runtool", &self.runtool)
+            .field("runtool_args", &self.runtool_args)
+            .field("enable-per-target-ignores", &self.enable_per_target_ignores)
             .finish()
     }
 }
@@ -414,7 +425,9 @@ impl Options {
             }
         }
 
-        let target = matches.opt_str("target").map(|target| {
+        let target = matches.opt_str("target").map_or(
+            TargetTriple::from_triple(host_triple()),
+            |target| {
             if target.ends_with(".json") {
                 TargetTriple::TargetPath(PathBuf::from(target))
             } else {
@@ -466,6 +479,9 @@ impl Options {
         let codegen_options_strs = matches.opt_strs("C");
         let lib_strs = matches.opt_strs("L");
         let extern_strs = matches.opt_strs("extern");
+        let runtool = matches.opt_str("runtool");
+        let runtool_args = matches.opt_strs("runtool-arg");
+        let enable_per_target_ignores = matches.opt_present("enable-per-target-ignores");
 
         let (lint_opts, describe_lints, lint_cap) = get_cmd_lint_options(matches, error_format);
 
@@ -496,6 +512,9 @@ impl Options {
             show_coverage,
             crate_version,
             persist_doctests,
+            runtool,
+            runtool_args,
+            enable_per_target_ignores,
             render_options: RenderOptions {
                 output,
                 external_html,

Diff for: src/librustdoc/core.rs

@@ -13,7 +13,6 @@ use rustc_interface::interface;
 use rustc_driver::abort_on_err;
 use rustc_resolve as resolve;
 use rustc_metadata::cstore::CStore;
-use rustc_target::spec::TargetTriple;
 
 use syntax::source_map;
 use syntax::attr;
@@ -294,7 +293,6 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
         }
     }).collect();
 
-    let host_triple = TargetTriple::from_triple(config::host_triple());
     let crate_types = if proc_macro_crate {
         vec![config::CrateType::ProcMacro]
     } else {
@@ -313,7 +311,7 @@ pub fn run_core(options: RustdocOptions) -> (clean::Crate, RenderInfo, RenderOpt
         lint_cap: Some(lint_cap.unwrap_or_else(|| lint::Forbid)),
         cg: codegen_options,
         externs,
-        target_triple: target.unwrap_or(host_triple),
+        target_triple: target,
         // Ensure that rustdoc works even if rustc is feature-staged
         unstable_features: UnstableFeatures::Allow,
         actually_rustdoc: true,

Diff for: src/librustdoc/html/markdown.rs

@@ -199,7 +199,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
         let ignore;
         let edition;
         if let Some(Event::Start(Tag::CodeBlock(lang))) = event {
-            let parse_result = LangString::parse(&lang, self.check_error_codes);
+            let parse_result = LangString::parse(&lang, self.check_error_codes, false);
             if !parse_result.rust {
                 return Some(Event::Start(Tag::CodeBlock(lang)));
             }
@@ -272,7 +272,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
             ))
         });
 
-        let tooltip = if ignore {
+        let tooltip = if ignore != Ignore::None {
             Some(("This example is not tested".to_owned(), "ignore"))
         } else if compile_fail {
             Some(("This example deliberately fails to compile".to_owned(), "compile_fail"))
@@ -286,7 +286,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
             s.push_str(&highlight::render_with_highlighting(
                 &text,
                 Some(&format!("rust-example-rendered{}",
-                                if ignore { " ignore" }
+                                if ignore != Ignore::None { " ignore" }
                                 else if compile_fail { " compile_fail" }
                                 else if explicit_edition { " edition " }
                                 else { "" })),
@@ -297,7 +297,7 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for CodeBlocks<'_, 'a, I> {
             s.push_str(&highlight::render_with_highlighting(
                 &text,
                 Some(&format!("rust-example-rendered{}",
-                                if ignore { " ignore" }
+                                if ignore != Ignore::None { " ignore" }
                                 else if compile_fail { " compile_fail" }
                                 else if explicit_edition { " edition " }
                                 else { "" })),
@@ -551,7 +551,8 @@ impl<'a, I: Iterator<Item = Event<'a>>> Iterator for Footnotes<'a, I> {
     }
 }
 
-pub fn find_testable_code<T: test::Tester>(doc: &str, tests: &mut T, error_codes: ErrorCodes) {
+pub fn find_testable_code<T: test::Tester>(doc: &str, tests: &mut T, error_codes: ErrorCodes,
+                                           enable_per_target_ignores: bool) {
     let mut parser = Parser::new(doc);
     let mut prev_offset = 0;
     let mut nb_lines = 0;
@@ -564,7 +565,7 @@ pub fn find_testable_code<T: test::Tester>(doc: &str, tests: &mut T, error_codes
                 let block_info = if s.is_empty() {
                     LangString::all_false()
                 } else {
-                    LangString::parse(&*s, error_codes)
+                    LangString::parse(&*s, error_codes, enable_per_target_ignores)
                 };
                 if !block_info.rust {
                     continue;
@@ -607,7 +608,7 @@ pub struct LangString {
     original: String,
     pub should_panic: bool,
     pub no_run: bool,
-    pub ignore: bool,
+    pub ignore: Ignore,
     pub rust: bool,
     pub test_harness: bool,
     pub compile_fail: bool,
@@ -616,13 +617,20 @@ pub struct LangString {
     pub edition: Option<Edition>
 }
 
+#[derive(Eq, PartialEq, Clone, Debug)]
+pub enum Ignore {
+    All,
+    None,
+    Some(Vec<String>),
+}
+
 impl LangString {
     fn all_false() -> LangString {
         LangString {
             original: String::new(),
             should_panic: false,
             no_run: false,
-            ignore: false,
+            ignore: Ignore::None,
             rust: true,  // NB This used to be `notrust = false`
             test_harness: false,
             compile_fail: false,
@@ -632,11 +640,16 @@ impl LangString {
         }
     }
 
-    fn parse(string: &str, allow_error_code_check: ErrorCodes) -> LangString {
+    fn parse(
+        string: &str,
+        allow_error_code_check: ErrorCodes,
+        enable_per_target_ignores: bool
+    ) -> LangString {
         let allow_error_code_check = allow_error_code_check.as_bool();
         let mut seen_rust_tags = false;
         let mut seen_other_tags = false;
         let mut data = LangString::all_false();
+        let mut ignores = vec![];
 
         data.original = string.to_owned();
         let tokens = string.split(|c: char|
@@ -651,7 +664,11 @@ impl LangString {
                     seen_rust_tags = seen_other_tags == false;
                 }
                 "no_run" => { data.no_run = true; seen_rust_tags = !seen_other_tags; }
-                "ignore" => { data.ignore = true; seen_rust_tags = !seen_other_tags; }
+                "ignore" => { data.ignore = Ignore::All; seen_rust_tags = !seen_other_tags; }
+                x if x.starts_with("ignore-") => if enable_per_target_ignores {
+                    ignores.push(x.trim_start_matches("ignore-").to_owned());
+                    seen_rust_tags = !seen_other_tags;
+                }
                 "allow_fail" => { data.allow_fail = true; seen_rust_tags = !seen_other_tags; }
                 "rust" => { data.rust = true; seen_rust_tags = true; }
                 "test_harness" => {
@@ -679,6 +696,10 @@ impl LangString {
                 _ => { seen_other_tags = true }
             }
         }
+        // ignore-foo overrides ignore
+        if !ignores.is_empty() {
+            data.ignore = Ignore::Some(ignores);
+        }
 
         data.rust &= !seen_other_tags || seen_rust_tags;
 
@@ -919,7 +940,7 @@ crate fn rust_code_blocks(md: &str) -> Vec<RustCodeBlock> {
                 let lang_string = if syntax.is_empty() {
                     LangString::all_false()
                 } else {
-                    LangString::parse(&*syntax, ErrorCodes::Yes)
+                    LangString::parse(&*syntax, ErrorCodes::Yes, false)
                 };
 
                 if lang_string.rust {

Diff for: src/librustdoc/html/markdown/tests.rs

@@ -1,4 +1,4 @@
-use super::{ErrorCodes, LangString, Markdown, MarkdownHtml, IdMap};
+use super::{ErrorCodes, LangString, Markdown, MarkdownHtml, IdMap, Ignore};
 use super::plain_summary_line;
 use std::cell::RefCell;
 use syntax::edition::{Edition, DEFAULT_EDITION};
@@ -26,10 +26,10 @@ fn test_unique_id() {
 #[test]
 fn test_lang_string_parse() {
     fn t(s: &str,
-        should_panic: bool, no_run: bool, ignore: bool, rust: bool, test_harness: bool,
+        should_panic: bool, no_run: bool, ignore: Ignore, rust: bool, test_harness: bool,
         compile_fail: bool, allow_fail: bool, error_codes: Vec<String>,
-         edition: Option<Edition>) {
-        assert_eq!(LangString::parse(s, ErrorCodes::Yes), LangString {
+        edition: Option<Edition>) {
+        assert_eq!(LangString::parse(s, ErrorCodes::Yes, true), LangString {
             should_panic,
             no_run,
             ignore,
@@ -42,6 +42,7 @@ fn test_lang_string_parse() {
             edition,
         })
     }
+    let ignore_foo = Ignore::Some(vec!("foo".to_string()));
 
     fn v() -> Vec<String> {
         Vec::new()
@@ -50,23 +51,24 @@ fn test_lang_string_parse() {
     // ignore-tidy-linelength
     // marker                | should_panic | no_run | ignore | rust | test_harness
     //                       | compile_fail | allow_fail | error_codes | edition
-    t("",                      false,         false,   false,   true,  false, false, false, v(), None);
-    t("rust",                  false,         false,   false,   true,  false, false, false, v(), None);
-    t("sh",                    false,         false,   false,   false, false, false, false, v(), None);
-    t("ignore",                false,         false,   true,    true,  false, false, false, v(), None);
-    t("should_panic",          true,          false,   false,   true,  false, false, false, v(), None);
-    t("no_run",                false,         true,    false,   true,  false, false, false, v(), None);
-    t("test_harness",          false,         false,   false,   true,  true,  false, false, v(), None);
-    t("compile_fail",          false,         true,    false,   true,  false, true,  false, v(), None);
-    t("allow_fail",            false,         false,   false,   true,  false, false, true,  v(), None);
-    t("{.no_run .example}",    false,         true,    false,   true,  false, false, false, v(), None);
-    t("{.sh .should_panic}",   true,          false,   false,   false, false, false, false, v(), None);
-    t("{.example .rust}",      false,         false,   false,   true,  false, false, false, v(), None);
-    t("{.test_harness .rust}", false,         false,   false,   true,  true,  false, false, v(), None);
-    t("text, no_run",          false,         true,    false,   false, false, false, false, v(), None);
-    t("text,no_run",           false,         true,    false,   false, false, false, false, v(), None);
-    t("edition2015",           false,         false,   false,   true,  false, false, false, v(), Some(Edition::Edition2015));
-    t("edition2018",           false,         false,   false,   true,  false, false, false, v(), Some(Edition::Edition2018));
+    t("",                      false,         false,   Ignore::None,   true,  false, false, false, v(), None);
+    t("rust",                  false,         false,   Ignore::None,   true,  false, false, false, v(), None);
+    t("sh",                    false,         false,   Ignore::None,   false, false, false, false, v(), None);
+    t("ignore",                false,         false,   Ignore::All,    true,  false, false, false, v(), None);
+    t("ignore-foo",            false,         false,   ignore_foo,     true,  false, false, false, v(), None);
+    t("should_panic",          true,          false,   Ignore::None,   true,  false, false, false, v(), None);
+    t("no_run",                false,         true,    Ignore::None,   true,  false, false, false, v(), None);
+    t("test_harness",          false,         false,   Ignore::None,   true,  true,  false, false, v(), None);
+    t("compile_fail",          false,         true,    Ignore::None,   true,  false, true,  false, v(), None);
+    t("allow_fail",            false,         false,   Ignore::None,   true,  false, false, true,  v(), None);
+    t("{.no_run .example}",    false,         true,    Ignore::None,   true,  false, false, false, v(), None);
+    t("{.sh .should_panic}",   true,          false,   Ignore::None,   false, false, false, false, v(), None);
+    t("{.example .rust}",      false,         false,   Ignore::None,   true,  false, false, false, v(), None);
+    t("{.test_harness .rust}", false,         false,   Ignore::None,   true,  true,  false, false, v(), None);
+    t("text, no_run",          false,         true,    Ignore::None,   false, false, false, false, v(), None);
+    t("text,no_run",           false,         true,    Ignore::None,   false, false, false, false, v(), None);
+    t("edition2015",           false,         false,   Ignore::None,   true,  false, false, false, v(), Some(Edition::Edition2015));
+    t("edition2018",           false,         false,   Ignore::None,   true,  false, false, false, v(), Some(Edition::Edition2018));
 }
 
 #[test]

Diff for: src/librustdoc/lib.rs

@@ -356,6 +356,23 @@ fn opts() -> Vec<RustcOptGroup> {
                       "show-coverage",
                       "calculate percentage of public items with documentation")
         }),
+        unstable("enable-per-target-ignores", |o| {
+            o.optflag("",
+                      "enable-per-target-ignores",
+                      "parse ignore-foo for ignoring doctests on a per-target basis")
+        }),
+        unstable("runtool", |o| {
+            o.optopt("",
+                     "runtool",
+                     "",
+                     "The tool to run tests with when building for a different target than host")
+        }),
+        unstable("runtool-arg", |o| {
+            o.optmulti("",
+                       "runtool-arg",
+                       "",
+                       "One (of possibly many) arguments to pass to the runtool")
+        }),
     ]
 }
 

Diff for: src/librustdoc/markdown.rs

@@ -143,11 +143,12 @@ pub fn test(mut options: Options, diag: &errors::Handler) -> i32 {
     opts.no_crate_inject = true;
     opts.display_warnings = options.display_warnings;
     let mut collector = Collector::new(options.input.display().to_string(), options.clone(),
-                                       true, opts, None, Some(options.input));
+                                       true, opts, None, Some(options.input),
+                                       options.enable_per_target_ignores);
     collector.set_position(DUMMY_SP);
     let codes = ErrorCodes::from(UnstableFeatures::from_environment().is_nightly_build());
 
-    find_testable_code(&input_str, &mut collector, codes);
+    find_testable_code(&input_str, &mut collector, codes, options.enable_per_target_ignores);
 
     options.test_args.insert(0, "rustdoctest".to_string());
     testing::test_main(&options.test_args, collector.tests,