"Fossies" - the Fresh Open Source Software Archive 
Member "rustc-1.72.1-src/src/librustdoc/doctest.rs" (13 Sep 2023, 49920 Bytes) of package /linux/misc/rustc-1.72.1-src.tar.xz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Rust source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
See also the last
Fossies "Diffs" side-by-side code changes report for "doctest.rs":
1.71.1_vs_1.72.0.
1 use rustc_ast as ast;
2 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
3 use rustc_data_structures::sync::Lrc;
4 use rustc_errors::{ColorConfig, ErrorGuaranteed, FatalError, TerminalUrl};
5 use rustc_hir::def_id::{LocalDefId, CRATE_DEF_ID, LOCAL_CRATE};
6 use rustc_hir::{self as hir, intravisit, CRATE_HIR_ID};
7 use rustc_interface::interface;
8 use rustc_middle::hir::map::Map;
9 use rustc_middle::hir::nested_filter;
10 use rustc_middle::ty::TyCtxt;
11 use rustc_parse::maybe_new_parser_from_source_str;
12 use rustc_parse::parser::attr::InnerAttrPolicy;
13 use rustc_session::config::{self, CrateType, ErrorOutputType};
14 use rustc_session::parse::ParseSess;
15 use rustc_session::{lint, EarlyErrorHandler, Session};
16 use rustc_span::edition::Edition;
17 use rustc_span::source_map::SourceMap;
18 use rustc_span::symbol::sym;
19 use rustc_span::{BytePos, FileName, Pos, Span, DUMMY_SP};
20 use rustc_target::spec::{Target, TargetTriple};
21 use tempfile::Builder as TempFileBuilder;
22
23 use std::env;
24 use std::io::{self, Write};
25 use std::panic;
26 use std::path::PathBuf;
27 use std::process::{self, Command, Stdio};
28 use std::str;
29 use std::sync::atomic::{AtomicUsize, Ordering};
30 use std::sync::{Arc, Mutex};
31
32 use crate::clean::{types::AttributesExt, Attributes};
33 use crate::config::Options as RustdocOptions;
34 use crate::html::markdown::{self, ErrorCodes, Ignore, LangString};
35 use crate::lint::init_lints;
36 use crate::passes::span_of_attrs;
37
38 /// Options that apply to all doctests in a crate or Markdown file (for `rustdoc foo.md`).
39 #[derive(Clone, Default)]
40 pub(crate) struct GlobalTestOptions {
41 /// Whether to disable the default `extern crate my_crate;` when creating doctests.
42 pub(crate) no_crate_inject: bool,
43 /// Additional crate-level attributes to add to doctests.
44 pub(crate) attrs: Vec<String>,
45 }
46
47 pub(crate) fn run(options: RustdocOptions) -> Result<(), ErrorGuaranteed> {
48 let input = config::Input::File(options.input.clone());
49
50 let invalid_codeblock_attributes_name = crate::lint::INVALID_CODEBLOCK_ATTRIBUTES.name;
51
52 // See core::create_config for what's going on here.
53 let allowed_lints = vec![
54 invalid_codeblock_attributes_name.to_owned(),
55 lint::builtin::UNKNOWN_LINTS.name.to_owned(),
56 lint::builtin::RENAMED_AND_REMOVED_LINTS.name.to_owned(),
57 ];
58
59 let (lint_opts, lint_caps) = init_lints(allowed_lints, options.lint_opts.clone(), |lint| {
60 if lint.name == invalid_codeblock_attributes_name {
61 None
62 } else {
63 Some((lint.name_lower(), lint::Allow))
64 }
65 });
66
67 debug!(?lint_opts);
68
69 let crate_types =
70 if options.proc_macro_crate { vec![CrateType::ProcMacro] } else { vec![CrateType::Rlib] };
71
72 let sessopts = config::Options {
73 maybe_sysroot: options.maybe_sysroot.clone(),
74 search_paths: options.libs.clone(),
75 crate_types,
76 lint_opts,
77 lint_cap: Some(options.lint_cap.unwrap_or(lint::Forbid)),
78 cg: options.codegen_options.clone(),
79 externs: options.externs.clone(),
80 unstable_features: options.unstable_features,
81 actually_rustdoc: true,
82 edition: options.edition,
83 target_triple: options.target.clone(),
84 crate_name: options.crate_name.clone(),
85 ..config::Options::default()
86 };
87
88 let early_error_handler = EarlyErrorHandler::new(ErrorOutputType::default());
89
90 let mut cfgs = options.cfgs.clone();
91 cfgs.push("doc".to_owned());
92 cfgs.push("doctest".to_owned());
93 let config = interface::Config {
94 opts: sessopts,
95 crate_cfg: interface::parse_cfgspecs(&early_error_handler, cfgs),
96 crate_check_cfg: interface::parse_check_cfg(
97 &early_error_handler,
98 options.check_cfgs.clone(),
99 ),
100 input,
101 output_file: None,
102 output_dir: None,
103 file_loader: None,
104 locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES,
105 lint_caps,
106 parse_sess_created: None,
107 register_lints: Some(Box::new(crate::lint::register_lints)),
108 override_queries: None,
109 make_codegen_backend: None,
110 registry: rustc_driver::diagnostics_registry(),
111 };
112
113 let test_args = options.test_args.clone();
114 let nocapture = options.nocapture;
115 let externs = options.externs.clone();
116 let json_unused_externs = options.json_unused_externs;
117
118 let (tests, unused_extern_reports, compiling_test_count) =
119 interface::run_compiler(config, |compiler| {
120 compiler.enter(|queries| {
121 let collector = queries.global_ctxt()?.enter(|tcx| {
122 let crate_attrs = tcx.hir().attrs(CRATE_HIR_ID);
123
124 let opts = scrape_test_config(crate_attrs);
125 let enable_per_target_ignores = options.enable_per_target_ignores;
126 let mut collector = Collector::new(
127 tcx.crate_name(LOCAL_CRATE).to_string(),
128 options,
129 false,
130 opts,
131 Some(compiler.session().parse_sess.clone_source_map()),
132 None,
133 enable_per_target_ignores,
134 );
135
136 let mut hir_collector = HirCollector {
137 sess: compiler.session(),
138 collector: &mut collector,
139 map: tcx.hir(),
140 codes: ErrorCodes::from(
141 compiler.session().opts.unstable_features.is_nightly_build(),
142 ),
143 tcx,
144 };
145 hir_collector.visit_testable(
146 "".to_string(),
147 CRATE_DEF_ID,
148 tcx.hir().span(CRATE_HIR_ID),
149 |this| tcx.hir().walk_toplevel_module(this),
150 );
151
152 collector
153 });
154 if compiler.session().diagnostic().has_errors_or_lint_errors().is_some() {
155 FatalError.raise();
156 }
157
158 let unused_extern_reports = collector.unused_extern_reports.clone();
159 let compiling_test_count = collector.compiling_test_count.load(Ordering::SeqCst);
160 Ok((collector.tests, unused_extern_reports, compiling_test_count))
161 })
162 })?;
163
164 run_tests(test_args, nocapture, tests);
165
166 // Collect and warn about unused externs, but only if we've gotten
167 // reports for each doctest
168 if json_unused_externs.is_enabled() {
169 let unused_extern_reports: Vec<_> =
170 std::mem::take(&mut unused_extern_reports.lock().unwrap());
171 if unused_extern_reports.len() == compiling_test_count {
172 let extern_names = externs.iter().map(|(name, _)| name).collect::<FxHashSet<&String>>();
173 let mut unused_extern_names = unused_extern_reports
174 .iter()
175 .map(|uexts| uexts.unused_extern_names.iter().collect::<FxHashSet<&String>>())
176 .fold(extern_names, |uextsa, uextsb| {
177 uextsa.intersection(&uextsb).copied().collect::<FxHashSet<&String>>()
178 })
179 .iter()
180 .map(|v| (*v).clone())
181 .collect::<Vec<String>>();
182 unused_extern_names.sort();
183 // Take the most severe lint level
184 let lint_level = unused_extern_reports
185 .iter()
186 .map(|uexts| uexts.lint_level.as_str())
187 .max_by_key(|v| match *v {
188 "warn" => 1,
189 "deny" => 2,
190 "forbid" => 3,
191 // The allow lint level is not expected,
192 // as if allow is specified, no message
193 // is to be emitted.
194 v => unreachable!("Invalid lint level '{}'", v),
195 })
196 .unwrap_or("warn")
197 .to_string();
198 let uext = UnusedExterns { lint_level, unused_extern_names };
199 let unused_extern_json = serde_json::to_string(&uext).unwrap();
200 eprintln!("{unused_extern_json}");
201 }
202 }
203
204 Ok(())
205 }
206
207 pub(crate) fn run_tests(
208 mut test_args: Vec<String>,
209 nocapture: bool,
210 mut tests: Vec<test::TestDescAndFn>,
211 ) {
212 test_args.insert(0, "rustdoctest".to_string());
213 if nocapture {
214 test_args.push("--nocapture".to_string());
215 }
216 tests.sort_by(|a, b| a.desc.name.as_slice().cmp(&b.desc.name.as_slice()));
217 test::test_main(&test_args, tests, None);
218 }
219
220 // Look for `#![doc(test(no_crate_inject))]`, used by crates in the std facade.
221 fn scrape_test_config(attrs: &[ast::Attribute]) -> GlobalTestOptions {
222 use rustc_ast_pretty::pprust;
223
224 let mut opts = GlobalTestOptions { no_crate_inject: false, attrs: Vec::new() };
225
226 let test_attrs: Vec<_> = attrs
227 .iter()
228 .filter(|a| a.has_name(sym::doc))
229 .flat_map(|a| a.meta_item_list().unwrap_or_default())
230 .filter(|a| a.has_name(sym::test))
231 .collect();
232 let attrs = test_attrs.iter().flat_map(|a| a.meta_item_list().unwrap_or(&[]));
233
234 for attr in attrs {
235 if attr.has_name(sym::no_crate_inject) {
236 opts.no_crate_inject = true;
237 }
238 if attr.has_name(sym::attr)
239 && let Some(l) = attr.meta_item_list()
240 {
241 for item in l {
242 opts.attrs.push(pprust::meta_list_item_to_string(item));
243 }
244 }
245 }
246
247 opts
248 }
249
250 /// Documentation test failure modes.
251 enum TestFailure {
252 /// The test failed to compile.
253 CompileError,
254 /// The test is marked `compile_fail` but compiled successfully.
255 UnexpectedCompilePass,
256 /// The test failed to compile (as expected) but the compiler output did not contain all
257 /// expected error codes.
258 MissingErrorCodes(Vec<String>),
259 /// The test binary was unable to be executed.
260 ExecutionError(io::Error),
261 /// The test binary exited with a non-zero exit code.
262 ///
263 /// This typically means an assertion in the test failed or another form of panic occurred.
264 ExecutionFailure(process::Output),
265 /// The test is marked `should_panic` but the test binary executed successfully.
266 UnexpectedRunPass,
267 }
268
269 enum DirState {
270 Temp(tempfile::TempDir),
271 Perm(PathBuf),
272 }
273
274 impl DirState {
275 fn path(&self) -> &std::path::Path {
276 match self {
277 DirState::Temp(t) => t.path(),
278 DirState::Perm(p) => p.as_path(),
279 }
280 }
281 }
282
283 // NOTE: Keep this in sync with the equivalent structs in rustc
284 // and cargo.
285 // We could unify this struct the one in rustc but they have different
286 // ownership semantics, so doing so would create wasteful allocations.
287 #[derive(serde::Serialize, serde::Deserialize)]
288 struct UnusedExterns {
289 /// Lint level of the unused_crate_dependencies lint
290 lint_level: String,
291 /// List of unused externs by their names.
292 unused_extern_names: Vec<String>,
293 }
294
295 fn add_exe_suffix(input: String, target: &TargetTriple) -> String {
296 let exe_suffix = match target {
297 TargetTriple::TargetTriple(_) => Target::expect_builtin(target).options.exe_suffix,
298 TargetTriple::TargetJson { contents, .. } => {
299 Target::from_json(contents.parse().unwrap()).unwrap().0.options.exe_suffix
300 }
301 };
302 input + &exe_suffix
303 }
304
305 fn run_test(
306 test: &str,
307 crate_name: &str,
308 line: usize,
309 rustdoc_options: RustdocOptions,
310 mut lang_string: LangString,
311 no_run: bool,
312 runtool: Option<String>,
313 runtool_args: Vec<String>,
314 target: TargetTriple,
315 opts: &GlobalTestOptions,
316 edition: Edition,
317 outdir: DirState,
318 path: PathBuf,
319 test_id: &str,
320 report_unused_externs: impl Fn(UnusedExterns),
321 ) -> Result<(), TestFailure> {
322 let (test, line_offset, supports_color) =
323 make_test(test, Some(crate_name), lang_string.test_harness, opts, edition, Some(test_id));
324
325 // Make sure we emit well-formed executable names for our target.
326 let rust_out = add_exe_suffix("rust_out".to_owned(), &target);
327 let output_file = outdir.path().join(rust_out);
328
329 let rustc_binary = rustdoc_options
330 .test_builder
331 .as_deref()
332 .unwrap_or_else(|| rustc_interface::util::rustc_path().expect("found rustc"));
333 let mut compiler = Command::new(&rustc_binary);
334 compiler.arg("--crate-type").arg("bin");
335 for cfg in &rustdoc_options.cfgs {
336 compiler.arg("--cfg").arg(&cfg);
337 }
338 if !rustdoc_options.check_cfgs.is_empty() {
339 compiler.arg("-Z").arg("unstable-options");
340 for check_cfg in &rustdoc_options.check_cfgs {
341 compiler.arg("--check-cfg").arg(&check_cfg);
342 }
343 }
344 if let Some(sysroot) = rustdoc_options.maybe_sysroot {
345 compiler.arg("--sysroot").arg(sysroot);
346 }
347 compiler.arg("--edition").arg(&edition.to_string());
348 compiler.env("UNSTABLE_RUSTDOC_TEST_PATH", path);
349 compiler.env("UNSTABLE_RUSTDOC_TEST_LINE", format!("{}", line as isize - line_offset as isize));
350 compiler.arg("-o").arg(&output_file);
351 if lang_string.test_harness {
352 compiler.arg("--test");
353 }
354 if rustdoc_options.json_unused_externs.is_enabled() && !lang_string.compile_fail {
355 compiler.arg("--error-format=json");
356 compiler.arg("--json").arg("unused-externs");
357 compiler.arg("-Z").arg("unstable-options");
358 compiler.arg("-W").arg("unused_crate_dependencies");
359 }
360 for lib_str in &rustdoc_options.lib_strs {
361 compiler.arg("-L").arg(&lib_str);
362 }
363 for extern_str in &rustdoc_options.extern_strs {
364 compiler.arg("--extern").arg(&extern_str);
365 }
366 compiler.arg("-Ccodegen-units=1");
367 for codegen_options_str in &rustdoc_options.codegen_options_strs {
368 compiler.arg("-C").arg(&codegen_options_str);
369 }
370 for unstable_option_str in &rustdoc_options.unstable_opts_strs {
371 compiler.arg("-Z").arg(&unstable_option_str);
372 }
373 if no_run && !lang_string.compile_fail && rustdoc_options.persist_doctests.is_none() {
374 compiler.arg("--emit=metadata");
375 }
376 compiler.arg("--target").arg(match target {
377 TargetTriple::TargetTriple(s) => s,
378 TargetTriple::TargetJson { path_for_rustdoc, .. } => {
379 path_for_rustdoc.to_str().expect("target path must be valid unicode").to_string()
380 }
381 });
382 if let ErrorOutputType::HumanReadable(kind) = rustdoc_options.error_format {
383 let (short, color_config) = kind.unzip();
384
385 if short {
386 compiler.arg("--error-format").arg("short");
387 }
388
389 match color_config {
390 ColorConfig::Never => {
391 compiler.arg("--color").arg("never");
392 }
393 ColorConfig::Always => {
394 compiler.arg("--color").arg("always");
395 }
396 ColorConfig::Auto => {
397 compiler.arg("--color").arg(if supports_color { "always" } else { "never" });
398 }
399 }
400 }
401
402 compiler.arg("-");
403 compiler.stdin(Stdio::piped());
404 compiler.stderr(Stdio::piped());
405
406 debug!("compiler invocation for doctest: {:?}", compiler);
407
408 let mut child = compiler.spawn().expect("Failed to spawn rustc process");
409 {
410 let stdin = child.stdin.as_mut().expect("Failed to open stdin");
411 stdin.write_all(test.as_bytes()).expect("could write out test sources");
412 }
413 let output = child.wait_with_output().expect("Failed to read stdout");
414
415 struct Bomb<'a>(&'a str);
416 impl Drop for Bomb<'_> {
417 fn drop(&mut self) {
418 eprint!("{}", self.0);
419 }
420 }
421 let mut out = str::from_utf8(&output.stderr)
422 .unwrap()
423 .lines()
424 .filter(|l| {
425 if let Ok(uext) = serde_json::from_str::<UnusedExterns>(l) {
426 report_unused_externs(uext);
427 false
428 } else {
429 true
430 }
431 })
432 .intersperse_with(|| "\n")
433 .collect::<String>();
434
435 // Add a \n to the end to properly terminate the last line,
436 // but only if there was output to be printed
437 if !out.is_empty() {
438 out.push('\n');
439 }
440
441 let _bomb = Bomb(&out);
442 match (output.status.success(), lang_string.compile_fail) {
443 (true, true) => {
444 return Err(TestFailure::UnexpectedCompilePass);
445 }
446 (true, false) => {}
447 (false, true) => {
448 if !lang_string.error_codes.is_empty() {
449 // We used to check if the output contained "error[{}]: " but since we added the
450 // colored output, we can't anymore because of the color escape characters before
451 // the ":".
452 lang_string.error_codes.retain(|err| !out.contains(&format!("error[{err}]")));
453
454 if !lang_string.error_codes.is_empty() {
455 return Err(TestFailure::MissingErrorCodes(lang_string.error_codes));
456 }
457 }
458 }
459 (false, false) => {
460 return Err(TestFailure::CompileError);
461 }
462 }
463
464 if no_run {
465 return Ok(());
466 }
467
468 // Run the code!
469 let mut cmd;
470
471 if let Some(tool) = runtool {
472 cmd = Command::new(tool);
473 cmd.args(runtool_args);
474 cmd.arg(output_file);
475 } else {
476 cmd = Command::new(output_file);
477 }
478 if let Some(run_directory) = rustdoc_options.test_run_directory {
479 cmd.current_dir(run_directory);
480 }
481
482 let result = if rustdoc_options.nocapture {
483 cmd.status().map(|status| process::Output {
484 status,
485 stdout: Vec::new(),
486 stderr: Vec::new(),
487 })
488 } else {
489 cmd.output()
490 };
491 match result {
492 Err(e) => return Err(TestFailure::ExecutionError(e)),
493 Ok(out) => {
494 if lang_string.should_panic && out.status.success() {
495 return Err(TestFailure::UnexpectedRunPass);
496 } else if !lang_string.should_panic && !out.status.success() {
497 return Err(TestFailure::ExecutionFailure(out));
498 }
499 }
500 }
501
502 Ok(())
503 }
504
505 /// Transforms a test into code that can be compiled into a Rust binary, and returns the number of
506 /// lines before the test code begins as well as if the output stream supports colors or not.
507 pub(crate) fn make_test(
508 s: &str,
509 crate_name: Option<&str>,
510 dont_insert_main: bool,
511 opts: &GlobalTestOptions,
512 edition: Edition,
513 test_id: Option<&str>,
514 ) -> (String, usize, bool) {
515 let (crate_attrs, everything_else, crates) = partition_source(s, edition);
516 let everything_else = everything_else.trim();
517 let mut line_offset = 0;
518 let mut prog = String::new();
519 let mut supports_color = false;
520
521 if opts.attrs.is_empty() {
522 // If there aren't any attributes supplied by #![doc(test(attr(...)))], then allow some
523 // lints that are commonly triggered in doctests. The crate-level test attributes are
524 // commonly used to make tests fail in case they trigger warnings, so having this there in
525 // that case may cause some tests to pass when they shouldn't have.
526 prog.push_str("#![allow(unused)]\n");
527 line_offset += 1;
528 }
529
530 // Next, any attributes that came from the crate root via #![doc(test(attr(...)))].
531 for attr in &opts.attrs {
532 prog.push_str(&format!("#![{attr}]\n"));
533 line_offset += 1;
534 }
535
536 // Now push any outer attributes from the example, assuming they
537 // are intended to be crate attributes.
538 prog.push_str(&crate_attrs);
539 prog.push_str(&crates);
540
541 // Uses librustc_ast to parse the doctest and find if there's a main fn and the extern
542 // crate already is included.
543 let result = rustc_driver::catch_fatal_errors(|| {
544 rustc_span::create_session_if_not_set_then(edition, |_| {
545 use rustc_errors::emitter::{Emitter, EmitterWriter};
546 use rustc_errors::Handler;
547 use rustc_parse::parser::ForceCollect;
548 use rustc_span::source_map::FilePathMapping;
549
550 let filename = FileName::anon_source_code(s);
551 let source = crates + everything_else;
552
553 // Any errors in parsing should also appear when the doctest is compiled for real, so just
554 // send all the errors that librustc_ast emits directly into a `Sink` instead of stderr.
555 let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
556 let fallback_bundle = rustc_errors::fallback_fluent_bundle(
557 rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
558 false,
559 );
560 supports_color = EmitterWriter::stderr(
561 ColorConfig::Auto,
562 None,
563 None,
564 fallback_bundle.clone(),
565 false,
566 false,
567 Some(80),
568 false,
569 false,
570 TerminalUrl::No,
571 )
572 .supports_color();
573
574 let emitter = EmitterWriter::new(
575 Box::new(io::sink()),
576 None,
577 None,
578 fallback_bundle,
579 false,
580 false,
581 false,
582 None,
583 false,
584 false,
585 TerminalUrl::No,
586 );
587
588 // FIXME(misdreavus): pass `-Z treat-err-as-bug` to the doctest parser
589 let handler = Handler::with_emitter(false, None, Box::new(emitter));
590 let sess = ParseSess::with_span_handler(handler, sm);
591
592 let mut found_main = false;
593 let mut found_extern_crate = crate_name.is_none();
594 let mut found_macro = false;
595
596 let mut parser = match maybe_new_parser_from_source_str(&sess, filename, source) {
597 Ok(p) => p,
598 Err(errs) => {
599 drop(errs);
600 return (found_main, found_extern_crate, found_macro);
601 }
602 };
603
604 loop {
605 match parser.parse_item(ForceCollect::No) {
606 Ok(Some(item)) => {
607 if !found_main &&
608 let ast::ItemKind::Fn(..) = item.kind &&
609 item.ident.name == sym::main
610 {
611 found_main = true;
612 }
613
614 if !found_extern_crate &&
615 let ast::ItemKind::ExternCrate(original) = item.kind
616 {
617 // This code will never be reached if `crate_name` is none because
618 // `found_extern_crate` is initialized to `true` if it is none.
619 let crate_name = crate_name.unwrap();
620
621 match original {
622 Some(name) => found_extern_crate = name.as_str() == crate_name,
623 None => found_extern_crate = item.ident.as_str() == crate_name,
624 }
625 }
626
627 if !found_macro && let ast::ItemKind::MacCall(..) = item.kind {
628 found_macro = true;
629 }
630
631 if found_main && found_extern_crate {
632 break;
633 }
634 }
635 Ok(None) => break,
636 Err(e) => {
637 e.cancel();
638 break;
639 }
640 }
641
642 // The supplied slice is only used for diagnostics,
643 // which are swallowed here anyway.
644 parser.maybe_consume_incorrect_semicolon(&[]);
645 }
646
647 // Reset errors so that they won't be reported as compiler bugs when dropping the
648 // handler. Any errors in the tests will be reported when the test file is compiled,
649 // Note that we still need to cancel the errors above otherwise `DiagnosticBuilder`
650 // will panic on drop.
651 sess.span_diagnostic.reset_err_count();
652
653 (found_main, found_extern_crate, found_macro)
654 })
655 });
656 let Ok((already_has_main, already_has_extern_crate, found_macro)) = result
657 else {
658 // If the parser panicked due to a fatal error, pass the test code through unchanged.
659 // The error will be reported during compilation.
660 return (s.to_owned(), 0, false);
661 };
662
663 // If a doctest's `fn main` is being masked by a wrapper macro, the parsing loop above won't
664 // see it. In that case, run the old text-based scan to see if they at least have a main
665 // function written inside a macro invocation. See
666 // https://github.com/rust-lang/rust/issues/56898
667 let already_has_main = if found_macro && !already_has_main {
668 s.lines()
669 .map(|line| {
670 let comment = line.find("//");
671 if let Some(comment_begins) = comment { &line[0..comment_begins] } else { line }
672 })
673 .any(|code| code.contains("fn main"))
674 } else {
675 already_has_main
676 };
677
678 // Don't inject `extern crate std` because it's already injected by the
679 // compiler.
680 if !already_has_extern_crate && !opts.no_crate_inject && crate_name != Some("std") {
681 if let Some(crate_name) = crate_name {
682 // Don't inject `extern crate` if the crate is never used.
683 // NOTE: this is terribly inaccurate because it doesn't actually
684 // parse the source, but only has false positives, not false
685 // negatives.
686 if s.contains(crate_name) {
687 // rustdoc implicitly inserts an `extern crate` item for the own crate
688 // which may be unused, so we need to allow the lint.
689 prog.push_str(&format!("#[allow(unused_extern_crates)]\n"));
690
691 prog.push_str(&format!("extern crate r#{crate_name};\n"));
692 line_offset += 1;
693 }
694 }
695 }
696
697 // FIXME: This code cannot yet handle no_std test cases yet
698 if dont_insert_main || already_has_main || prog.contains("![no_std]") {
699 prog.push_str(everything_else);
700 } else {
701 let returns_result = everything_else.trim_end().ends_with("(())");
702 // Give each doctest main function a unique name.
703 // This is for example needed for the tooling around `-C instrument-coverage`.
704 let inner_fn_name = if let Some(test_id) = test_id {
705 format!("_doctest_main_{test_id}")
706 } else {
707 "_inner".into()
708 };
709 let inner_attr = if test_id.is_some() { "#[allow(non_snake_case)] " } else { "" };
710 let (main_pre, main_post) = if returns_result {
711 (
712 format!(
713 "fn main() {{ {inner_attr}fn {inner_fn_name}() -> Result<(), impl core::fmt::Debug> {{\n",
714 ),
715 format!("\n}} {inner_fn_name}().unwrap() }}"),
716 )
717 } else if test_id.is_some() {
718 (
719 format!("fn main() {{ {inner_attr}fn {inner_fn_name}() {{\n",),
720 format!("\n}} {inner_fn_name}() }}"),
721 )
722 } else {
723 ("fn main() {\n".into(), "\n}".into())
724 };
725 // Note on newlines: We insert a line/newline *before*, and *after*
726 // the doctest and adjust the `line_offset` accordingly.
727 // In the case of `-C instrument-coverage`, this means that the generated
728 // inner `main` function spans from the doctest opening codeblock to the
729 // closing one. For example
730 // /// ``` <- start of the inner main
731 // /// <- code under doctest
732 // /// ``` <- end of the inner main
733 line_offset += 1;
734
735 prog.extend([&main_pre, everything_else, &main_post].iter().cloned());
736 }
737
738 debug!("final doctest:\n{prog}");
739
740 (prog, line_offset, supports_color)
741 }
742
743 fn check_if_attr_is_complete(source: &str, edition: Edition) -> bool {
744 if source.is_empty() {
745 // Empty content so nothing to check in here...
746 return true;
747 }
748 rustc_driver::catch_fatal_errors(|| {
749 rustc_span::create_session_if_not_set_then(edition, |_| {
750 use rustc_errors::emitter::EmitterWriter;
751 use rustc_errors::Handler;
752 use rustc_span::source_map::FilePathMapping;
753
754 let filename = FileName::anon_source_code(source);
755 // Any errors in parsing should also appear when the doctest is compiled for real, so just
756 // send all the errors that librustc_ast emits directly into a `Sink` instead of stderr.
757 let sm = Lrc::new(SourceMap::new(FilePathMapping::empty()));
758 let fallback_bundle = rustc_errors::fallback_fluent_bundle(
759 rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(),
760 false,
761 );
762
763 let emitter = EmitterWriter::new(
764 Box::new(io::sink()),
765 None,
766 None,
767 fallback_bundle,
768 false,
769 false,
770 false,
771 None,
772 false,
773 false,
774 TerminalUrl::No,
775 );
776
777 let handler = Handler::with_emitter(false, None, Box::new(emitter));
778 let sess = ParseSess::with_span_handler(handler, sm);
779 let mut parser =
780 match maybe_new_parser_from_source_str(&sess, filename, source.to_owned()) {
781 Ok(p) => p,
782 Err(_) => {
783 // If there is an unclosed delimiter, an error will be returned by the tokentrees.
784 return false;
785 }
786 };
787 // If a parsing error happened, it's very likely that the attribute is incomplete.
788 if let Err(e) = parser.parse_attribute(InnerAttrPolicy::Permitted) {
789 e.cancel();
790 return false;
791 }
792 true
793 })
794 })
795 .unwrap_or(false)
796 }
797
798 fn partition_source(s: &str, edition: Edition) -> (String, String, String) {
799 #[derive(Copy, Clone, PartialEq)]
800 enum PartitionState {
801 Attrs,
802 Crates,
803 Other,
804 }
805 let mut state = PartitionState::Attrs;
806 let mut before = String::new();
807 let mut crates = String::new();
808 let mut after = String::new();
809
810 let mut mod_attr_pending = String::new();
811
812 for line in s.lines() {
813 let trimline = line.trim();
814
815 // FIXME(misdreavus): if a doc comment is placed on an extern crate statement, it will be
816 // shunted into "everything else"
817 match state {
818 PartitionState::Attrs => {
819 state = if trimline.starts_with("#![") {
820 if !check_if_attr_is_complete(line, edition) {
821 mod_attr_pending = line.to_owned();
822 } else {
823 mod_attr_pending.clear();
824 }
825 PartitionState::Attrs
826 } else if trimline.chars().all(|c| c.is_whitespace())
827 || (trimline.starts_with("//") && !trimline.starts_with("///"))
828 {
829 PartitionState::Attrs
830 } else if trimline.starts_with("extern crate")
831 || trimline.starts_with("#[macro_use] extern crate")
832 {
833 PartitionState::Crates
834 } else {
835 // First we check if the previous attribute was "complete"...
836 if !mod_attr_pending.is_empty() {
837 // If not, then we append the new line into the pending attribute to check
838 // if this time it's complete...
839 mod_attr_pending.push_str(line);
840 if !trimline.is_empty()
841 && check_if_attr_is_complete(&mod_attr_pending, edition)
842 {
843 // If it's complete, then we can clear the pending content.
844 mod_attr_pending.clear();
845 }
846 // In any case, this is considered as `PartitionState::Attrs` so it's
847 // prepended before rustdoc's inserts.
848 PartitionState::Attrs
849 } else {
850 PartitionState::Other
851 }
852 };
853 }
854 PartitionState::Crates => {
855 state = if trimline.starts_with("extern crate")
856 || trimline.starts_with("#[macro_use] extern crate")
857 || trimline.chars().all(|c| c.is_whitespace())
858 || (trimline.starts_with("//") && !trimline.starts_with("///"))
859 {
860 PartitionState::Crates
861 } else {
862 PartitionState::Other
863 };
864 }
865 PartitionState::Other => {}
866 }
867
868 match state {
869 PartitionState::Attrs => {
870 before.push_str(line);
871 before.push('\n');
872 }
873 PartitionState::Crates => {
874 crates.push_str(line);
875 crates.push('\n');
876 }
877 PartitionState::Other => {
878 after.push_str(line);
879 after.push('\n');
880 }
881 }
882 }
883
884 debug!("before:\n{before}");
885 debug!("crates:\n{crates}");
886 debug!("after:\n{after}");
887
888 (before, after, crates)
889 }
890
891 pub(crate) trait Tester {
892 fn add_test(&mut self, test: String, config: LangString, line: usize);
893 fn get_line(&self) -> usize {
894 0
895 }
896 fn register_header(&mut self, _name: &str, _level: u32) {}
897 }
898
899 pub(crate) struct Collector {
900 pub(crate) tests: Vec<test::TestDescAndFn>,
901
902 // The name of the test displayed to the user, separated by `::`.
903 //
904 // In tests from Rust source, this is the path to the item
905 // e.g., `["std", "vec", "Vec", "push"]`.
906 //
907 // In tests from a markdown file, this is the titles of all headers (h1~h6)
908 // of the sections that contain the code block, e.g., if the markdown file is
909 // written as:
910 //
911 // ``````markdown
912 // # Title
913 //
914 // ## Subtitle
915 //
916 // ```rust
917 // assert!(true);
918 // ```
919 // ``````
920 //
921 // the `names` vector of that test will be `["Title", "Subtitle"]`.
922 names: Vec<String>,
923
924 rustdoc_options: RustdocOptions,
925 use_headers: bool,
926 enable_per_target_ignores: bool,
927 crate_name: String,
928 opts: GlobalTestOptions,
929 position: Span,
930 source_map: Option<Lrc<SourceMap>>,
931 filename: Option<PathBuf>,
932 visited_tests: FxHashMap<(String, usize), usize>,
933 unused_extern_reports: Arc<Mutex<Vec<UnusedExterns>>>,
934 compiling_test_count: AtomicUsize,
935 }
936
937 impl Collector {
938 pub(crate) fn new(
939 crate_name: String,
940 rustdoc_options: RustdocOptions,
941 use_headers: bool,
942 opts: GlobalTestOptions,
943 source_map: Option<Lrc<SourceMap>>,
944 filename: Option<PathBuf>,
945 enable_per_target_ignores: bool,
946 ) -> Collector {
947 Collector {
948 tests: Vec::new(),
949 names: Vec::new(),
950 rustdoc_options,
951 use_headers,
952 enable_per_target_ignores,
953 crate_name,
954 opts,
955 position: DUMMY_SP,
956 source_map,
957 filename,
958 visited_tests: FxHashMap::default(),
959 unused_extern_reports: Default::default(),
960 compiling_test_count: AtomicUsize::new(0),
961 }
962 }
963
964 fn generate_name(&self, line: usize, filename: &FileName) -> String {
965 let mut item_path = self.names.join("::");
966 item_path.retain(|c| c != ' ');
967 if !item_path.is_empty() {
968 item_path.push(' ');
969 }
970 format!("{} - {}(line {})", filename.prefer_local(), item_path, line)
971 }
972
973 pub(crate) fn set_position(&mut self, position: Span) {
974 self.position = position;
975 }
976
977 fn get_filename(&self) -> FileName {
978 if let Some(ref source_map) = self.source_map {
979 let filename = source_map.span_to_filename(self.position);
980 if let FileName::Real(ref filename) = filename &&
981 let Ok(cur_dir) = env::current_dir() &&
982 let Some(local_path) = filename.local_path() &&
983 let Ok(path) = local_path.strip_prefix(&cur_dir)
984 {
985 return path.to_owned().into();
986 }
987 filename
988 } else if let Some(ref filename) = self.filename {
989 filename.clone().into()
990 } else {
991 FileName::Custom("input".to_owned())
992 }
993 }
994 }
995
996 impl Tester for Collector {
997 fn add_test(&mut self, test: String, config: LangString, line: usize) {
998 let filename = self.get_filename();
999 let name = self.generate_name(line, &filename);
1000 let crate_name = self.crate_name.clone();
1001 let opts = self.opts.clone();
1002 let edition = config.edition.unwrap_or(self.rustdoc_options.edition);
1003 let rustdoc_options = self.rustdoc_options.clone();
1004 let runtool = self.rustdoc_options.runtool.clone();
1005 let runtool_args = self.rustdoc_options.runtool_args.clone();
1006 let target = self.rustdoc_options.target.clone();
1007 let target_str = target.to_string();
1008 let unused_externs = self.unused_extern_reports.clone();
1009 let no_run = config.no_run || rustdoc_options.no_run;
1010 if !config.compile_fail {
1011 self.compiling_test_count.fetch_add(1, Ordering::SeqCst);
1012 }
1013
1014 let path = match &filename {
1015 FileName::Real(path) => {
1016 if let Some(local_path) = path.local_path() {
1017 local_path.to_path_buf()
1018 } else {
1019 // Somehow we got the filename from the metadata of another crate, should never happen
1020 unreachable!("doctest from a different crate");
1021 }
1022 }
1023 _ => PathBuf::from(r"doctest.rs"),
1024 };
1025
1026 // For example `module/file.rs` would become `module_file_rs`
1027 let file = filename
1028 .prefer_local()
1029 .to_string_lossy()
1030 .chars()
1031 .map(|c| if c.is_ascii_alphanumeric() { c } else { '_' })
1032 .collect::<String>();
1033 let test_id = format!(
1034 "{file}_{line}_{number}",
1035 file = file,
1036 line = line,
1037 number = {
1038 // Increases the current test number, if this file already
1039 // exists or it creates a new entry with a test number of 0.
1040 self.visited_tests.entry((file.clone(), line)).and_modify(|v| *v += 1).or_insert(0)
1041 },
1042 );
1043 let outdir = if let Some(mut path) = rustdoc_options.persist_doctests.clone() {
1044 path.push(&test_id);
1045
1046 if let Err(err) = std::fs::create_dir_all(&path) {
1047 eprintln!("Couldn't create directory for doctest executables: {}", err);
1048 panic::resume_unwind(Box::new(()));
1049 }
1050
1051 DirState::Perm(path)
1052 } else {
1053 DirState::Temp(
1054 TempFileBuilder::new()
1055 .prefix("rustdoctest")
1056 .tempdir()
1057 .expect("rustdoc needs a tempdir"),
1058 )
1059 };
1060
1061 debug!("creating test {name}: {test}");
1062 self.tests.push(test::TestDescAndFn {
1063 desc: test::TestDesc {
1064 name: test::DynTestName(name),
1065 ignore: match config.ignore {
1066 Ignore::All => true,
1067 Ignore::None => false,
1068 Ignore::Some(ref ignores) => ignores.iter().any(|s| target_str.contains(s)),
1069 },
1070 ignore_message: None,
1071 source_file: "",
1072 start_line: 0,
1073 start_col: 0,
1074 end_line: 0,
1075 end_col: 0,
1076 // compiler failures are test failures
1077 should_panic: test::ShouldPanic::No,
1078 compile_fail: config.compile_fail,
1079 no_run,
1080 test_type: test::TestType::DocTest,
1081 },
1082 testfn: test::DynTestFn(Box::new(move || {
1083 let report_unused_externs = |uext| {
1084 unused_externs.lock().unwrap().push(uext);
1085 };
1086 let res = run_test(
1087 &test,
1088 &crate_name,
1089 line,
1090 rustdoc_options,
1091 config,
1092 no_run,
1093 runtool,
1094 runtool_args,
1095 target,
1096 &opts,
1097 edition,
1098 outdir,
1099 path,
1100 &test_id,
1101 report_unused_externs,
1102 );
1103
1104 if let Err(err) = res {
1105 match err {
1106 TestFailure::CompileError => {
1107 eprint!("Couldn't compile the test.");
1108 }
1109 TestFailure::UnexpectedCompilePass => {
1110 eprint!("Test compiled successfully, but it's marked `compile_fail`.");
1111 }
1112 TestFailure::UnexpectedRunPass => {
1113 eprint!("Test executable succeeded, but it's marked `should_panic`.");
1114 }
1115 TestFailure::MissingErrorCodes(codes) => {
1116 eprint!("Some expected error codes were not found: {:?}", codes);
1117 }
1118 TestFailure::ExecutionError(err) => {
1119 eprint!("Couldn't run the test: {err}");
1120 if err.kind() == io::ErrorKind::PermissionDenied {
1121 eprint!(" - maybe your tempdir is mounted with noexec?");
1122 }
1123 }
1124 TestFailure::ExecutionFailure(out) => {
1125 eprintln!("Test executable failed ({reason}).", reason = out.status);
1126
1127 // FIXME(#12309): An unfortunate side-effect of capturing the test
1128 // executable's output is that the relative ordering between the test's
1129 // stdout and stderr is lost. However, this is better than the
1130 // alternative: if the test executable inherited the parent's I/O
1131 // handles the output wouldn't be captured at all, even on success.
1132 //
1133 // The ordering could be preserved if the test process' stderr was
1134 // redirected to stdout, but that functionality does not exist in the
1135 // standard library, so it may not be portable enough.
1136 let stdout = str::from_utf8(&out.stdout).unwrap_or_default();
1137 let stderr = str::from_utf8(&out.stderr).unwrap_or_default();
1138
1139 if !stdout.is_empty() || !stderr.is_empty() {
1140 eprintln!();
1141
1142 if !stdout.is_empty() {
1143 eprintln!("stdout:\n{stdout}");
1144 }
1145
1146 if !stderr.is_empty() {
1147 eprintln!("stderr:\n{stderr}");
1148 }
1149 }
1150 }
1151 }
1152
1153 panic::resume_unwind(Box::new(()));
1154 }
1155 Ok(())
1156 })),
1157 });
1158 }
1159
1160 fn get_line(&self) -> usize {
1161 if let Some(ref source_map) = self.source_map {
1162 let line = self.position.lo().to_usize();
1163 let line = source_map.lookup_char_pos(BytePos(line as u32)).line;
1164 if line > 0 { line - 1 } else { line }
1165 } else {
1166 0
1167 }
1168 }
1169
1170 fn register_header(&mut self, name: &str, level: u32) {
1171 if self.use_headers {
1172 // We use these headings as test names, so it's good if
1173 // they're valid identifiers.
1174 let name = name
1175 .chars()
1176 .enumerate()
1177 .map(|(i, c)| {
1178 if (i == 0 && rustc_lexer::is_id_start(c))
1179 || (i != 0 && rustc_lexer::is_id_continue(c))
1180 {
1181 c
1182 } else {
1183 '_'
1184 }
1185 })
1186 .collect::<String>();
1187
1188 // Here we try to efficiently assemble the header titles into the
1189 // test name in the form of `h1::h2::h3::h4::h5::h6`.
1190 //
1191 // Suppose that originally `self.names` contains `[h1, h2, h3]`...
1192 let level = level as usize;
1193 if level <= self.names.len() {
1194 // ... Consider `level == 2`. All headers in the lower levels
1195 // are irrelevant in this new level. So we should reset
1196 // `self.names` to contain headers until <h2>, and replace that
1197 // slot with the new name: `[h1, name]`.
1198 self.names.truncate(level);
1199 self.names[level - 1] = name;
1200 } else {
1201 // ... On the other hand, consider `level == 5`. This means we
1202 // need to extend `self.names` to contain five headers. We fill
1203 // in the missing level (<h4>) with `_`. Thus `self.names` will
1204 // become `[h1, h2, h3, "_", name]`.
1205 if level - 1 > self.names.len() {
1206 self.names.resize(level - 1, "_".to_owned());
1207 }
1208 self.names.push(name);
1209 }
1210 }
1211 }
1212 }
1213
1214 struct HirCollector<'a, 'hir, 'tcx> {
1215 sess: &'a Session,
1216 collector: &'a mut Collector,
1217 map: Map<'hir>,
1218 codes: ErrorCodes,
1219 tcx: TyCtxt<'tcx>,
1220 }
1221
1222 impl<'a, 'hir, 'tcx> HirCollector<'a, 'hir, 'tcx> {
1223 fn visit_testable<F: FnOnce(&mut Self)>(
1224 &mut self,
1225 name: String,
1226 def_id: LocalDefId,
1227 sp: Span,
1228 nested: F,
1229 ) {
1230 let ast_attrs = self.tcx.hir().attrs(self.tcx.hir().local_def_id_to_hir_id(def_id));
1231 if let Some(ref cfg) = ast_attrs.cfg(self.tcx, &FxHashSet::default()) {
1232 if !cfg.matches(&self.sess.parse_sess, Some(self.tcx.features())) {
1233 return;
1234 }
1235 }
1236
1237 let has_name = !name.is_empty();
1238 if has_name {
1239 self.collector.names.push(name);
1240 }
1241
1242 // The collapse-docs pass won't combine sugared/raw doc attributes, or included files with
1243 // anything else, this will combine them for us.
1244 let attrs = Attributes::from_ast(ast_attrs);
1245 if let Some(doc) = attrs.opt_doc_value() {
1246 // Use the outermost invocation, so that doctest names come from where the docs were written.
1247 let span = ast_attrs
1248 .iter()
1249 .find(|attr| attr.doc_str().is_some())
1250 .map(|attr| attr.span.ctxt().outer_expn().expansion_cause().unwrap_or(attr.span))
1251 .unwrap_or(DUMMY_SP);
1252 self.collector.set_position(span);
1253 markdown::find_testable_code(
1254 &doc,
1255 self.collector,
1256 self.codes,
1257 self.collector.enable_per_target_ignores,
1258 Some(&crate::html::markdown::ExtraInfo::new(
1259 self.tcx,
1260 def_id.to_def_id(),
1261 span_of_attrs(&attrs).unwrap_or(sp),
1262 )),
1263 );
1264 }
1265
1266 nested(self);
1267
1268 if has_name {
1269 self.collector.names.pop();
1270 }
1271 }
1272 }
1273
1274 impl<'a, 'hir, 'tcx> intravisit::Visitor<'hir> for HirCollector<'a, 'hir, 'tcx> {
1275 type NestedFilter = nested_filter::All;
1276
1277 fn nested_visit_map(&mut self) -> Self::Map {
1278 self.map
1279 }
1280
1281 fn visit_item(&mut self, item: &'hir hir::Item<'_>) {
1282 let name = match &item.kind {
1283 hir::ItemKind::Impl(impl_) => {
1284 rustc_hir_pretty::id_to_string(&self.map, impl_.self_ty.hir_id)
1285 }
1286 _ => item.ident.to_string(),
1287 };
1288
1289 self.visit_testable(name, item.owner_id.def_id, item.span, |this| {
1290 intravisit::walk_item(this, item);
1291 });
1292 }
1293
1294 fn visit_trait_item(&mut self, item: &'hir hir::TraitItem<'_>) {
1295 self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| {
1296 intravisit::walk_trait_item(this, item);
1297 });
1298 }
1299
1300 fn visit_impl_item(&mut self, item: &'hir hir::ImplItem<'_>) {
1301 self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| {
1302 intravisit::walk_impl_item(this, item);
1303 });
1304 }
1305
1306 fn visit_foreign_item(&mut self, item: &'hir hir::ForeignItem<'_>) {
1307 self.visit_testable(item.ident.to_string(), item.owner_id.def_id, item.span, |this| {
1308 intravisit::walk_foreign_item(this, item);
1309 });
1310 }
1311
1312 fn visit_variant(&mut self, v: &'hir hir::Variant<'_>) {
1313 self.visit_testable(v.ident.to_string(), v.def_id, v.span, |this| {
1314 intravisit::walk_variant(this, v);
1315 });
1316 }
1317
1318 fn visit_field_def(&mut self, f: &'hir hir::FieldDef<'_>) {
1319 self.visit_testable(f.ident.to_string(), f.def_id, f.span, |this| {
1320 intravisit::walk_field_def(this, f);
1321 });
1322 }
1323 }
1324
1325 #[cfg(test)]
1326 mod tests;