🧚‍♀️Normalize doc attributes into ///

Stable `rustfmt` does not support normalizing doc attributes: rust-lang/rustfmt#3351

So, we "normalize" them ourselves by rendering `///`.

Author: Ivan Dubrov

cli/Cargo.toml

@@ -26,3 +26,12 @@ dunce = "1.0.0"
 quote = "1.0.0"
 copy_dir = "0.1.2"
 pretty_assertions = "0.6.1"
+
+[features]
+# Disable normalizing doc comments (`#[doc = r" hello"]`) into `///`.
+# On nightly, one can make `rustfmt` to do that via `normalize_doc_attributes` configuration parameter for `rustfmt`,
+# but on current stable this is not supported. So we support doing our own normalization by default. This feature
+# is to disable that normalization
+disable_normalize_doc_attributes = []
+
+default = []

cli/src/generate.rs

@@ -406,6 +406,13 @@ impl std::fmt::Display for Replacement<'_> {
             f.write_char('\r')?;
         }
         f.write_char('\n')?;
-        write!(f, "{}", self.tokens)
+
+        #[cfg(feature = "disable_normalize_doc_attributes")]
+        write!(f, "{}", self.tokens)?;
+
+        #[cfg(not(feature = "disable_normalize_doc_attributes"))]
+        crate::normalize::write_tokens_normalized(f, self.tokens.clone())?;
+
+        Ok(())
     }
 }

cli/src/lib.rs

@@ -13,6 +13,8 @@ use std::path::Path;
 mod error;
 mod generate;
 mod mods;
+#[cfg(not(feature = "disable_normalize_doc_attributes"))]
+mod normalize;
 mod region;
 mod rustfmt;
 

cli/src/normalize.rs

@@ -0,0 +1,82 @@
+use proc_macro2::{Delimiter, Spacing, TokenStream, TokenTree};
+use syn::Lit;
+
+/// Write tokens same way as `TokenStream::to_string` would do, but with normalization of doc
+/// attributes into `///`.
+pub fn write_tokens_normalized(
+    f: &mut std::fmt::Formatter,
+    tokens: TokenStream,
+) -> std::fmt::Result {
+    let mut tokens = tokens.into_iter().peekable();
+    let mut joint = false;
+    let mut first = true;
+    while let Some(tt) = tokens.next() {
+        if !first && !joint {
+            write!(f, " ")?;
+        }
+        first = false;
+        joint = false;
+
+        if let Some(comment) = tokens
+            .peek()
+            .and_then(|lookahead| as_doc_comment(&tt, lookahead))
+        {
+            let _ignore = tokens.next();
+            writeln!(f, "///{}", comment)?;
+            continue;
+        }
+        match tt {
+            TokenTree::Group(ref tt) => {
+                let (start, end) = match tt.delimiter() {
+                    Delimiter::Parenthesis => ("(", ")"),
+                    Delimiter::Brace => ("{", "}"),
+                    Delimiter::Bracket => ("[", "]"),
+                    Delimiter::None => ("", ""),
+                };
+                if tt.stream().into_iter().next().is_none() {
+                    write!(f, "{} {}", start, end)?
+                } else {
+                    write!(f, "{} ", start)?;
+                    write_tokens_normalized(f, tt.stream())?;
+                    write!(f, " {}", end)?
+                }
+            }
+            TokenTree::Ident(ref tt) => write!(f, "{}", tt)?,
+            TokenTree::Punct(ref tt) => {
+                write!(f, "{}", tt.as_char())?;
+                match tt.spacing() {
+                    Spacing::Alone => {}
+                    Spacing::Joint => joint = true,
+                }
+            }
+            TokenTree::Literal(ref tt) => write!(f, "{}", tt)?,
+        }
+    }
+    Ok(())
+}
+
+fn as_doc_comment(first: &TokenTree, second: &TokenTree) -> Option<String> {
+    match (first, second) {
+        (TokenTree::Punct(first), TokenTree::Group(group))
+            if first.as_char() == '#' && group.delimiter() == Delimiter::Bracket =>
+        {
+            let mut it = group.stream().into_iter();
+            match (it.next(), it.next(), it.next()) {
+                (
+                    Some(TokenTree::Ident(ident)),
+                    Some(TokenTree::Punct(punct)),
+                    Some(TokenTree::Literal(lit)),
+                ) => {
+                    if ident == "doc" && punct.as_char() == '=' {
+                        if let Lit::Str(lit) = Lit::new(lit) {
+                            return Some(lit.value());
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+        _ => {}
+    }
+    None
+}

cli/tests/test_data/004-doc-comments/expected/src/lib.rs

@@ -1,6 +1,6 @@
 #[sourcegen::sourcegen(generator = "generate-doc-comments")]
 // Generated. All manual edits to the block annotated with #[sourcegen...] will be discarded.
-#[doc = r" Some generated comment here"]
+/// Some generated comment here
 struct Hello {
     pub hello: String,
 }

cli/tests/test_data/005-complete-file/expected/src/lib.rs

@@ -1,6 +1,6 @@
 #![sourcegen::sourcegen(generator = "generate-file")]
 // Generated. All manual edits below this line will be discarded.
-#[doc = r" Some generated comment here"]
+/// Some generated comment here
 struct Hello {
     pub hello: String,
 }

cli/tests/test_data/006-complete-file-workaround/expected/src/lib.rs

@@ -2,7 +2,7 @@
 struct __Unused;
 
 // Generated. All manual edits below this line will be discarded.
-#[doc = r" Some generated comment here"]
+/// Some generated comment here
 struct Hello {
     pub hello: String,
 }