Auto merge of #102944 - nnethercote:ast-Lit-third-time-lucky, r=petro…

…chenkov

Use `token::Lit` in `ast::ExprKind::Lit`.

Instead of `ast::Lit`.

Literal lowering now happens at two different times. Expression literals are lowered when HIR is crated. Attribute literals are lowered during parsing.

r? `@petrochenkov`

compiler/rustc_ast/src/ast.rs

@@ -1332,7 +1332,7 @@ pub enum ExprKind {
     /// A unary operation (e.g., `!x`, `*x`).
     Unary(UnOp, P<Expr>),
     /// A literal (e.g., `1`, `"foo"`).
-    Lit(Lit),
+    Lit(token::Lit),
     /// A cast (e.g., `foo as f64`).
     Cast(P<Expr>, P<Ty>),
     /// A type ascription (e.g., `42: usize`).
@@ -1698,16 +1698,12 @@ pub struct StrLit {
 }
 
 impl StrLit {
-    pub fn as_lit(&self) -> Lit {
+    pub fn as_token_lit(&self) -> token::Lit {
         let token_kind = match self.style {
             StrStyle::Cooked => token::Str,
             StrStyle::Raw(n) => token::StrRaw(n),
         };
-        Lit {
-            token_lit: token::Lit::new(token_kind, self.symbol, self.suffix),
-            span: self.span,
-            kind: LitKind::Str(self.symbol_unescaped, self.style),
-        }
+        token::Lit::new(token_kind, self.symbol, self.suffix)
     }
 }
 
@@ -1733,9 +1729,10 @@ pub enum LitFloatType {
     Unsuffixed,
 }
 
-/// Literal kind.
-///
-/// E.g., `"foo"`, `42`, `12.34`, or `bool`.
+/// Note that the entire literal (including the suffix) is considered when
+/// deciding the `LitKind`. This means that float literals like `1f32` are
+/// classified by this type as `Float`. This is different to `token::LitKind`
+/// which does *not* consider the suffix.
 #[derive(Clone, Encodable, Decodable, Debug, Hash, Eq, PartialEq, HashStable_Generic)]
 pub enum LitKind {
     /// A string literal (`"foo"`). The symbol is unescaped, and so may differ
@@ -1749,10 +1746,11 @@ pub enum LitKind {
     Char(char),
     /// An integer literal (`1`).
     Int(u128, LitIntType),
-    /// A float literal (`1f64` or `1E10f64`). Stored as a symbol rather than
-    /// `f64` so that `LitKind` can impl `Eq` and `Hash`.
+    /// A float literal (`1.0`, `1f64` or `1E10f64`). The pre-suffix part is
+    /// stored as a symbol rather than `f64` so that `LitKind` can impl `Eq`
+    /// and `Hash`.
     Float(Symbol, LitFloatType),
-    /// A boolean literal.
+    /// A boolean literal (`true`, `false`).
     Bool(bool),
     /// Placeholder for a literal that wasn't well-formed in some way.
     Err,

compiler/rustc_ast/src/attr/mod.rs

@@ -533,7 +533,7 @@ impl MetaItemKind {
             MetaItemKind::NameValue(lit) => {
                 let expr = P(ast::Expr {
                     id: ast::DUMMY_NODE_ID,
-                    kind: ast::ExprKind::Lit(lit.clone()),
+                    kind: ast::ExprKind::Lit(lit.token_lit.clone()),
                     span: lit.span,
                     attrs: ast::AttrVec::new(),
                     tokens: None,
@@ -605,7 +605,7 @@ impl MetaItemKind {
                 MetaItemKind::name_value_from_tokens(&mut inner_tokens.into_trees())
             }
             Some(TokenTree::Token(token, _)) => {
-                Lit::from_token(&token).ok().map(MetaItemKind::NameValue)
+                Lit::from_token(&token).map(MetaItemKind::NameValue)
             }
             _ => None,
         }
@@ -618,8 +618,10 @@ impl MetaItemKind {
                 MetaItemKind::list_from_tokens(tokens.clone())
             }
             MacArgs::Delimited(..) => None,
-            MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match &expr.kind {
-                ast::ExprKind::Lit(lit) => Some(MetaItemKind::NameValue(lit.clone())),
+            MacArgs::Eq(_, MacArgsEq::Ast(expr)) => match expr.kind {
+                ast::ExprKind::Lit(token_lit) => Some(MetaItemKind::NameValue(
+                    Lit::from_token_lit(token_lit, expr.span).expect("token_lit in from_mac_args"),
+                )),
                 _ => None,
             },
             MacArgs::Eq(_, MacArgsEq::Hir(lit)) => Some(MetaItemKind::NameValue(lit.clone())),
@@ -668,7 +670,7 @@ impl NestedMetaItem {
     {
         match tokens.peek() {
             Some(TokenTree::Token(token, _))
-                if let Ok(lit) = Lit::from_token(token) =>
+                if let Some(lit) = Lit::from_token(token) =>
             {
                 tokens.next();
                 return Some(NestedMetaItem::Literal(lit));

compiler/rustc_ast/src/token.rs

@@ -59,13 +59,17 @@ pub enum Delimiter {
     Invisible,
 }
 
+// Note that the suffix is *not* considered when deciding the `LitKind` in this
+// type. This means that float literals like `1f32` are classified by this type
+// as `Int`. Only upon conversion to `ast::LitKind` will such a literal be
+// given the `Float` kind.
 #[derive(Clone, Copy, PartialEq, Encodable, Decodable, Debug, HashStable_Generic)]
 pub enum LitKind {
     Bool, // AST only, must never appear in a `Token`
     Byte,
     Char,
-    Integer,
-    Float,
+    Integer, // e.g. `1`, `1u8`, `1f32`
+    Float,   // e.g. `1.`, `1.0`, `1e3f32`
     Str,
     StrRaw(u8), // raw string delimited by `n` hash symbols
     ByteStr,
@@ -81,6 +85,42 @@ pub struct Lit {
     pub suffix: Option<Symbol>,
 }
 
+impl Lit {
+    pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
+        Lit { kind, symbol, suffix }
+    }
+
+    /// Returns `true` if this is semantically a float literal. This includes
+    /// ones like `1f32` that have an `Integer` kind but a float suffix.
+    pub fn is_semantic_float(&self) -> bool {
+        match self.kind {
+            LitKind::Float => true,
+            LitKind::Integer => match self.suffix {
+                Some(sym) => sym == sym::f32 || sym == sym::f64,
+                None => false,
+            },
+            _ => false,
+        }
+    }
+
+    /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
+    pub fn from_token(token: &Token) -> Option<Lit> {
+        match token.uninterpolate().kind {
+            Ident(name, false) if name.is_bool_lit() => {
+                Some(Lit::new(Bool, name, None))
+            }
+            Literal(token_lit) => Some(token_lit),
+            Interpolated(ref nt)
+                if let NtExpr(expr) | NtLiteral(expr) = &**nt
+                && let ast::ExprKind::Lit(token_lit) = expr.kind =>
+            {
+                Some(token_lit.clone())
+            }
+            _ => None,
+        }
+    }
+}
+
 impl fmt::Display for Lit {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         let Lit { kind, symbol, suffix } = *self;
@@ -139,12 +179,6 @@ impl LitKind {
     }
 }
 
-impl Lit {
-    pub fn new(kind: LitKind, symbol: Symbol, suffix: Option<Symbol>) -> Lit {
-        Lit { kind, symbol, suffix }
-    }
-}
-
 pub fn ident_can_begin_expr(name: Symbol, span: Span, is_raw: bool) -> bool {
     let ident_token = Token::new(Ident(name, is_raw), span);
 

compiler/rustc_ast/src/util/literal.rs

@@ -8,8 +8,8 @@ use rustc_span::symbol::{kw, sym, Symbol};
 use rustc_span::Span;
 use std::ascii;
 
+#[derive(Debug)]
 pub enum LitError {
-    NotLiteral,
     LexerError,
     InvalidSuffix,
     InvalidIntSuffix,
@@ -202,27 +202,10 @@ impl Lit {
         Ok(Lit { token_lit, kind: LitKind::from_token_lit(token_lit)?, span })
     }
 
-    /// Converts arbitrary token into an AST literal.
-    ///
-    /// Keep this in sync with `Token::can_begin_literal_or_bool` excluding unary negation.
-    pub fn from_token(token: &Token) -> Result<Lit, LitError> {
-        let lit = match token.uninterpolate().kind {
-            token::Ident(name, false) if name.is_bool_lit() => {
-                token::Lit::new(token::Bool, name, None)
-            }
-            token::Literal(lit) => lit,
-            token::Interpolated(ref nt) => {
-                if let token::NtExpr(expr) | token::NtLiteral(expr) = &**nt
-                    && let ast::ExprKind::Lit(lit) = &expr.kind
-                {
-                    return Ok(lit.clone());
-                }
-                return Err(LitError::NotLiteral);
-            }
-            _ => return Err(LitError::NotLiteral),
-        };
-
-        Lit::from_token_lit(lit, token.span)
+    /// Converts an arbitrary token into an AST literal.
+    pub fn from_token(token: &Token) -> Option<Lit> {
+        token::Lit::from_token(token)
+            .and_then(|token_lit| Lit::from_token_lit(token_lit, token.span).ok())
     }
 
     /// Attempts to recover an AST literal from semantic literal.

compiler/rustc_ast_lowering/src/expr.rs

@@ -14,6 +14,7 @@ use rustc_data_structures::stack::ensure_sufficient_stack;
 use rustc_hir as hir;
 use rustc_hir::def::Res;
 use rustc_hir::definitions::DefPathData;
+use rustc_session::errors::report_lit_error;
 use rustc_span::source_map::{respan, DesugaringKind, Span, Spanned};
 use rustc_span::symbol::{sym, Ident};
 use rustc_span::DUMMY_SP;
@@ -84,8 +85,15 @@ impl<'hir> LoweringContext<'_, 'hir> {
                     let ohs = self.lower_expr(ohs);
                     hir::ExprKind::Unary(op, ohs)
                 }
-                ExprKind::Lit(ref l) => {
-                    hir::ExprKind::Lit(respan(self.lower_span(l.span), l.kind.clone()))
+                ExprKind::Lit(token_lit) => {
+                    let lit_kind = match LitKind::from_token_lit(token_lit) {
+                        Ok(lit_kind) => lit_kind,
+                        Err(err) => {
+                            report_lit_error(&self.tcx.sess.parse_sess, err, token_lit, e.span);
+                            LitKind::Err
+                        }
+                    };
+                    hir::ExprKind::Lit(respan(self.lower_span(e.span), lit_kind))
                 }
                 ExprKind::IncludedBytes(ref bytes) => hir::ExprKind::Lit(respan(
                     self.lower_span(e.span),

compiler/rustc_ast_lowering/src/lib.rs

@@ -959,8 +959,15 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
             MacArgs::Eq(eq_span, MacArgsEq::Ast(ref expr)) => {
                 // In valid code the value always ends up as a single literal. Otherwise, a dummy
                 // literal suffices because the error is handled elsewhere.
-                let lit = if let ExprKind::Lit(lit) = &expr.kind {
-                    lit.clone()
+                let lit = if let ExprKind::Lit(token_lit) = expr.kind {
+                    match Lit::from_token_lit(token_lit, expr.span) {
+                        Ok(lit) => lit,
+                        Err(_err) => Lit {
+                            token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),
+                            kind: LitKind::Err,
+                            span: DUMMY_SP,
+                        },
+                    }
                 } else {
                     Lit {
                         token_lit: token::Lit::new(token::LitKind::Err, kw::Empty, None),

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -373,8 +373,12 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
     }
 
     fn print_literal(&mut self, lit: &ast::Lit) {
-        self.maybe_print_comment(lit.span.lo());
-        self.word(lit.token_lit.to_string())
+        self.print_token_literal(lit.token_lit, lit.span)
+    }
+
+    fn print_token_literal(&mut self, token_lit: token::Lit, span: Span) {
+        self.maybe_print_comment(span.lo());
+        self.word(token_lit.to_string())
     }
 
     fn print_string(&mut self, st: &str, style: ast::StrStyle) {
@@ -1735,7 +1739,7 @@ impl<'a> State<'a> {
             }
             ast::Extern::Explicit(abi, _) => {
                 self.word_nbsp("extern");
-                self.print_literal(&abi.as_lit());
+                self.print_token_literal(abi.as_token_lit(), abi.span);
                 self.nbsp();
             }
         }

compiler/rustc_ast_pretty/src/pprust/state/expr.rs

@@ -319,8 +319,8 @@ impl<'a> State<'a> {
             ast::ExprKind::AddrOf(k, m, ref expr) => {
                 self.print_expr_addr_of(k, m, expr);
             }
-            ast::ExprKind::Lit(ref lit) => {
-                self.print_literal(lit);
+            ast::ExprKind::Lit(token_lit) => {
+                self.print_token_literal(token_lit, expr.span);
             }
             ast::ExprKind::IncludedBytes(ref bytes) => {
                 let lit = ast::Lit::from_included_bytes(bytes, expr.span);

compiler/rustc_ast_pretty/src/pprust/state/item.rs

@@ -207,7 +207,7 @@ impl<'a> State<'a> {
                     s.word("extern");
                 }));
                 if let Some(abi) = nmod.abi {
-                    self.print_literal(&abi.as_lit());
+                    self.print_token_literal(abi.as_token_lit(), abi.span);
                     self.nbsp();
                 }
                 self.bopen();

compiler/rustc_builtin_macros/src/asm.rs

@@ -172,7 +172,11 @@ pub fn parse_asm_args<'a>(
             // If it can't possibly expand to a string, provide diagnostics here to include other
             // things it could have been.
             match template.kind {
-                ast::ExprKind::Lit(ast::Lit { kind: ast::LitKind::Str(..), .. }) => {}
+                ast::ExprKind::Lit(token_lit)
+                    if matches!(
+                        token_lit.kind,
+                        token::LitKind::Str | token::LitKind::StrRaw(_)
+                    ) => {}
                 ast::ExprKind::MacCall(..) => {}
                 _ => {
                     let errstr = if is_global_asm {

compiler/rustc_builtin_macros/src/concat.rs

@@ -1,6 +1,7 @@
 use rustc_ast as ast;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_expand::base::{self, DummyResult};
+use rustc_session::errors::report_lit_error;
 use rustc_span::symbol::Symbol;
 
 use std::string::String;
@@ -18,28 +19,28 @@ pub fn expand_concat(
     let mut has_errors = false;
     for e in es {
         match e.kind {
-            ast::ExprKind::Lit(ref lit) => match lit.kind {
-                ast::LitKind::Str(ref s, _) | ast::LitKind::Float(ref s, _) => {
+            ast::ExprKind::Lit(token_lit) => match ast::LitKind::from_token_lit(token_lit) {
+                Ok(ast::LitKind::Str(ref s, _) | ast::LitKind::Float(ref s, _)) => {
                     accumulator.push_str(s.as_str());
                 }
-                ast::LitKind::Char(c) => {
+                Ok(ast::LitKind::Char(c)) => {
                     accumulator.push(c);
                 }
-                ast::LitKind::Int(
-                    i,
-                    ast::LitIntType::Unsigned(_)
-                    | ast::LitIntType::Signed(_)
-                    | ast::LitIntType::Unsuffixed,
-                ) => {
+                Ok(ast::LitKind::Int(i, _)) => {
                     accumulator.push_str(&i.to_string());
                 }
-                ast::LitKind::Bool(b) => {
+                Ok(ast::LitKind::Bool(b)) => {
                     accumulator.push_str(&b.to_string());
                 }
-                ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..) => {
+                Ok(ast::LitKind::Byte(..) | ast::LitKind::ByteStr(..)) => {
                     cx.span_err(e.span, "cannot concatenate a byte string literal");
+                    has_errors = true;
+                }
+                Ok(ast::LitKind::Err) => {
+                    has_errors = true;
                 }
-                ast::LitKind::Err => {
+                Err(err) => {
+                    report_lit_error(&cx.sess.parse_sess, err, token_lit, e.span);
                     has_errors = true;
                 }
             },