diff --git a/compiler/rustc_ast/src/tokenstream.rs b/compiler/rustc_ast/src/tokenstream.rs index 84d8829c398e6..37de90d64c774 100644 --- a/compiler/rustc_ast/src/tokenstream.rs +++ b/compiler/rustc_ast/src/tokenstream.rs @@ -25,7 +25,7 @@ use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; use rustc_span::{Span, DUMMY_SP}; use smallvec::{smallvec, SmallVec}; -use std::{fmt, iter, mem}; +use std::{fmt, iter}; /// When the main Rust parser encounters a syntax-extension invocation, it /// parses the arguments to the invocation as a token tree. This is a very @@ -399,45 +399,6 @@ impl TokenStream { self.0.len() } - pub fn from_streams(mut streams: SmallVec<[TokenStream; 2]>) -> TokenStream { - match streams.len() { - 0 => TokenStream::default(), - 1 => streams.pop().unwrap(), - _ => { - // We are going to extend the first stream in `streams` with - // the elements from the subsequent streams. This requires - // using `make_mut()` on the first stream, and in practice this - // doesn't cause cloning 99.9% of the time. - // - // One very common use case is when `streams` has two elements, - // where the first stream has any number of elements within - // (often 1, but sometimes many more) and the second stream has - // a single element within. - - // Determine how much the first stream will be extended. - // Needed to avoid quadratic blow up from on-the-fly - // reallocations (#57735). - let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum(); - - // Get the first stream. If it's `None`, create an empty - // stream. - let mut iter = streams.drain(..); - let mut first_stream_lrc = iter.next().unwrap().0; - - // Append the elements to the first stream, after reserving - // space for them. - let first_vec_mut = Lrc::make_mut(&mut first_stream_lrc); - first_vec_mut.reserve(num_appends); - for stream in iter { - first_vec_mut.extend(stream.0.iter().cloned()); - } - - // Create the final `TokenStream`. - TokenStream(first_stream_lrc) - } - } - } - pub fn trees(&self) -> CursorRef<'_> { CursorRef::new(self) } @@ -562,50 +523,65 @@ impl TokenStreamBuilder { } pub fn push>(&mut self, stream: T) { - let mut stream = stream.into(); - - // If `self` is not empty and the last tree within the last stream is a - // token tree marked with `Joint`... - if let Some(TokenStream(ref mut last_stream_lrc)) = self.0.last_mut() - && let Some((TokenTree::Token(last_token), Spacing::Joint)) = last_stream_lrc.last() - // ...and `stream` is not empty and the first tree within it is - // a token tree... - && let TokenStream(ref mut stream_lrc) = stream - && let Some((TokenTree::Token(token), spacing)) = stream_lrc.first() - // ...and the two tokens can be glued together... - && let Some(glued_tok) = last_token.glue(&token) - { - // ...then do so, by overwriting the last token - // tree in `self` and removing the first token tree - // from `stream`. This requires using `make_mut()` - // on the last stream in `self` and on `stream`, - // and in practice this doesn't cause cloning 99.9% - // of the time. - - // Overwrite the last token tree with the merged - // token. - let last_vec_mut = Lrc::make_mut(last_stream_lrc); - *last_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); - - // Remove the first token tree from `stream`. (This - // is almost always the only tree in `stream`.) - let stream_vec_mut = Lrc::make_mut(stream_lrc); - stream_vec_mut.remove(0); - - // Don't push `stream` if it's empty -- that could - // block subsequent token gluing, by getting - // between two token trees that should be glued - // together. - if !stream.is_empty() { - self.0.push(stream); - } - return; - } - self.0.push(stream); + self.0.push(stream.into()); } pub fn build(self) -> TokenStream { - TokenStream::from_streams(self.0) + let mut streams = self.0; + match streams.len() { + 0 => TokenStream::default(), + 1 => streams.pop().unwrap(), + _ => { + // We will extend the first stream in `streams` with the + // elements from the subsequent streams. This requires using + // `make_mut()` on the first stream, and in practice this + // doesn't cause cloning 99.9% of the time. + // + // One very common use case is when `streams` has two elements, + // where the first stream has any number of elements within + // (often 1, but sometimes many more) and the second stream has + // a single element within. + + // Determine how much the first stream will be extended. + // Needed to avoid quadratic blow up from on-the-fly + // reallocations (#57735). + let num_appends = streams.iter().skip(1).map(|ts| ts.len()).sum(); + + // Get the first stream, which will become the result stream. + // If it's `None`, create an empty stream. + let mut iter = streams.drain(..); + let mut res_stream_lrc = iter.next().unwrap().0; + + // Append the subsequent elements to the result stream, after + // reserving space for them. + let res_vec_mut = Lrc::make_mut(&mut res_stream_lrc); + res_vec_mut.reserve(num_appends); + for stream in iter { + let stream_iter = stream.0.iter().cloned(); + + // If (a) `res_mut_vec` is not empty and the last tree + // within it is a token tree marked with `Joint`, and (b) + // `stream` is not empty and the first tree within it is a + // token tree, and (c) the two tokens can be glued + // together... + if let Some((TokenTree::Token(last_tok), Spacing::Joint)) = res_vec_mut.last() + && let Some((TokenTree::Token(tok), spacing)) = stream.0.first() + && let Some(glued_tok) = last_tok.glue(&tok) + { + // ...then overwrite the last token tree in + // `res_vec_mut` with the glued token, and skip the + // first token tree from `stream`. + *res_vec_mut.last_mut().unwrap() = (TokenTree::Token(glued_tok), *spacing); + res_vec_mut.extend(stream_iter.skip(1)); + } else { + // Append all of `stream`. + res_vec_mut.extend(stream_iter); + } + } + + TokenStream(res_stream_lrc) + } + } } } @@ -679,20 +655,6 @@ impl Cursor { }) } - pub fn index(&self) -> usize { - self.index - } - - pub fn append(&mut self, new_stream: TokenStream) { - if new_stream.is_empty() { - return; - } - let index = self.index; - let stream = mem::take(&mut self.stream); - *self = TokenStream::from_streams(smallvec![stream, new_stream]).into_trees(); - self.index = index; - } - pub fn look_ahead(&self, n: usize) -> Option<&TokenTree> { self.stream.0[self.index..].get(n).map(|(tree, _)| tree) } diff --git a/compiler/rustc_expand/src/proc_macro_server.rs b/compiler/rustc_expand/src/proc_macro_server.rs index 093896c339d29..1e4193a5a16cc 100644 --- a/compiler/rustc_expand/src/proc_macro_server.rs +++ b/compiler/rustc_expand/src/proc_macro_server.rs @@ -329,6 +329,7 @@ impl Ident { sess.symbol_gallery.insert(sym, span); Ident { sym, is_raw, span } } + fn dollar_crate(span: Span) -> Ident { // `$crate` is accepted as an ident only if it comes from the compiler. Ident { sym: kw::DollarCrate, is_raw: false, span } @@ -403,6 +404,7 @@ impl server::TokenStream for Rustc<'_, '_> { fn is_empty(&mut self, stream: &Self::TokenStream) -> bool { stream.is_empty() } + fn from_str(&mut self, src: &str) -> Self::TokenStream { parse_stream_from_source_str( FileName::proc_macro_source_code(src), @@ -411,9 +413,11 @@ impl server::TokenStream for Rustc<'_, '_> { Some(self.call_site), ) } + fn to_string(&mut self, stream: &Self::TokenStream) -> String { pprust::tts_to_string(stream) } + fn expand_expr(&mut self, stream: &Self::TokenStream) -> Result { // Parse the expression from our tokenstream. let expr: PResult<'_, _> = try { @@ -464,12 +468,14 @@ impl server::TokenStream for Rustc<'_, '_> { _ => Err(()), } } + fn from_token_tree( &mut self, tree: TokenTree, ) -> Self::TokenStream { tree.to_internal() } + fn concat_trees( &mut self, base: Option, @@ -484,6 +490,7 @@ impl server::TokenStream for Rustc<'_, '_> { } builder.build() } + fn concat_streams( &mut self, base: Option, @@ -498,6 +505,7 @@ impl server::TokenStream for Rustc<'_, '_> { } builder.build() } + fn into_trees( &mut self, stream: Self::TokenStream, @@ -522,10 +530,10 @@ impl server::TokenStream for Rustc<'_, '_> { // FIXME: It needs to be removed, but there are some // compatibility issues (see #73345). if group.flatten { - cursor.append(group.stream); - continue; + tts.append(&mut self.into_trees(group.stream)); + } else { + tts.push(TokenTree::Group(group)); } - tts.push(TokenTree::Group(group)); } Some(tt) => tts.push(tt), None => return tts, @@ -543,21 +551,27 @@ impl server::Group for Rustc<'_, '_> { flatten: false, } } + fn delimiter(&mut self, group: &Self::Group) -> Delimiter { group.delimiter } + fn stream(&mut self, group: &Self::Group) -> Self::TokenStream { group.stream.clone() } + fn span(&mut self, group: &Self::Group) -> Self::Span { group.span.entire() } + fn span_open(&mut self, group: &Self::Group) -> Self::Span { group.span.open } + fn span_close(&mut self, group: &Self::Group) -> Self::Span { group.span.close } + fn set_span(&mut self, group: &mut Self::Group, span: Self::Span) { group.span = DelimSpan::from_single(span); } @@ -567,15 +581,19 @@ impl server::Punct for Rustc<'_, '_> { fn new(&mut self, ch: char, spacing: Spacing) -> Self::Punct { Punct::new(ch, spacing == Spacing::Joint, server::Span::call_site(self)) } + fn as_char(&mut self, punct: Self::Punct) -> char { punct.ch } + fn spacing(&mut self, punct: Self::Punct) -> Spacing { if punct.joint { Spacing::Joint } else { Spacing::Alone } } + fn span(&mut self, punct: Self::Punct) -> Self::Span { punct.span } + fn with_span(&mut self, punct: Self::Punct, span: Self::Span) -> Self::Punct { Punct { span, ..punct } } @@ -585,9 +603,11 @@ impl server::Ident for Rustc<'_, '_> { fn new(&mut self, string: &str, span: Self::Span, is_raw: bool) -> Self::Ident { Ident::new(self.sess(), Symbol::intern(string), is_raw, span) } + fn span(&mut self, ident: Self::Ident) -> Self::Span { ident.span } + fn with_span(&mut self, ident: Self::Ident, span: Self::Span) -> Self::Ident { Ident { span, ..ident } } @@ -639,45 +659,57 @@ impl server::Literal for Rustc<'_, '_> { Ok(Literal { lit, span: self.call_site }) } + fn to_string(&mut self, literal: &Self::Literal) -> String { literal.lit.to_string() } + fn debug_kind(&mut self, literal: &Self::Literal) -> String { format!("{:?}", literal.lit.kind) } + fn symbol(&mut self, literal: &Self::Literal) -> String { literal.lit.symbol.to_string() } + fn suffix(&mut self, literal: &Self::Literal) -> Option { literal.lit.suffix.as_ref().map(Symbol::to_string) } + fn integer(&mut self, n: &str) -> Self::Literal { self.lit(token::Integer, Symbol::intern(n), None) } + fn typed_integer(&mut self, n: &str, kind: &str) -> Self::Literal { self.lit(token::Integer, Symbol::intern(n), Some(Symbol::intern(kind))) } + fn float(&mut self, n: &str) -> Self::Literal { self.lit(token::Float, Symbol::intern(n), None) } + fn f32(&mut self, n: &str) -> Self::Literal { self.lit(token::Float, Symbol::intern(n), Some(sym::f32)) } + fn f64(&mut self, n: &str) -> Self::Literal { self.lit(token::Float, Symbol::intern(n), Some(sym::f64)) } + fn string(&mut self, string: &str) -> Self::Literal { let quoted = format!("{:?}", string); assert!(quoted.starts_with('"') && quoted.ends_with('"')); let symbol = "ed[1..quoted.len() - 1]; self.lit(token::Str, Symbol::intern(symbol), None) } + fn character(&mut self, ch: char) -> Self::Literal { let quoted = format!("{:?}", ch); assert!(quoted.starts_with('\'') && quoted.ends_with('\'')); let symbol = "ed[1..quoted.len() - 1]; self.lit(token::Char, Symbol::intern(symbol), None) } + fn byte_string(&mut self, bytes: &[u8]) -> Self::Literal { let string = bytes .iter() @@ -687,12 +719,15 @@ impl server::Literal for Rustc<'_, '_> { .collect::(); self.lit(token::ByteStr, Symbol::intern(&string), None) } + fn span(&mut self, literal: &Self::Literal) -> Self::Span { literal.span } + fn set_span(&mut self, literal: &mut Self::Literal, span: Self::Span) { literal.span = span; } + fn subspan( &mut self, literal: &Self::Literal, @@ -735,6 +770,7 @@ impl server::SourceFile for Rustc<'_, '_> { fn eq(&mut self, file1: &Self::SourceFile, file2: &Self::SourceFile) -> bool { Lrc::ptr_eq(file1, file2) } + fn path(&mut self, file: &Self::SourceFile) -> String { match file.name { FileName::Real(ref name) => name @@ -746,6 +782,7 @@ impl server::SourceFile for Rustc<'_, '_> { _ => file.name.prefer_local().to_string(), } } + fn is_real(&mut self, file: &Self::SourceFile) -> bool { file.is_real_file() } @@ -755,6 +792,7 @@ impl server::MultiSpan for Rustc<'_, '_> { fn new(&mut self) -> Self::MultiSpan { vec![] } + fn push(&mut self, spans: &mut Self::MultiSpan, span: Self::Span) { spans.push(span) } @@ -766,6 +804,7 @@ impl server::Diagnostic for Rustc<'_, '_> { diag.set_span(MultiSpan::from_spans(spans)); diag } + fn sub( &mut self, diag: &mut Self::Diagnostic, @@ -775,6 +814,7 @@ impl server::Diagnostic for Rustc<'_, '_> { ) { diag.sub(level.to_internal(), msg, MultiSpan::from_spans(spans), None); } + fn emit(&mut self, mut diag: Self::Diagnostic) { self.sess().span_diagnostic.emit_diagnostic(&mut diag); } @@ -788,38 +828,49 @@ impl server::Span for Rustc<'_, '_> { format!("{:?} bytes({}..{})", span.ctxt(), span.lo().0, span.hi().0) } } + fn def_site(&mut self) -> Self::Span { self.def_site } + fn call_site(&mut self) -> Self::Span { self.call_site } + fn mixed_site(&mut self) -> Self::Span { self.mixed_site } + fn source_file(&mut self, span: Self::Span) -> Self::SourceFile { self.sess().source_map().lookup_char_pos(span.lo()).file } + fn parent(&mut self, span: Self::Span) -> Option { span.parent_callsite() } + fn source(&mut self, span: Self::Span) -> Self::Span { span.source_callsite() } + fn start(&mut self, span: Self::Span) -> LineColumn { let loc = self.sess().source_map().lookup_char_pos(span.lo()); LineColumn { line: loc.line, column: loc.col.to_usize() } } + fn end(&mut self, span: Self::Span) -> LineColumn { let loc = self.sess().source_map().lookup_char_pos(span.hi()); LineColumn { line: loc.line, column: loc.col.to_usize() } } + fn before(&mut self, span: Self::Span) -> Self::Span { span.shrink_to_lo() } + fn after(&mut self, span: Self::Span) -> Self::Span { span.shrink_to_hi() } + fn join(&mut self, first: Self::Span, second: Self::Span) -> Option { let self_loc = self.sess().source_map().lookup_char_pos(first.lo()); let other_loc = self.sess().source_map().lookup_char_pos(second.lo()); @@ -830,9 +881,11 @@ impl server::Span for Rustc<'_, '_> { Some(first.to(second)) } + fn resolved_at(&mut self, span: Self::Span, at: Self::Span) -> Self::Span { span.with_ctxt(at.ctxt()) } + fn source_text(&mut self, span: Self::Span) -> Option { self.sess().source_map().span_to_snippet(span).ok() } @@ -863,6 +916,7 @@ impl server::Span for Rustc<'_, '_> { fn save_span(&mut self, span: Self::Span) -> usize { self.sess().save_proc_macro_span(span) } + fn recover_proc_macro_span(&mut self, id: usize) -> Self::Span { let (resolver, krate, def_site) = (&*self.ecx.resolver, self.krate, self.def_site); *self.rebased_spans.entry(id).or_insert_with(|| { diff --git a/compiler/rustc_expand/src/tokenstream/tests.rs b/compiler/rustc_expand/src/tokenstream/tests.rs index 270532f8edeec..e4a4db204d922 100644 --- a/compiler/rustc_expand/src/tokenstream/tests.rs +++ b/compiler/rustc_expand/src/tokenstream/tests.rs @@ -4,7 +4,6 @@ use rustc_ast::token; use rustc_ast::tokenstream::{Spacing, TokenStream, TokenStreamBuilder, TokenTree}; use rustc_span::create_default_session_globals_then; use rustc_span::{BytePos, Span, Symbol}; -use smallvec::smallvec; fn string_to_ts(string: &str) -> TokenStream { string_to_stream(string.to_owned()) @@ -24,7 +23,10 @@ fn test_concat() { let test_res = string_to_ts("foo::bar::baz"); let test_fst = string_to_ts("foo::bar"); let test_snd = string_to_ts("::baz"); - let eq_res = TokenStream::from_streams(smallvec![test_fst, test_snd]); + let mut builder = TokenStreamBuilder::new(); + builder.push(test_fst); + builder.push(test_snd); + let eq_res = builder.build(); assert_eq!(test_res.trees().count(), 5); assert_eq!(eq_res.trees().count(), 5); assert_eq!(test_res.eq_unspanned(&eq_res), true);