Auto merge of #63580 - wesleywiser:move_promoted_out, r=oli-obk

Move promoted MIR out of `mir::Body`

r? @oli-obk

src/librustc/arena.rs

@@ -25,6 +25,16 @@ macro_rules! arena_types {
             [] adt_def: rustc::ty::AdtDef,
             [] steal_mir: rustc::ty::steal::Steal<rustc::mir::Body<$tcx>>,
             [] mir: rustc::mir::Body<$tcx>,
+            [] steal_promoted: rustc::ty::steal::Steal<
+                rustc_data_structures::indexed_vec::IndexVec<
+                    rustc::mir::Promoted,
+                    rustc::mir::Body<$tcx>
+                >
+            >,
+            [] promoted: rustc_data_structures::indexed_vec::IndexVec<
+                rustc::mir::Promoted,
+                rustc::mir::Body<$tcx>
+            >,
             [] tables: rustc::ty::TypeckTables<$tcx>,
             [] const_allocs: rustc::mir::interpret::Allocation,
             [] vtable_method: Option<(

src/librustc/mir/mod.rs

@@ -108,11 +108,6 @@ pub struct Body<'tcx> {
     /// needn't) be tracked across crates.
     pub source_scope_local_data: ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>>,
 
-    /// Rvalues promoted from this function, such as borrows of constants.
-    /// Each of them is the Body of a constant with the fn's type parameters
-    /// in scope, but a separate set of locals.
-    pub promoted: IndexVec<Promoted, Body<'tcx>>,
-
     /// Yields type of the function, if it is a generator.
     pub yield_ty: Option<Ty<'tcx>>,
 
@@ -174,7 +169,6 @@ impl<'tcx> Body<'tcx> {
         basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
         source_scopes: IndexVec<SourceScope, SourceScopeData>,
         source_scope_local_data: ClearCrossCrate<IndexVec<SourceScope, SourceScopeLocalData>>,
-        promoted: IndexVec<Promoted, Body<'tcx>>,
         yield_ty: Option<Ty<'tcx>>,
         local_decls: LocalDecls<'tcx>,
         user_type_annotations: CanonicalUserTypeAnnotations<'tcx>,
@@ -196,7 +190,6 @@ impl<'tcx> Body<'tcx> {
             basic_blocks,
             source_scopes,
             source_scope_local_data,
-            promoted,
             yield_ty,
             generator_drop: None,
             generator_layout: None,
@@ -418,7 +411,6 @@ impl_stable_hash_for!(struct Body<'tcx> {
     basic_blocks,
     source_scopes,
     source_scope_local_data,
-    promoted,
     yield_ty,
     generator_drop,
     generator_layout,
@@ -1737,23 +1729,25 @@ pub enum PlaceBase<'tcx> {
 }
 
 /// We store the normalized type to avoid requiring normalization when reading MIR
-#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, RustcEncodable, RustcDecodable)]
+#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, RustcEncodable, RustcDecodable)]
 pub struct Static<'tcx> {
     pub ty: Ty<'tcx>,
-    pub kind: StaticKind,
+    pub kind: StaticKind<'tcx>,
+    pub def_id: DefId,
 }
 
 #[derive(
-    Clone, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable, RustcEncodable, RustcDecodable,
+    Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, HashStable, RustcEncodable, RustcDecodable,
 )]
-pub enum StaticKind {
-    Promoted(Promoted),
-    Static(DefId),
+pub enum StaticKind<'tcx> {
+    Promoted(Promoted, SubstsRef<'tcx>),
+    Static,
 }
 
 impl_stable_hash_for!(struct Static<'tcx> {
     ty,
-    kind
+    kind,
+    def_id
 });
 
 /// The `Projection` data structure defines things of the form `base.x`, `*b` or `b[index]`.
@@ -2114,10 +2108,12 @@ impl Debug for PlaceBase<'_> {
     fn fmt(&self, fmt: &mut Formatter<'_>) -> fmt::Result {
         match *self {
             PlaceBase::Local(id) => write!(fmt, "{:?}", id),
-            PlaceBase::Static(box self::Static { ty, kind: StaticKind::Static(def_id) }) => {
+            PlaceBase::Static(box self::Static { ty, kind: StaticKind::Static, def_id }) => {
                 write!(fmt, "({}: {:?})", ty::tls::with(|tcx| tcx.def_path_str(def_id)), ty)
             }
-            PlaceBase::Static(box self::Static { ty, kind: StaticKind::Promoted(promoted) }) => {
+            PlaceBase::Static(box self::Static {
+                ty, kind: StaticKind::Promoted(promoted, _), def_id: _
+            }) => {
                 write!(fmt, "({:?}: {:?})", promoted, ty)
             }
         }
@@ -3032,7 +3028,6 @@ BraceStructTypeFoldableImpl! {
         basic_blocks,
         source_scopes,
         source_scope_local_data,
-        promoted,
         yield_ty,
         generator_drop,
         generator_layout,
@@ -3226,13 +3221,63 @@ impl<'tcx> TypeFoldable<'tcx> for Terminator<'tcx> {
 impl<'tcx> TypeFoldable<'tcx> for Place<'tcx> {
     fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
         Place {
-            base: self.base.clone(),
+            base: self.base.fold_with(folder),
             projection: self.projection.fold_with(folder),
         }
     }
 
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
-        self.projection.visit_with(visitor)
+        self.base.visit_with(visitor) || self.projection.visit_with(visitor)
+    }
+}
+
+impl<'tcx> TypeFoldable<'tcx> for PlaceBase<'tcx> {
+    fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        match self {
+            PlaceBase::Local(local) => PlaceBase::Local(local.fold_with(folder)),
+            PlaceBase::Static(static_) => PlaceBase::Static(static_.fold_with(folder)),
+        }
+    }
+
+    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
+        match self {
+            PlaceBase::Local(local) => local.visit_with(visitor),
+            PlaceBase::Static(static_) => (**static_).visit_with(visitor),
+        }
+    }
+}
+
+impl<'tcx> TypeFoldable<'tcx> for Static<'tcx> {
+    fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        Static {
+            ty: self.ty.fold_with(folder),
+            kind: self.kind.fold_with(folder),
+            def_id: self.def_id,
+        }
+    }
+
+    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
+        let Static { ty, kind, def_id: _ } = self;
+
+        ty.visit_with(visitor) || kind.visit_with(visitor)
+    }
+}
+
+impl<'tcx> TypeFoldable<'tcx> for StaticKind<'tcx> {
+    fn super_fold_with<F: TypeFolder<'tcx>>(&self, folder: &mut F) -> Self {
+        match self {
+            StaticKind::Promoted(promoted, substs) =>
+                StaticKind::Promoted(promoted.fold_with(folder), substs.fold_with(folder)),
+            StaticKind::Static => StaticKind::Static
+        }
+    }
+
+    fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {
+        match self {
+            StaticKind::Promoted(promoted, substs) =>
+                promoted.visit_with(visitor) || substs.visit_with(visitor),
+            StaticKind::Static => { false }
+        }
     }
 }
 

src/librustc/mir/visit.rs

@@ -708,7 +708,7 @@ macro_rules! make_mir_visitor {
                     PlaceBase::Local(local) => {
                         self.visit_local(local, context, location);
                     }
-                    PlaceBase::Static(box Static { kind: _, ty }) => {
+                    PlaceBase::Static(box Static { kind: _, ty, def_id: _ }) => {
                         self.visit_ty(& $($mutability)? *ty, TyContext::Location(location));
                     }
                 }

src/librustc/query/mod.rs

@@ -110,7 +110,11 @@ rustc_queries! {
             no_hash
         }
 
-        query mir_validated(_: DefId) -> &'tcx Steal<mir::Body<'tcx>> {
+        query mir_validated(_: DefId) ->
+            (
+                &'tcx Steal<mir::Body<'tcx>>,
+                &'tcx Steal<IndexVec<mir::Promoted, mir::Body<'tcx>>>
+            ) {
             no_hash
         }
 
@@ -125,7 +129,17 @@ rustc_queries! {
             }
         }
 
-        query promoted_mir(key: DefId) -> &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>> { }
+        query promoted_mir(key: DefId) -> &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>> {
+            cache_on_disk_if { key.is_local() }
+            load_cached(tcx, id) {
+                let promoted: Option<
+                    rustc_data_structures::indexed_vec::IndexVec<
+                        crate::mir::Promoted,
+                        crate::mir::Body<'tcx>
+                    >> = tcx.queries.on_disk_cache.try_load_query_result(tcx, id);
+                promoted.map(|p| &*tcx.arena.alloc(p))
+            }
+        }
     }
 
     TypeChecking {

src/librustc/ty/context.rs

@@ -21,7 +21,7 @@ use crate::middle::cstore::EncodedMetadata;
 use crate::middle::lang_items;
 use crate::middle::resolve_lifetime::{self, ObjectLifetimeDefault};
 use crate::middle::stability;
-use crate::mir::{Body, interpret, ProjectionKind};
+use crate::mir::{Body, interpret, ProjectionKind, Promoted};
 use crate::mir::interpret::{ConstValue, Allocation, Scalar};
 use crate::ty::subst::{Kind, InternalSubsts, SubstsRef, Subst};
 use crate::ty::ReprOptions;
@@ -1096,6 +1096,16 @@ impl<'tcx> TyCtxt<'tcx> {
         self.arena.alloc(Steal::new(mir))
     }
 
+    pub fn alloc_steal_promoted(self, promoted: IndexVec<Promoted, Body<'tcx>>) ->
+        &'tcx Steal<IndexVec<Promoted, Body<'tcx>>> {
+        self.arena.alloc(Steal::new(promoted))
+    }
+
+    pub fn intern_promoted(self, promoted: IndexVec<Promoted, Body<'tcx>>) ->
+        &'tcx IndexVec<Promoted, Body<'tcx>> {
+        self.arena.alloc(promoted)
+    }
+
     pub fn alloc_adt_def(
         self,
         did: DefId,

src/librustc_codegen_ssa/mir/block.rs

@@ -609,17 +609,19 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                         mir::Operand::Copy(
                             Place {
                                 base: PlaceBase::Static(box Static {
-                                    kind: StaticKind::Promoted(promoted),
+                                    kind: StaticKind::Promoted(promoted, _),
                                     ty,
+                                    def_id: _,
                                 }),
                                 projection: None,
                             }
                         ) |
                         mir::Operand::Move(
                             Place {
                                 base: PlaceBase::Static(box Static {
-                                    kind: StaticKind::Promoted(promoted),
+                                    kind: StaticKind::Promoted(promoted, _),
                                     ty,
+                                    def_id: _,
                                 }),
                                 projection: None,
                             }

src/librustc_codegen_ssa/mir/place.rs

@@ -1,4 +1,4 @@
-use rustc::ty::{self, Ty};
+use rustc::ty::{self, Instance, Ty};
 use rustc::ty::layout::{self, Align, TyLayout, LayoutOf, VariantIdx, HasTyCtxt};
 use rustc::mir;
 use rustc::mir::tcx::PlaceTy;
@@ -454,13 +454,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::PlaceRef {
                 base: mir::PlaceBase::Static(box mir::Static {
                     ty,
-                    kind: mir::StaticKind::Promoted(promoted),
+                    kind: mir::StaticKind::Promoted(promoted, substs),
+                    def_id,
                 }),
                 projection: None,
             } => {
                 let param_env = ty::ParamEnv::reveal_all();
+                let instance = Instance::new(*def_id, self.monomorphize(substs));
                 let cid = mir::interpret::GlobalId {
-                    instance: self.instance,
+                    instance: instance,
                     promoted: Some(*promoted),
                 };
                 let layout = cx.layout_of(self.monomorphize(&ty));
@@ -487,7 +489,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
             mir::PlaceRef {
                 base: mir::PlaceBase::Static(box mir::Static {
                     ty,
-                    kind: mir::StaticKind::Static(def_id),
+                    kind: mir::StaticKind::Static,
+                    def_id,
                 }),
                 projection: None,
             } => {

src/librustc_incremental/persist/dirty_clean.rs

@@ -24,6 +24,7 @@ use rustc::hir::itemlikevisit::ItemLikeVisitor;
 use rustc::hir::intravisit;
 use rustc::ich::{ATTR_DIRTY, ATTR_CLEAN};
 use rustc::ty::TyCtxt;
+use rustc_data_structures::fingerprint::Fingerprint;
 use rustc_data_structures::fx::FxHashSet;
 use syntax::ast::{self, Attribute, NestedMetaItem};
 use syntax::symbol::{Symbol, sym};
@@ -71,6 +72,7 @@ const BASE_IMPL: &[&str] = &[
 /// code, i.e., functions+methods
 const BASE_MIR: &[&str] = &[
     label_strs::optimized_mir,
+    label_strs::promoted_mir,
     label_strs::mir_built,
 ];
 
@@ -472,26 +474,39 @@ impl DirtyCleanVisitor<'tcx> {
     fn assert_dirty(&self, item_span: Span, dep_node: DepNode) {
         debug!("assert_dirty({:?})", dep_node);
 
-        let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node);
-        let current_fingerprint = self.tcx.dep_graph.fingerprint_of(dep_node_index);
+        let current_fingerprint = self.get_fingerprint(&dep_node);
         let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node);
 
-        if Some(current_fingerprint) == prev_fingerprint {
+        if current_fingerprint == prev_fingerprint {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx.sess.span_err(
                 item_span,
                 &format!("`{}` should be dirty but is not", dep_node_str));
         }
     }
 
+    fn get_fingerprint(&self, dep_node: &DepNode) -> Option<Fingerprint> {
+        if self.tcx.dep_graph.dep_node_exists(dep_node) {
+            let dep_node_index = self.tcx.dep_graph.dep_node_index_of(dep_node);
+            Some(self.tcx.dep_graph.fingerprint_of(dep_node_index))
+        } else {
+            None
+        }
+    }
+
     fn assert_clean(&self, item_span: Span, dep_node: DepNode) {
         debug!("assert_clean({:?})", dep_node);
 
-        let dep_node_index = self.tcx.dep_graph.dep_node_index_of(&dep_node);
-        let current_fingerprint = self.tcx.dep_graph.fingerprint_of(dep_node_index);
+        let current_fingerprint = self.get_fingerprint(&dep_node);
         let prev_fingerprint = self.tcx.dep_graph.prev_fingerprint_of(&dep_node);
 
-        if Some(current_fingerprint) != prev_fingerprint {
+        // if the node wasn't previously evaluated and now is (or vice versa),
+        // then the node isn't actually clean or dirty.
+        if (current_fingerprint == None) ^ (prev_fingerprint == None) {
+            return;
+        }
+
+        if current_fingerprint != prev_fingerprint {
             let dep_node_str = self.dep_node_str(&dep_node);
             self.tcx.sess.span_err(
                 item_span,

src/librustc_metadata/cstore_impl.rs

@@ -136,6 +136,15 @@ provide! { <'tcx> tcx, def_id, other, cdata,
 
         mir
     }
+    promoted_mir => {
+        let promoted = cdata.maybe_get_promoted_mir(tcx, def_id.index).unwrap_or_else(|| {
+            bug!("get_promoted_mir: missing promoted MIR for `{:?}`", def_id)
+        });
+
+        let promoted = tcx.arena.alloc(promoted);
+
+        promoted
+    }
     mir_const_qualif => {
         (cdata.mir_const_qualif(def_id.index), tcx.arena.alloc(BitSet::new_empty(0)))
     }