diff --git a/src/librustc/ty/layout.rs b/src/librustc/ty/layout.rs index d1a8a9a34e155..6d7b0926c7ae2 100644 --- a/src/librustc/ty/layout.rs +++ b/src/librustc/ty/layout.rs @@ -19,6 +19,12 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableHasherResult}; pub use rustc_target::abi::*; +use rustc_target::spec::{HasTargetSpec, abi::Abi as SpecAbi}; +use rustc_target::abi::call::{ + ArgAttribute, ArgAttributes, ArgType, Conv, FnType, IgnoreMode, PassMode, Reg, RegKind +}; + + pub trait IntegerExt { fn to_ty<'a, 'tcx>(&self, tcx: TyCtxt<'a, 'tcx, 'tcx>, signed: bool) -> Ty<'tcx>; @@ -2259,3 +2265,380 @@ impl<'a, 'gcx> HashStable> for LayoutError<'gcx> } } } + +pub trait FnTypeExt<'tcx, C> +where + C: LayoutOf, TyLayout = TyLayout<'tcx>> + + HasDataLayout + + HasTargetSpec + + HasTyCtxt<'tcx> + + HasParamEnv<'tcx>, +{ + fn of_instance(cx: &C, instance: &ty::Instance<'tcx>) -> Self; + fn new(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self; + fn new_vtable(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self; + fn new_internal( + cx: &C, + sig: ty::FnSig<'tcx>, + extra_args: &[Ty<'tcx>], + mk_arg_type: impl Fn(Ty<'tcx>, Option) -> ArgType<'tcx, Ty<'tcx>>, + ) -> Self; + fn adjust_for_abi(&mut self, cx: &C, abi: SpecAbi); +} + +impl<'tcx, C> FnTypeExt<'tcx, C> for call::FnType<'tcx, Ty<'tcx>> +where + C: LayoutOf, TyLayout = TyLayout<'tcx>> + + HasDataLayout + + HasTargetSpec + + HasTyCtxt<'tcx> + + HasParamEnv<'tcx>, +{ + fn of_instance(cx: &C, instance: &ty::Instance<'tcx>) -> Self { + let sig = instance.fn_sig(cx.tcx()); + let sig = cx + .tcx() + .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig); + call::FnType::new(cx, sig, &[]) + } + + fn new(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self { + call::FnType::new_internal(cx, sig, extra_args, |ty, _| ArgType::new(cx.layout_of(ty))) + } + + fn new_vtable(cx: &C, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> Self { + FnTypeExt::new_internal(cx, sig, extra_args, |ty, arg_idx| { + let mut layout = cx.layout_of(ty); + // Don't pass the vtable, it's not an argument of the virtual fn. + // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait` + // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen + if arg_idx == Some(0) { + let fat_pointer_ty = if layout.is_unsized() { + // unsized `self` is passed as a pointer to `self` + // FIXME (mikeyhew) change this to use &own if it is ever added to the language + cx.tcx().mk_mut_ptr(layout.ty) + } else { + match layout.abi { + Abi::ScalarPair(..) => (), + _ => bug!("receiver type has unsupported layout: {:?}", layout), + } + + // In the case of Rc, we need to explicitly pass a *mut RcBox + // with a Scalar (not ScalarPair) ABI. This is a hack that is understood + // elsewhere in the compiler as a method on a `dyn Trait`. + // To get the type `*mut RcBox`, we just keep unwrapping newtypes until we + // get a built-in pointer type + let mut fat_pointer_layout = layout; + 'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr() + && !fat_pointer_layout.ty.is_region_ptr() + { + 'iter_fields: for i in 0..fat_pointer_layout.fields.count() { + let field_layout = fat_pointer_layout.field(cx, i); + + if !field_layout.is_zst() { + fat_pointer_layout = field_layout; + continue 'descend_newtypes; + } + } + + bug!( + "receiver has no non-zero-sized fields {:?}", + fat_pointer_layout + ); + } + + fat_pointer_layout.ty + }; + + // we now have a type like `*mut RcBox` + // change its layout to that of `*mut ()`, a thin pointer, but keep the same type + // this is understood as a special case elsewhere in the compiler + let unit_pointer_ty = cx.tcx().mk_mut_ptr(cx.tcx().mk_unit()); + layout = cx.layout_of(unit_pointer_ty); + layout.ty = fat_pointer_ty; + } + ArgType::new(layout) + }) + } + + fn new_internal( + cx: &C, + sig: ty::FnSig<'tcx>, + extra_args: &[Ty<'tcx>], + mk_arg_type: impl Fn(Ty<'tcx>, Option) -> ArgType<'tcx, Ty<'tcx>>, + ) -> Self { + debug!("FnType::new_internal({:?}, {:?})", sig, extra_args); + + use rustc_target::spec::abi::Abi::*; + let conv = match cx.tcx().sess.target.target.adjust_abi(sig.abi) { + RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::C, + + // It's the ABI's job to select this, not ours. + System => bug!("system abi should be selected elsewhere"), + + Stdcall => Conv::X86Stdcall, + Fastcall => Conv::X86Fastcall, + Vectorcall => Conv::X86VectorCall, + Thiscall => Conv::X86ThisCall, + C => Conv::C, + Unadjusted => Conv::C, + Win64 => Conv::X86_64Win64, + SysV64 => Conv::X86_64SysV, + Aapcs => Conv::ArmAapcs, + PtxKernel => Conv::PtxKernel, + Msp430Interrupt => Conv::Msp430Intr, + X86Interrupt => Conv::X86Intr, + AmdGpuKernel => Conv::AmdGpuKernel, + + // These API constants ought to be more specific... + Cdecl => Conv::C, + }; + + let mut inputs = sig.inputs(); + let extra_args = if sig.abi == RustCall { + assert!(!sig.c_variadic && extra_args.is_empty()); + + match sig.inputs().last().unwrap().sty { + ty::Tuple(tupled_arguments) => { + inputs = &sig.inputs()[0..sig.inputs().len() - 1]; + tupled_arguments.iter().map(|k| k.expect_ty()).collect() + } + _ => { + bug!( + "argument to function with \"rust-call\" ABI \ + is not a tuple" + ); + } + } + } else { + assert!(sig.c_variadic || extra_args.is_empty()); + extra_args.to_vec() + }; + + let target = &cx.tcx().sess.target.target; + let win_x64_gnu = + target.target_os == "windows" && target.arch == "x86_64" && target.target_env == "gnu"; + let linux_s390x = + target.target_os == "linux" && target.arch == "s390x" && target.target_env == "gnu"; + let linux_sparc64 = + target.target_os == "linux" && target.arch == "sparc64" && target.target_env == "gnu"; + let rust_abi = match sig.abi { + RustIntrinsic | PlatformIntrinsic | Rust | RustCall => true, + _ => false, + }; + + // Handle safe Rust thin and fat pointers. + let adjust_for_rust_scalar = |attrs: &mut ArgAttributes, + scalar: &Scalar, + layout: TyLayout<'tcx>, + offset: Size, + is_return: bool| { + // Booleans are always an i1 that needs to be zero-extended. + if scalar.is_bool() { + attrs.set(ArgAttribute::ZExt); + return; + } + + // Only pointer types handled below. + if scalar.value != Pointer { + return; + } + + if scalar.valid_range.start() < scalar.valid_range.end() { + if *scalar.valid_range.start() > 0 { + attrs.set(ArgAttribute::NonNull); + } + } + + if let Some(pointee) = layout.pointee_info_at(cx, offset) { + if let Some(kind) = pointee.safe { + attrs.pointee_size = pointee.size; + attrs.pointee_align = Some(pointee.align); + + // `Box` pointer parameters never alias because ownership is transferred + // `&mut` pointer parameters never alias other parameters, + // or mutable global data + // + // `&T` where `T` contains no `UnsafeCell` is immutable, + // and can be marked as both `readonly` and `noalias`, as + // LLVM's definition of `noalias` is based solely on memory + // dependencies rather than pointer equality + let no_alias = match kind { + PointerKind::Shared => false, + PointerKind::UniqueOwned => true, + PointerKind::Frozen | PointerKind::UniqueBorrowed => !is_return, + }; + if no_alias { + attrs.set(ArgAttribute::NoAlias); + } + + if kind == PointerKind::Frozen && !is_return { + attrs.set(ArgAttribute::ReadOnly); + } + } + } + }; + + // Store the index of the last argument. This is useful for working with + // C-compatible variadic arguments. + let last_arg_idx = if sig.inputs().is_empty() { + None + } else { + Some(sig.inputs().len() - 1) + }; + + let arg_of = |ty: Ty<'tcx>, arg_idx: Option| { + let is_return = arg_idx.is_none(); + let mut arg = mk_arg_type(ty, arg_idx); + if arg.layout.is_zst() { + // For some forsaken reason, x86_64-pc-windows-gnu + // doesn't ignore zero-sized struct arguments. + // The same is true for s390x-unknown-linux-gnu + // and sparc64-unknown-linux-gnu. + if is_return || rust_abi || (!win_x64_gnu && !linux_s390x && !linux_sparc64) { + arg.mode = PassMode::Ignore(IgnoreMode::Zst); + } + } + + // If this is a C-variadic function, this is not the return value, + // and there is one or more fixed arguments; ensure that the `VaList` + // is ignored as an argument. + if sig.c_variadic { + match (last_arg_idx, arg_idx) { + (Some(last_idx), Some(cur_idx)) if last_idx == cur_idx => { + let va_list_did = match cx.tcx().lang_items().va_list() { + Some(did) => did, + None => bug!("`va_list` lang item required for C-variadic functions"), + }; + match ty.sty { + ty::Adt(def, _) if def.did == va_list_did => { + // This is the "spoofed" `VaList`. Set the arguments mode + // so that it will be ignored. + arg.mode = PassMode::Ignore(IgnoreMode::CVarArgs); + } + _ => (), + } + } + _ => {} + } + } + + // FIXME(eddyb) other ABIs don't have logic for scalar pairs. + if !is_return && rust_abi { + if let Abi::ScalarPair(ref a, ref b) = arg.layout.abi { + let mut a_attrs = ArgAttributes::new(); + let mut b_attrs = ArgAttributes::new(); + adjust_for_rust_scalar(&mut a_attrs, a, arg.layout, Size::ZERO, false); + adjust_for_rust_scalar( + &mut b_attrs, + b, + arg.layout, + a.value.size(cx).align_to(b.value.align(cx).abi), + false, + ); + arg.mode = PassMode::Pair(a_attrs, b_attrs); + return arg; + } + } + + if let Abi::Scalar(ref scalar) = arg.layout.abi { + if let PassMode::Direct(ref mut attrs) = arg.mode { + adjust_for_rust_scalar(attrs, scalar, arg.layout, Size::ZERO, is_return); + } + } + + arg + }; + + let mut fn_ty = FnType { + ret: arg_of(sig.output(), None), + args: inputs + .iter() + .cloned() + .chain(extra_args) + .enumerate() + .map(|(i, ty)| arg_of(ty, Some(i))) + .collect(), + c_variadic: sig.c_variadic, + conv, + }; + fn_ty.adjust_for_abi(cx, sig.abi); + fn_ty + } + + fn adjust_for_abi(&mut self, cx: &C, abi: SpecAbi) { + if abi == SpecAbi::Unadjusted { + return; + } + + if abi == SpecAbi::Rust + || abi == SpecAbi::RustCall + || abi == SpecAbi::RustIntrinsic + || abi == SpecAbi::PlatformIntrinsic + { + let fixup = |arg: &mut ArgType<'tcx, Ty<'tcx>>| { + if arg.is_ignore() { + return; + } + + match arg.layout.abi { + Abi::Aggregate { .. } => {} + + // This is a fun case! The gist of what this is doing is + // that we want callers and callees to always agree on the + // ABI of how they pass SIMD arguments. If we were to *not* + // make these arguments indirect then they'd be immediates + // in LLVM, which means that they'd used whatever the + // appropriate ABI is for the callee and the caller. That + // means, for example, if the caller doesn't have AVX + // enabled but the callee does, then passing an AVX argument + // across this boundary would cause corrupt data to show up. + // + // This problem is fixed by unconditionally passing SIMD + // arguments through memory between callers and callees + // which should get them all to agree on ABI regardless of + // target feature sets. Some more information about this + // issue can be found in #44367. + // + // Note that the platform intrinsic ABI is exempt here as + // that's how we connect up to LLVM and it's unstable + // anyway, we control all calls to it in libstd. + Abi::Vector { .. } + if abi != SpecAbi::PlatformIntrinsic + && cx.tcx().sess.target.target.options.simd_types_indirect => + { + arg.make_indirect(); + return; + } + + _ => return, + } + + let size = arg.layout.size; + if arg.layout.is_unsized() || size > Pointer.size(cx) { + arg.make_indirect(); + } else { + // We want to pass small aggregates as immediates, but using + // a LLVM aggregate type for this leads to bad optimizations, + // so we pick an appropriately sized integer type instead. + arg.cast_to(Reg { + kind: RegKind::Integer, + size, + }); + } + }; + fixup(&mut self.ret); + for arg in &mut self.args { + fixup(arg); + } + if let PassMode::Indirect(ref mut attrs, _) = self.ret.mode { + attrs.set(ArgAttribute::StructRet); + } + return; + } + + if let Err(msg) = self.adjust_for_cabi(cx, abi) { + cx.tcx().sess.fatal(&msg); + } + } +} diff --git a/src/librustc_codegen_llvm/abi.rs b/src/librustc_codegen_llvm/abi.rs index 70d184240fccd..38d4b7e3f9d85 100644 --- a/src/librustc_codegen_llvm/abi.rs +++ b/src/librustc_codegen_llvm/abi.rs @@ -11,9 +11,9 @@ use rustc_target::abi::call::ArgType; use rustc_codegen_ssa::traits::*; -use rustc_target::abi::{HasDataLayout, LayoutOf, Size, TyLayout, Abi as LayoutAbi}; -use rustc::ty::{self, Ty, Instance}; -use rustc::ty::layout::{self, PointerKind}; +use rustc_target::abi::{HasDataLayout, LayoutOf}; +use rustc::ty::{Ty}; +use rustc::ty::layout::{self}; use libc::c_uint; @@ -294,23 +294,7 @@ impl ArgTypeMethods<'tcx> for Builder<'a, 'll, 'tcx> { } } -pub trait FnTypeExt<'tcx> { - fn of_instance(cx: &CodegenCx<'ll, 'tcx>, instance: &ty::Instance<'tcx>) -> Self; - fn new(cx: &CodegenCx<'ll, 'tcx>, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>]) -> Self; - fn new_vtable(cx: &CodegenCx<'ll, 'tcx>, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>]) -> Self; - fn new_internal( - cx: &CodegenCx<'ll, 'tcx>, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>], - mk_arg_type: impl Fn(Ty<'tcx>, Option) -> ArgType<'tcx, Ty<'tcx>>, - ) -> Self; - fn adjust_for_abi(&mut self, - cx: &CodegenCx<'ll, 'tcx>, - abi: Abi); +pub trait FnTypeLlvmExt<'tcx> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; fn llvm_cconv(&self) -> llvm::CallConv; @@ -318,356 +302,7 @@ pub trait FnTypeExt<'tcx> { fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value); } -impl<'tcx> FnTypeExt<'tcx> for FnType<'tcx, Ty<'tcx>> { - fn of_instance(cx: &CodegenCx<'ll, 'tcx>, instance: &ty::Instance<'tcx>) -> Self { - let sig = instance.fn_sig(cx.tcx); - let sig = cx.tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig); - FnType::new(cx, sig, &[]) - } - - fn new(cx: &CodegenCx<'ll, 'tcx>, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>]) -> Self { - FnType::new_internal(cx, sig, extra_args, |ty, _| { - ArgType::new(cx.layout_of(ty)) - }) - } - - fn new_vtable(cx: &CodegenCx<'ll, 'tcx>, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>]) -> Self { - FnType::new_internal(cx, sig, extra_args, |ty, arg_idx| { - let mut layout = cx.layout_of(ty); - // Don't pass the vtable, it's not an argument of the virtual fn. - // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait` - // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen - if arg_idx == Some(0) { - let fat_pointer_ty = if layout.is_unsized() { - // unsized `self` is passed as a pointer to `self` - // FIXME (mikeyhew) change this to use &own if it is ever added to the language - cx.tcx.mk_mut_ptr(layout.ty) - } else { - match layout.abi { - LayoutAbi::ScalarPair(..) => (), - _ => bug!("receiver type has unsupported layout: {:?}", layout) - } - - // In the case of Rc, we need to explicitly pass a *mut RcBox - // with a Scalar (not ScalarPair) ABI. This is a hack that is understood - // elsewhere in the compiler as a method on a `dyn Trait`. - // To get the type `*mut RcBox`, we just keep unwrapping newtypes until we - // get a built-in pointer type - let mut fat_pointer_layout = layout; - 'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr() - && !fat_pointer_layout.ty.is_region_ptr() - { - 'iter_fields: for i in 0..fat_pointer_layout.fields.count() { - let field_layout = fat_pointer_layout.field(cx, i); - - if !field_layout.is_zst() { - fat_pointer_layout = field_layout; - continue 'descend_newtypes - } - } - - bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout); - } - - fat_pointer_layout.ty - }; - - // we now have a type like `*mut RcBox` - // change its layout to that of `*mut ()`, a thin pointer, but keep the same type - // this is understood as a special case elsewhere in the compiler - let unit_pointer_ty = cx.tcx.mk_mut_ptr(cx.tcx.mk_unit()); - layout = cx.layout_of(unit_pointer_ty); - layout.ty = fat_pointer_ty; - } - ArgType::new(layout) - }) - } - - fn new_internal( - cx: &CodegenCx<'ll, 'tcx>, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>], - mk_arg_type: impl Fn(Ty<'tcx>, Option) -> ArgType<'tcx, Ty<'tcx>>, - ) -> Self { - debug!("FnType::new_internal({:?}, {:?})", sig, extra_args); - - use self::Abi::*; - let conv = match cx.sess().target.target.adjust_abi(sig.abi) { - RustIntrinsic | PlatformIntrinsic | - Rust | RustCall => Conv::C, - - // It's the ABI's job to select this, not ours. - System => bug!("system abi should be selected elsewhere"), - - Stdcall => Conv::X86Stdcall, - Fastcall => Conv::X86Fastcall, - Vectorcall => Conv::X86VectorCall, - Thiscall => Conv::X86ThisCall, - C => Conv::C, - Unadjusted => Conv::C, - Win64 => Conv::X86_64Win64, - SysV64 => Conv::X86_64SysV, - Aapcs => Conv::ArmAapcs, - PtxKernel => Conv::PtxKernel, - Msp430Interrupt => Conv::Msp430Intr, - X86Interrupt => Conv::X86Intr, - AmdGpuKernel => Conv::AmdGpuKernel, - - // These API constants ought to be more specific... - Cdecl => Conv::C, - }; - - let mut inputs = sig.inputs(); - let extra_args = if sig.abi == RustCall { - assert!(!sig.c_variadic && extra_args.is_empty()); - - match sig.inputs().last().unwrap().sty { - ty::Tuple(tupled_arguments) => { - inputs = &sig.inputs()[0..sig.inputs().len() - 1]; - tupled_arguments.iter().map(|k| k.expect_ty()).collect() - } - _ => { - bug!("argument to function with \"rust-call\" ABI \ - is not a tuple"); - } - } - } else { - assert!(sig.c_variadic || extra_args.is_empty()); - extra_args.to_vec() - }; - - let target = &cx.sess().target.target; - let win_x64_gnu = target.target_os == "windows" - && target.arch == "x86_64" - && target.target_env == "gnu"; - let linux_s390x = target.target_os == "linux" - && target.arch == "s390x" - && target.target_env == "gnu"; - let linux_sparc64 = target.target_os == "linux" - && target.arch == "sparc64" - && target.target_env == "gnu"; - let rust_abi = match sig.abi { - RustIntrinsic | PlatformIntrinsic | Rust | RustCall => true, - _ => false - }; - - // Handle safe Rust thin and fat pointers. - let adjust_for_rust_scalar = |attrs: &mut ArgAttributes, - scalar: &layout::Scalar, - layout: TyLayout<'tcx, Ty<'tcx>>, - offset: Size, - is_return: bool| { - // Booleans are always an i1 that needs to be zero-extended. - if scalar.is_bool() { - attrs.set(ArgAttribute::ZExt); - return; - } - - // Only pointer types handled below. - if scalar.value != layout::Pointer { - return; - } - - if scalar.valid_range.start() < scalar.valid_range.end() { - if *scalar.valid_range.start() > 0 { - attrs.set(ArgAttribute::NonNull); - } - } - - if let Some(pointee) = layout.pointee_info_at(cx, offset) { - if let Some(kind) = pointee.safe { - attrs.pointee_size = pointee.size; - attrs.pointee_align = Some(pointee.align); - - // `Box` pointer parameters never alias because ownership is transferred - // `&mut` pointer parameters never alias other parameters, - // or mutable global data - // - // `&T` where `T` contains no `UnsafeCell` is immutable, - // and can be marked as both `readonly` and `noalias`, as - // LLVM's definition of `noalias` is based solely on memory - // dependencies rather than pointer equality - let no_alias = match kind { - PointerKind::Shared => false, - PointerKind::UniqueOwned => true, - PointerKind::Frozen | - PointerKind::UniqueBorrowed => !is_return - }; - if no_alias { - attrs.set(ArgAttribute::NoAlias); - } - - if kind == PointerKind::Frozen && !is_return { - attrs.set(ArgAttribute::ReadOnly); - } - } - } - }; - - // Store the index of the last argument. This is useful for working with - // C-compatible variadic arguments. - let last_arg_idx = if sig.inputs().is_empty() { - None - } else { - Some(sig.inputs().len() - 1) - }; - - let arg_of = |ty: Ty<'tcx>, arg_idx: Option| { - let is_return = arg_idx.is_none(); - let mut arg = mk_arg_type(ty, arg_idx); - if arg.layout.is_zst() { - // For some forsaken reason, x86_64-pc-windows-gnu - // doesn't ignore zero-sized struct arguments. - // The same is true for s390x-unknown-linux-gnu - // and sparc64-unknown-linux-gnu. - if is_return || rust_abi || (!win_x64_gnu && !linux_s390x && !linux_sparc64) { - arg.mode = PassMode::Ignore(IgnoreMode::Zst); - } - } - - // If this is a C-variadic function, this is not the return value, - // and there is one or more fixed arguments; ensure that the `VaList` - // is ignored as an argument. - if sig.c_variadic { - match (last_arg_idx, arg_idx) { - (Some(last_idx), Some(cur_idx)) if last_idx == cur_idx => { - let va_list_did = match cx.tcx.lang_items().va_list() { - Some(did) => did, - None => bug!("`va_list` lang item required for C-variadic functions"), - }; - match ty.sty { - ty::Adt(def, _) if def.did == va_list_did => { - // This is the "spoofed" `VaList`. Set the arguments mode - // so that it will be ignored. - arg.mode = PassMode::Ignore(IgnoreMode::CVarArgs); - }, - _ => (), - } - } - _ => {} - } - } - - // FIXME(eddyb) other ABIs don't have logic for scalar pairs. - if !is_return && rust_abi { - if let layout::Abi::ScalarPair(ref a, ref b) = arg.layout.abi { - let mut a_attrs = ArgAttributes::new(); - let mut b_attrs = ArgAttributes::new(); - adjust_for_rust_scalar(&mut a_attrs, - a, - arg.layout, - Size::ZERO, - false); - adjust_for_rust_scalar(&mut b_attrs, - b, - arg.layout, - a.value.size(cx).align_to(b.value.align(cx).abi), - false); - arg.mode = PassMode::Pair(a_attrs, b_attrs); - return arg; - } - } - - if let layout::Abi::Scalar(ref scalar) = arg.layout.abi { - if let PassMode::Direct(ref mut attrs) = arg.mode { - adjust_for_rust_scalar(attrs, - scalar, - arg.layout, - Size::ZERO, - is_return); - } - } - - arg - }; - - let mut fn_ty = FnType { - ret: arg_of(sig.output(), None), - args: inputs.iter().cloned().chain(extra_args).enumerate().map(|(i, ty)| { - arg_of(ty, Some(i)) - }).collect(), - c_variadic: sig.c_variadic, - conv, - }; - fn_ty.adjust_for_abi(cx, sig.abi); - fn_ty - } - - fn adjust_for_abi(&mut self, - cx: &CodegenCx<'ll, 'tcx>, - abi: Abi) { - if abi == Abi::Unadjusted { return } - - if abi == Abi::Rust || abi == Abi::RustCall || - abi == Abi::RustIntrinsic || abi == Abi::PlatformIntrinsic { - let fixup = |arg: &mut ArgType<'tcx, Ty<'tcx>>| { - if arg.is_ignore() { return; } - - match arg.layout.abi { - layout::Abi::Aggregate { .. } => {} - - // This is a fun case! The gist of what this is doing is - // that we want callers and callees to always agree on the - // ABI of how they pass SIMD arguments. If we were to *not* - // make these arguments indirect then they'd be immediates - // in LLVM, which means that they'd used whatever the - // appropriate ABI is for the callee and the caller. That - // means, for example, if the caller doesn't have AVX - // enabled but the callee does, then passing an AVX argument - // across this boundary would cause corrupt data to show up. - // - // This problem is fixed by unconditionally passing SIMD - // arguments through memory between callers and callees - // which should get them all to agree on ABI regardless of - // target feature sets. Some more information about this - // issue can be found in #44367. - // - // Note that the platform intrinsic ABI is exempt here as - // that's how we connect up to LLVM and it's unstable - // anyway, we control all calls to it in libstd. - layout::Abi::Vector { .. } - if abi != Abi::PlatformIntrinsic && - cx.sess().target.target.options.simd_types_indirect => - { - arg.make_indirect(); - return - } - - _ => return - } - - let size = arg.layout.size; - if arg.layout.is_unsized() || size > layout::Pointer.size(cx) { - arg.make_indirect(); - } else { - // We want to pass small aggregates as immediates, but using - // a LLVM aggregate type for this leads to bad optimizations, - // so we pick an appropriately sized integer type instead. - arg.cast_to(Reg { - kind: RegKind::Integer, - size - }); - } - }; - fixup(&mut self.ret); - for arg in &mut self.args { - fixup(arg); - } - if let PassMode::Indirect(ref mut attrs, _) = self.ret.mode { - attrs.set(ArgAttribute::StructRet); - } - return; - } - - if let Err(msg) = self.adjust_for_cabi(cx, abi) { - cx.sess().fatal(&msg); - } - } - +impl<'tcx> FnTypeLlvmExt<'tcx> for FnType<'tcx, Ty<'tcx>> { fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { let args_capacity: usize = self.args.iter().map(|arg| if arg.pad.is_some() { 1 } else { 0 } + @@ -836,22 +471,6 @@ impl<'tcx> FnTypeExt<'tcx> for FnType<'tcx, Ty<'tcx>> { } } -impl AbiMethods<'tcx> for CodegenCx<'ll, 'tcx> { - fn new_fn_type(&self, sig: ty::FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> FnType<'tcx, Ty<'tcx>> { - FnType::new(&self, sig, extra_args) - } - fn new_vtable( - &self, - sig: ty::FnSig<'tcx>, - extra_args: &[Ty<'tcx>] - ) -> FnType<'tcx, Ty<'tcx>> { - FnType::new_vtable(&self, sig, extra_args) - } - fn fn_type_of_instance(&self, instance: &Instance<'tcx>) -> FnType<'tcx, Ty<'tcx>> { - FnType::of_instance(&self, instance) - } -} - impl AbiBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> { fn apply_attrs_callsite( &mut self, diff --git a/src/librustc_codegen_llvm/builder.rs b/src/librustc_codegen_llvm/builder.rs index bc2bb97a19e54..48808eea3045e 100644 --- a/src/librustc_codegen_llvm/builder.rs +++ b/src/librustc_codegen_llvm/builder.rs @@ -18,6 +18,7 @@ use rustc_codegen_ssa::traits::*; use rustc_codegen_ssa::base::to_immediate; use rustc_codegen_ssa::mir::operand::{OperandValue, OperandRef}; use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_target::spec::{HasTargetSpec, Target}; use std::borrow::Cow; use std::ops::{Deref, Range}; use std::ptr; @@ -72,6 +73,12 @@ impl ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { } } +impl HasTargetSpec for Builder<'_, '_, 'tcx> { + fn target_spec(&self) -> &Target { + &self.cx.target_spec() + } +} + impl ty::layout::LayoutOf for Builder<'_, '_, 'tcx> { type Ty = Ty<'tcx>; type TyLayout = TyLayout<'tcx>; diff --git a/src/librustc_codegen_llvm/declare.rs b/src/librustc_codegen_llvm/declare.rs index 3febcb019ce29..bcb14b8899ec2 100644 --- a/src/librustc_codegen_llvm/declare.rs +++ b/src/librustc_codegen_llvm/declare.rs @@ -13,13 +13,13 @@ use crate::llvm; use crate::llvm::AttributePlace::Function; -use crate::abi::{FnType, FnTypeExt}; +use crate::abi::{FnType, FnTypeLlvmExt}; use crate::attributes; use crate::context::CodegenCx; use crate::type_::Type; use crate::value::Value; use rustc::ty::{self, PolyFnSig}; -use rustc::ty::layout::LayoutOf; +use rustc::ty::layout::{FnTypeExt, LayoutOf}; use rustc::session::config::Sanitizer; use rustc_data_structures::small_c_str::SmallCStr; use rustc_codegen_ssa::traits::*; diff --git a/src/librustc_codegen_llvm/type_.rs b/src/librustc_codegen_llvm/type_.rs index a5ed64a66a39d..a3d3f0756a5f3 100644 --- a/src/librustc_codegen_llvm/type_.rs +++ b/src/librustc_codegen_llvm/type_.rs @@ -10,7 +10,7 @@ use rustc_codegen_ssa::traits::*; use crate::common; use crate::type_of::LayoutLlvmExt; -use crate::abi::{LlvmType, FnTypeExt}; +use crate::abi::{LlvmType, FnTypeLlvmExt}; use syntax::ast; use rustc::ty::Ty; use rustc::ty::layout::{self, Align, Size, TyLayout}; diff --git a/src/librustc_codegen_llvm/type_of.rs b/src/librustc_codegen_llvm/type_of.rs index ff25ed9256613..800bf505125d6 100644 --- a/src/librustc_codegen_llvm/type_of.rs +++ b/src/librustc_codegen_llvm/type_of.rs @@ -1,8 +1,8 @@ -use crate::abi::{FnType, FnTypeExt}; +use crate::abi::{FnType}; use crate::common::*; use crate::type_::Type; use rustc::ty::{self, Ty, TypeFoldable}; -use rustc::ty::layout::{self, Align, LayoutOf, PointeeInfo, Size, TyLayout}; +use rustc::ty::layout::{self, Align, LayoutOf, FnTypeExt, PointeeInfo, Size, TyLayout}; use rustc_target::abi::{FloatTy, TyLayoutMethods}; use rustc_mir::monomorphize::item::DefPathBasedNames; use rustc_codegen_ssa::traits::*; diff --git a/src/librustc_codegen_ssa/mir/block.rs b/src/librustc_codegen_ssa/mir/block.rs index e64c847db651b..96b8558c1d2c4 100644 --- a/src/librustc_codegen_ssa/mir/block.rs +++ b/src/librustc_codegen_ssa/mir/block.rs @@ -1,6 +1,6 @@ use rustc::middle::lang_items; use rustc::ty::{self, Ty, TypeFoldable}; -use rustc::ty::layout::{self, LayoutOf, HasTyCtxt}; +use rustc::ty::layout::{self, LayoutOf, HasTyCtxt, FnTypeExt}; use rustc::mir::{self, Place, PlaceBase, Static, StaticKind}; use rustc::mir::interpret::InterpError; use rustc_target::abi::call::{ArgType, FnType, PassMode, IgnoreMode}; @@ -334,14 +334,14 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { ty::ParamEnv::reveal_all(), &sig, ); - let fn_ty = bx.new_vtable(sig, &[]); + let fn_ty = FnType::new_vtable(&bx, sig, &[]); let vtable = args[1]; args = &args[..1]; (meth::DESTRUCTOR.get_fn(&mut bx, vtable, &fn_ty), fn_ty) } _ => { (bx.get_fn(drop_fn), - bx.fn_type_of_instance(&drop_fn)) + FnType::of_instance(&bx, &drop_fn)) } }; helper.do_call(self, &mut bx, fn_ty, drop_fn, args, @@ -439,7 +439,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { // Obtain the panic entry point. let def_id = common::langcall(bx.tcx(), Some(span), "", lang_item); let instance = ty::Instance::mono(bx.tcx(), def_id); - let fn_ty = bx.fn_type_of_instance(&instance); + let fn_ty = FnType::of_instance(&bx, &instance); let llfn = bx.get_fn(instance); // Codegen the actual panic invoke/call. @@ -518,7 +518,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let fn_ty = match def { Some(ty::InstanceDef::Virtual(..)) => { - bx.new_vtable(sig, &extra_args) + FnType::new_vtable(&bx, sig, &extra_args) } Some(ty::InstanceDef::DropGlue(_, None)) => { // Empty drop glue; a no-op. @@ -526,7 +526,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { helper.funclet_br(self, &mut bx, target); return; } - _ => bx.new_fn_type(sig, &extra_args) + _ => FnType::new(&bx, sig, &extra_args) }; // Emit a panic or a no-op for `panic_if_uninhabited`. @@ -556,7 +556,7 @@ impl<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { let def_id = common::langcall(bx.tcx(), Some(span), "", lang_items::PanicFnLangItem); let instance = ty::Instance::mono(bx.tcx(), def_id); - let fn_ty = bx.fn_type_of_instance(&instance); + let fn_ty = FnType::of_instance(&bx, &instance); let llfn = bx.get_fn(instance); // Codegen the actual panic invoke/call. diff --git a/src/librustc_codegen_ssa/mir/mod.rs b/src/librustc_codegen_ssa/mir/mod.rs index fcf099235aa14..060d7d18625fc 100644 --- a/src/librustc_codegen_ssa/mir/mod.rs +++ b/src/librustc_codegen_ssa/mir/mod.rs @@ -1,5 +1,5 @@ use rustc::ty::{self, Ty, TypeFoldable, UpvarSubsts}; -use rustc::ty::layout::{TyLayout, HasTyCtxt}; +use rustc::ty::layout::{TyLayout, HasTyCtxt, FnTypeExt}; use rustc::mir::{self, Mir}; use rustc::session::config::DebugInfo; use rustc_mir::monomorphize::Instance; @@ -202,7 +202,7 @@ pub fn codegen_mir<'a, 'tcx: 'a, Bx: BuilderMethods<'a, 'tcx>>( ) { assert!(!instance.substs.needs_infer()); - let fn_ty = cx.new_fn_type(sig, &[]); + let fn_ty = FnType::new(cx, sig, &[]); debug!("fn_ty: {:?}", fn_ty); let mut debug_context = cx.create_function_debug_context(instance, sig, llfn, mir); diff --git a/src/librustc_codegen_ssa/traits/abi.rs b/src/librustc_codegen_ssa/traits/abi.rs index a8fd4e1d2c7c7..509255c37be70 100644 --- a/src/librustc_codegen_ssa/traits/abi.rs +++ b/src/librustc_codegen_ssa/traits/abi.rs @@ -1,13 +1,7 @@ use super::BackendTypes; -use rustc::ty::{FnSig, Instance, Ty}; +use rustc::ty::{Ty}; use rustc_target::abi::call::FnType; -pub trait AbiMethods<'tcx> { - fn new_fn_type(&self, sig: FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> FnType<'tcx, Ty<'tcx>>; - fn new_vtable(&self, sig: FnSig<'tcx>, extra_args: &[Ty<'tcx>]) -> FnType<'tcx, Ty<'tcx>>; - fn fn_type_of_instance(&self, instance: &Instance<'tcx>) -> FnType<'tcx, Ty<'tcx>>; -} - pub trait AbiBuilderMethods<'tcx>: BackendTypes { fn apply_attrs_callsite(&mut self, ty: &FnType<'tcx, Ty<'tcx>>, callsite: Self::Value); fn get_param(&self, index: usize) -> Self::Value; diff --git a/src/librustc_codegen_ssa/traits/builder.rs b/src/librustc_codegen_ssa/traits/builder.rs index a3f99cd869e28..0c4c4547a7955 100644 --- a/src/librustc_codegen_ssa/traits/builder.rs +++ b/src/librustc_codegen_ssa/traits/builder.rs @@ -11,6 +11,7 @@ use crate::mir::place::PlaceRef; use crate::MemFlags; use rustc::ty::Ty; use rustc::ty::layout::{Align, Size, HasParamEnv}; +use rustc_target::spec::{HasTargetSpec}; use std::ops::Range; use std::iter::TrustedLen; @@ -30,6 +31,7 @@ pub trait BuilderMethods<'a, 'tcx: 'a>: + AsmBuilderMethods<'tcx> + StaticBuilderMethods<'tcx> + HasParamEnv<'tcx> + + HasTargetSpec { fn new_block<'b>(cx: &'a Self::CodegenCx, llfn: Self::Value, name: &'b str) -> Self; diff --git a/src/librustc_codegen_ssa/traits/mod.rs b/src/librustc_codegen_ssa/traits/mod.rs index c237cd8bd2645..2bb619e79f5e0 100644 --- a/src/librustc_codegen_ssa/traits/mod.rs +++ b/src/librustc_codegen_ssa/traits/mod.rs @@ -27,7 +27,7 @@ mod statics; mod type_; mod write; -pub use self::abi::{AbiBuilderMethods, AbiMethods}; +pub use self::abi::{AbiBuilderMethods}; pub use self::asm::{AsmBuilderMethods, AsmMethods}; pub use self::backend::{Backend, BackendTypes, ExtraBackendMethods}; pub use self::builder::{BuilderMethods, OverflowOp}; @@ -41,7 +41,8 @@ pub use self::type_::{ ArgTypeMethods, BaseTypeMethods, DerivedTypeMethods, LayoutTypeMethods, TypeMethods, }; pub use self::write::{ModuleBufferMethods, ThinBufferMethods, WriteBackendMethods}; -use rustc::ty::layout::{HasParamEnv}; +use rustc::ty::layout::{HasParamEnv, HasTyCtxt}; +use rustc_target::spec::{HasTargetSpec}; use std::fmt; @@ -56,11 +57,12 @@ pub trait CodegenMethods<'tcx>: + ConstMethods<'tcx> + StaticMethods + DebugInfoMethods<'tcx> - + AbiMethods<'tcx> + DeclareMethods<'tcx> + AsmMethods<'tcx> + PreDefineMethods<'tcx> + HasParamEnv<'tcx> + + HasTyCtxt<'tcx> + + HasTargetSpec { } @@ -71,11 +73,12 @@ impl<'tcx, T> CodegenMethods<'tcx> for T where + ConstMethods<'tcx> + StaticMethods + DebugInfoMethods<'tcx> - + AbiMethods<'tcx> + DeclareMethods<'tcx> + AsmMethods<'tcx> + PreDefineMethods<'tcx> + HasParamEnv<'tcx> + + HasTyCtxt<'tcx> + + HasTargetSpec { }