diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 76354362d57d..2a94e81cbf9e 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -83,6 +83,7 @@ enum class ReadyToRunSectionType : uint32_t OwnerCompositeExecutable = 116, // Added in V4.1 PgoInstrumentationData = 117, // Added in V5.2 ManifestAssemblyMvids = 118, // Added in V5.3 + Scratch = 119, // This is meant to be a scratch area just for prototyping // If you add a new section consider whether it is a breaking or non-breaking change. // Usually it is non-breaking, but if it is preferable to have older runtimes fail diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs index 30d8af4efb23..2c2ae5779f85 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs @@ -67,6 +67,7 @@ public enum ReadyToRunSectionType OwnerCompositeExecutable = 116, // Added in 4.1 PgoInstrumentationData = 117, // Added in 5.2 ManifestAssemblyMvids = 118, // Added in 5.3 + Scratch = 119, // This is meant to be a scratch area for prototyping only // // NativeAOT ReadyToRun sections diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 3c1eb891ac4f..016833e1f0c9 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -30,6 +30,7 @@ #if READYTORUN using System.Reflection.Metadata.Ecma335; using ILCompiler.DependencyAnalysis.ReadyToRun; +using Internal.Text; #endif namespace Internal.JitInterface @@ -422,6 +423,20 @@ private void PublishCode() , isFoldable: (_compilation._compilationOptions & RyuJitCompilationOptions.MethodBodyFolding) != 0 #endif ); +#if READYTORUN + if (_methodColdCodeNode != null) + { + // TODO, Study how this work. + var relocs2 = _coldCodeRelocs.ToArray(); + Array.Sort(relocs2, (x, y) => (x.Offset - y.Offset)); + var coldObjectData = new ObjectNode.ObjectData(_coldCode, + relocs2, + alignment, + new ISymbolDefinitionNode[] { _methodColdCodeNode }); + _methodColdCodeNode.SetCode(coldObjectData); + _methodCodeNode.SetColdCodeNode(_methodColdCodeNode); + } +#endif _methodCodeNode.InitializeFrameInfos(_frameInfos); _methodCodeNode.InitializeDebugEHClauseInfos(debugEHClauseInfos); @@ -563,7 +578,9 @@ private void CompileMethodCleanup() } _methodCodeNode = null; - +#if READYTORUN + _methodColdCodeNode = null; +#endif _code = null; _coldCode = null; @@ -572,7 +589,9 @@ private void CompileMethodCleanup() _codeRelocs = new ArrayBuilder(); _roDataRelocs = new ArrayBuilder(); - +#if READYTORUN + _coldCodeRelocs = new ArrayBuilder(); +#endif _numFrameInfos = 0; _usedFrameInfos = 0; _frameInfos = null; @@ -3396,6 +3415,10 @@ private void allocMem(ref AllocMemArgs args) if (args.coldCodeSize != 0) { + +#if READYTORUN + this._methodColdCodeNode = new MethodColdCodeNode(MethodBeingCompiled); +#endif args.coldCodeBlock = (void*)GetPin(_coldCode = new byte[args.coldCodeSize]); args.coldCodeBlockRW = args.coldCodeBlock; } @@ -3443,7 +3466,10 @@ private void allocMem(ref AllocMemArgs args) private void reserveUnwindInfo(bool isFunclet, bool isColdCode, uint unwindSize) { - _numFrameInfos++; + if (!isColdCode) + { + _numFrameInfos++; + } } private void allocUnwindInfo(byte* pHotCode, byte* pColdCode, uint startOffset, uint endOffset, uint unwindSize, byte* pUnwindBlock, CorJitFuncKind funcKind) @@ -3474,8 +3500,12 @@ private void allocUnwindInfo(byte* pHotCode, byte* pColdCode, uint startOffset, blobData = CompressARM64CFI(blobData); } #endif - - _frameInfos[_usedFrameInfos++] = new FrameInfo(flags, (int)startOffset, (int)endOffset, blobData); +#if READYTORUN + if (blobData.Length > 0) +#endif + { + _frameInfos[_usedFrameInfos++] = new FrameInfo(flags, (int)startOffset, (int)endOffset, blobData); + } } private void* allocGCInfo(UIntPtr size) @@ -3510,7 +3540,9 @@ private void recordCallSite(uint instrOffset, CORINFO_SIG_INFO* callSig, CORINFO private ArrayBuilder _codeRelocs; private ArrayBuilder _roDataRelocs; - +#if READYTORUN + private ArrayBuilder _coldCodeRelocs; +#endif /// /// Various type of block. @@ -3588,6 +3620,11 @@ private ref ArrayBuilder findRelocBlock(BlockType blockType, out int case BlockType.ROData: length = _roData.Length; return ref _roDataRelocs; +#if READYTORUN + case BlockType.ColdCode: + length = _coldCode.Length; + return ref _coldCodeRelocs; +#endif default: throw new NotImplementedException("Arbitrary relocs"); } @@ -3640,8 +3677,13 @@ private void recordRelocation(void* location, void* locationRW, void* target, us break; case BlockType.ColdCode: - // TODO: Arbitrary relocs +#if READYTORUN + Debug.Assert(_methodColdCodeNode != null); + relocTarget = _methodColdCodeNode; + break; +#else throw new NotImplementedException("ColdCode relocs"); +#endif case BlockType.ROData: relocTarget = _roDataBlob; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ObjectWriter.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ObjectWriter.cs index 5f195f345747..a2119383e743 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ObjectWriter.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/ObjectWriter.cs @@ -605,6 +605,7 @@ public void BuildCFIMap(NodeFactory factory, ObjectNode node) int start = frameInfo.StartOffset; int end = frameInfo.EndOffset; + int len = frameInfo.BlobData.Length; byte[] blob = frameInfo.BlobData; diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodColdCodeNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodColdCodeNode.cs new file mode 100644 index 000000000000..aadbab9d0ea4 --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodColdCodeNode.cs @@ -0,0 +1,84 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Reflection; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; +using System.Runtime.InteropServices; + +using Internal.IL; +using Internal.IL.Stubs; +using Internal.Text; +using Internal.TypeSystem; +using Internal.TypeSystem.Ecma; +using Internal.TypeSystem.Interop; +using Internal.CorConstants; +using Internal.Pgo; +using Internal.ReadyToRunConstants; + +using ILCompiler; +using ILCompiler.DependencyAnalysis; +using ILCompiler.DependencyAnalysis.ReadyToRun; +using System.Text; + +namespace ILCompiler.DependencyAnalysis.ReadyToRun +{ + public class MethodColdCodeNode : ObjectNode, ISymbolDefinitionNode + { + private ObjectData _methodColdCode; + private MethodDesc _owningMethod; + + public MethodColdCodeNode(MethodDesc owningMethod) + { + _owningMethod = owningMethod; + } + + public int Offset => 0; + + public override ObjectNodeSection Section + { + get + { + // TODO, Unix + return ObjectNodeSection.ManagedCodeWindowsContentSection; + } + } + + public override bool IsShareable => false; + + // This ClassCode must be larger than that of MethodCodeNode to ensure it got sorted at the end of the code + public override int ClassCode => 788492408; + + public override bool StaticDependenciesAreComputed => _methodColdCode != null; + + public void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) + { + sb.Append("__coldcode_" + nameMangler.GetMangledMethodName(_owningMethod)); + } + + public override int CompareToImpl(ISortableNode other, CompilerComparer comparer) + { + MethodColdCodeNode otherNode = (MethodColdCodeNode)other; + return comparer.Compare(_owningMethod, otherNode._owningMethod); + } + + public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) => _methodColdCode; + + protected override string GetName(NodeFactory context) => throw new NotImplementedException(); + + public void SetCode(ObjectData data) + { + Debug.Assert(_methodColdCode == null); + _methodColdCode = data; + } + + public int GetColdCodeSize() + { + return _methodColdCode.Data.Length; + } + } +} \ No newline at end of file diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs index a43ad0335ccc..3d15125143e3 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodGCInfoNode.cs @@ -39,7 +39,12 @@ protected override void OnMarked(NodeFactory factory) public int[] CalculateFuncletOffsets(NodeFactory factory) { - int[] offsets = new int[_methodNode.FrameInfos.Length]; + int coldCodeUnwindInfoCount = 0; + if (_methodNode.GetColdCodeNode() != null) + { + coldCodeUnwindInfoCount = 1; + } + int[] offsets = new int[_methodNode.FrameInfos.Length + coldCodeUnwindInfoCount]; if (!factory.RuntimeFunctionsGCInfo.Deduplicator.TryGetValue(this, out var deduplicatedResult)) { throw new Exception("Did not properly initialize deduplicator"); @@ -60,6 +65,10 @@ public int[] CalculateFuncletOffsets(NodeFactory factory) offset += (-offset & 3); // 4-alignment after GC info in 1st funclet } } + if (coldCodeUnwindInfoCount == 1) + { + offsets[_methodNode.FrameInfos.Length] = offset; + } return offsets; } @@ -164,6 +173,22 @@ private IEnumerable EncodeDataCore(NodeFactory factory) yield return new GCInfoComponent(_methodNode.GCInfo); } } +#if READYTORUN + if (_methodNode.GetColdCodeNode() != null) + { + byte[] header = new byte[4]; + int i = 0; + header[i++] = 1 + (4 << 3); // Version = 1, UNW_FLAG_CHAININFO + header[i++] = 0; // SizeOfProlog = 0 + header[i++] = 0; // CountOfCode = 0 + header[i++] = 0; // Frame = 0 + yield return new GCInfoComponent(header); + yield return new GCInfoComponent(_methodNode, 0); + yield return new GCInfoComponent(_methodNode, _methodNode.Size); + // TODO: Is this correct? + yield return new GCInfoComponent(factory.RuntimeFunctionsGCInfo.StartSymbol, this.OffsetFromBeginningOfArray); + } +#endif } class MethodGCInfoNodeDeduplicatingComparer : IEqualityComparer diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodWithGCInfo.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodWithGCInfo.cs index cad838bfe7b1..a4d67cf21513 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodWithGCInfo.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/MethodWithGCInfo.cs @@ -4,7 +4,8 @@ using System; using System.Collections.Generic; using System.Diagnostics; - +using System.Linq; +using System.Reflection.Metadata.Ecma335; using Internal.JitInterface; using Internal.Text; using Internal.TypeSystem; @@ -19,6 +20,9 @@ public class MethodWithGCInfo : ObjectNode, IMethodBodyNode, ISymbolDefinitionNo private readonly MethodDesc _method; private ObjectData _methodCode; +#if READYTORUN + private MethodColdCodeNode _methodColdCodeNode; +#endif private FrameInfo[] _frameInfos; private byte[] _gcInfo; private ObjectData _ehInfo; @@ -129,11 +133,27 @@ public int Compare(FixupCell a, FixupCell b) } } + public MethodColdCodeNode GetColdCodeNode() => _methodColdCodeNode; public byte[] GetFixupBlob(NodeFactory factory) { Relocation[] relocations = GetData(factory, relocsOnly: true).Relocs; +#if READYTORUN + if (_methodColdCodeNode != null) + { + Relocation[] coldRelocations = _methodColdCodeNode.GetData(factory, relocsOnly: true).Relocs; + if (relocations == null) + { + relocations = coldRelocations; + } + else if (coldRelocations != null) + { + relocations = Enumerable.Concat(relocations, coldRelocations).ToArray(); + } + } +#endif + if (relocations == null) { return null; @@ -249,6 +269,10 @@ protected override DependencyList ComputeNonRelocationBasedDependencies(NodeFact { dependencyList.Add(node, "classMustBeLoadedBeforeCodeIsRun"); } + if (_methodColdCodeNode != null) + { + dependencyList.Add(_methodColdCodeNode, "cold"); + } return dependencyList; } @@ -358,5 +382,12 @@ public void InitializeInliningInfo(MethodDesc[] inlinedMethods, NodeFactory fact public override bool ShouldSkipEmittingObjectNode(NodeFactory factory) => IsEmpty; public override string ToString() => _method.ToString(); + +#if READYTORUN + public void SetColdCodeNode(MethodColdCodeNode methodColdCodeNode) + { + _methodColdCodeNode = methodColdCodeNode; + } +#endif } } diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs index f2d72fd9d4d2..e486b59229dd 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/RuntimeFunctionsTableNode.cs @@ -60,6 +60,7 @@ private void LayoutRuntimeFunctions() public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) { // This node does not trigger generation of other nodes. + // TODO: Make this generate the generation of the Scratch node if (relocsOnly) return new ObjectData(Array.Empty(), Array.Empty(), 1, new ISymbolDefinitionNode[] { this }); @@ -72,6 +73,7 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) // Add the symbol representing this object node runtimeFunctionsBuilder.AddSymbol(this); + uint runtimeFunctionIndex = 0; foreach (MethodWithGCInfo method in _methodNodes) { int[] funcletOffsets = method.GCInfoNode.CalculateFuncletOffsets(factory); @@ -94,9 +96,42 @@ public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) runtimeFunctionsBuilder.EmitReloc(method, RelocType.IMAGE_REL_BASED_ADDR32NB, delta: frameInfo.EndOffset); } runtimeFunctionsBuilder.EmitReloc(factory.RuntimeFunctionsGCInfo.StartSymbol, RelocType.IMAGE_REL_BASED_ADDR32NB, funcletOffsets[frameIndex]); + runtimeFunctionIndex++; } } + List mapping = new List(); +#if READYTORUN + // Emitting a RuntimeFunction entry for cold code + foreach (MethodWithGCInfo method in _methodNodes) + { + MethodColdCodeNode methodColdCodeNode = method.GetColdCodeNode(); + if (methodColdCodeNode != null) + { + int[] funcletOffsets = method.GCInfoNode.CalculateFuncletOffsets(factory); + // TODO: Avoid code duplication + // StartOffset of the runtime function + int codeDelta = 0; + if (Target.Architecture == TargetArchitecture.ARM) + { + // THUMB_CODE + codeDelta = 1; + } + runtimeFunctionsBuilder.EmitReloc(methodColdCodeNode, RelocType.IMAGE_REL_BASED_ADDR32NB, delta: codeDelta); + if (!relocsOnly && Target.Architecture == TargetArchitecture.X64) + { + // On Amd64, the 2nd word contains the EndOffset of the runtime function + runtimeFunctionsBuilder.EmitReloc(methodColdCodeNode, RelocType.IMAGE_REL_BASED_ADDR32NB, delta: methodColdCodeNode.GetColdCodeSize()); + } + runtimeFunctionsBuilder.EmitReloc(factory.RuntimeFunctionsGCInfo.StartSymbol, RelocType.IMAGE_REL_BASED_ADDR32NB, funcletOffsets[funcletOffsets.Length - 1]); + mapping.Add(runtimeFunctionIndex); + mapping.Add((uint)_insertedMethodNodes[method]); + runtimeFunctionIndex++; + } + } +#endif + _nodeFactory.Scratch.mapping = mapping.ToArray(); + // Emit sentinel entry runtimeFunctionsBuilder.EmitUInt(~0u); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ScratchNode.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ScratchNode.cs new file mode 100644 index 000000000000..02218477bdc5 --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRun/ScratchNode.cs @@ -0,0 +1,47 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; + +using Internal.Text; +using Internal.TypeSystem; + +using Debug = System.Diagnostics.Debug; + +namespace ILCompiler.DependencyAnalysis.ReadyToRun +{ + public class ScratchNode : HeaderTableNode + { + public uint[] mapping; + + public ScratchNode(NodeFactory nodeFactory) + : base(nodeFactory.Target) + { + // _nodeFactory = nodeFactory; + } + + public override int ClassCode => 28963035; + + public override void AppendMangledName(NameMangler nameMangler, Utf8StringBuilder sb) + { + sb.Append(nameMangler.CompilationUnitPrefix); + sb.Append("__Scratch"); + } + + public override ObjectData GetData(NodeFactory factory, bool relocsOnly = false) + { + // This node does not trigger generation of other nodes. + if (relocsOnly) + return new ObjectData(Array.Empty(), Array.Empty(), 1, new ISymbolDefinitionNode[] { this }); + + ObjectDataBuilder builder = new ObjectDataBuilder(factory, relocsOnly); + builder.AddSymbol(this); + foreach (uint m in this.mapping) + { + builder.EmitUInt(m); + } + return builder.ToObjectData(); + } + } +} diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs index 44925d84cfe2..401fffe8b213 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/DependencyAnalysis/ReadyToRunCodegenNodeFactory.cs @@ -319,6 +319,8 @@ private void CreateNodeCaches() public RuntimeFunctionsTableNode RuntimeFunctionsTable; + public ScratchNode Scratch; + public RuntimeFunctionsGCInfoNode RuntimeFunctionsGCInfo; public DelayLoadMethodCallThunkNodeRange DelayLoadMethodCallThunks; @@ -609,6 +611,11 @@ public void AttachToDependencyGraph(DependencyAnalyzerBase graph) RuntimeFunctionsTable = new RuntimeFunctionsTableNode(this); Header.Add(Internal.Runtime.ReadyToRunSectionType.RuntimeFunctions, RuntimeFunctionsTable, RuntimeFunctionsTable); + Scratch = new ScratchNode(this); + Header.Add(Internal.Runtime.ReadyToRunSectionType.Scratch, Scratch, Scratch); + // TODO, this should be dependent on the existence of cold code blocks. + graph.AddRoot(Scratch, "Scratch is always there!"); + RuntimeFunctionsGCInfo = new RuntimeFunctionsGCInfoNode(); graph.AddRoot(RuntimeFunctionsGCInfo, "GC info is always generated"); diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs index 03a583173545..4b1059b3f78e 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/Compiler/ReadyToRunCodegenCompilationBuilder.cs @@ -38,6 +38,7 @@ public sealed class ReadyToRunCodegenCompilationBuilder : CompilationBuilder private ReadyToRunFileLayoutAlgorithm _r2rFileLayoutAlgorithm; private int _customPESectionAlignment; private bool _verifyTypeAndFieldLayout; + private bool _hotColdSplitting; private CompositeImageSettings _compositeImageSettings; private ulong _imageBase; @@ -186,6 +187,12 @@ public ReadyToRunCodegenCompilationBuilder UseVerifyTypeAndFieldLayout(bool veri return this; } + public ReadyToRunCodegenCompilationBuilder UseHotColdSplitting(bool hotColdSplitting) + { + _hotColdSplitting = hotColdSplitting; + return this; + } + public ReadyToRunCodegenCompilationBuilder UseCompositeImageSettings(CompositeImageSettings compositeImageSettings) { _compositeImageSettings = compositeImageSettings; @@ -249,7 +256,12 @@ public override ICompilation ToCompilation() IComparer> comparer = new SortableDependencyNode.ObjectNodeComparer(CompilerComparer.Instance); DependencyAnalyzerBase graph = CreateDependencyGraph(factory, comparer); + List corJitFlags = new List { CorJitFlag.CORJIT_FLAG_DEBUG_INFO }; + if (_hotColdSplitting) + { + corJitFlags.Add(CorJitFlag.CORJIT_FLAG_PROCSPLIT); + } switch (_optimizationMode) { diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj index 7f31574bcf6f..5c63065d05a1 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/ILCompiler.ReadyToRun.csproj @@ -155,6 +155,7 @@ + @@ -169,6 +170,7 @@ + diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs index 6e62f81bdbf3..d2a50515b0d6 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs @@ -359,6 +359,7 @@ unsafe partial class CorInfoImpl private readonly ReadyToRunCodegenCompilation _compilation; private MethodWithGCInfo _methodCodeNode; + private MethodColdCodeNode _methodColdCodeNode; private OffsetMapping[] _debugLocInfos; private NativeVarInfo[] _debugVarInfos; private HashSet _inlinedMethods; diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/Amd64/UnwindInfo.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/Amd64/UnwindInfo.cs index b4338ffef290..eb703e0ca9cb 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/Amd64/UnwindInfo.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/Amd64/UnwindInfo.cs @@ -59,7 +59,7 @@ public class UnwindCode public UnwindCode() { } /// - /// Unwinde code parsing is based on src\jit\unwindamd64.cpp DumpUnwindInfo + /// Unwind code parsing is based on src\jit\unwindamd64.cpp DumpUnwindInfo /// public UnwindCode(byte[] image, ref int frameOffset, ref int offset) { diff --git a/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs b/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs index 9e0a4ba94591..dfd69e611b09 100644 --- a/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs +++ b/src/coreclr/tools/aot/crossgen2/CommandLineOptions.cs @@ -63,6 +63,7 @@ internal class CommandLineOptions public string MethodLayout; public string FileLayout; public bool VerifyTypeAndFieldLayout; + public bool HotColdSplitting; public string CallChainProfileFile; public string ImageBase; @@ -167,6 +168,7 @@ public CommandLineOptions(string[] args) syntax.DefineOption("method-layout", ref MethodLayout, SR.MethodLayoutOption); syntax.DefineOption("file-layout", ref FileLayout, SR.FileLayoutOption); syntax.DefineOption("verify-type-and-field-layout", ref VerifyTypeAndFieldLayout, SR.VerifyTypeAndFieldLayoutOption); + syntax.DefineOption("hot-cold-splitting", ref HotColdSplitting, SR.HotColdSplittingOption); syntax.DefineOption("callchain-profile", ref CallChainProfileFile, SR.CallChainProfileFile); syntax.DefineOption("make-repro-path", ref MakeReproPath, SR.MakeReproPathHelp); diff --git a/src/coreclr/tools/aot/crossgen2/Program.cs b/src/coreclr/tools/aot/crossgen2/Program.cs index 8667b0bb1eec..5fb6624ea666 100644 --- a/src/coreclr/tools/aot/crossgen2/Program.cs +++ b/src/coreclr/tools/aot/crossgen2/Program.cs @@ -747,6 +747,7 @@ private void RunSingleCompilation(Dictionary inFilePaths, Instru .UseInstructionSetSupport(instructionSetSupport) .UseCustomPESectionAlignment(_commandLineOptions.CustomPESectionAlignment) .UseVerifyTypeAndFieldLayout(_commandLineOptions.VerifyTypeAndFieldLayout) + .UseHotColdSplitting(_commandLineOptions.HotColdSplitting) .GenerateOutputFile(outFile) .UseImageBase(_imageBase) .UseILProvider(ilProvider) diff --git a/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx b/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx index e5deb22319cd..5cbcf31f140b 100644 --- a/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx +++ b/src/coreclr/tools/aot/crossgen2/Properties/Resources.resx @@ -324,6 +324,9 @@ Verify that struct type layout and field offsets match between compile time and runtime. Use only for diagnostic purposes. + + Turn on hot cold splitting optimization. + Generate a CSV formatted map file diff --git a/src/coreclr/vm/codeman.cpp b/src/coreclr/vm/codeman.cpp index 37502f21ef1d..3b4d8577a41a 100644 --- a/src/coreclr/vm/codeman.cpp +++ b/src/coreclr/vm/codeman.cpp @@ -5830,7 +5830,7 @@ GCInfoToken ReadyToRunJitManager::GetGCInfoToken(const METHODTOKEN& MethodToken) SIZE_T nUnwindDataSize; PTR_VOID pUnwindData = GetUnwindDataBlob(baseAddress, pRuntimeFunction, &nUnwindDataSize); - // GCInfo immediatelly follows unwind data + // GCInfo immediately follows unwind data PTR_BYTE gcInfo = dac_cast(pUnwindData) + nUnwindDataSize; UINT32 gcInfoVersion = JitTokenToGCInfoVersion(MethodToken); @@ -6073,8 +6073,6 @@ BOOL ReadyToRunJitManager::JitCodeToMethodInfo(RangeSection * pRangeSection, SUPPORTS_DAC; } CONTRACTL_END; - // READYTORUN: FUTURE: Hot-cold spliting - // If the address is in a thunk, return NULL. if (GetStubCodeBlockKind(pRangeSection, currentPC) != STUB_CODE_BLOCK_UNKNOWN) { @@ -6113,6 +6111,19 @@ BOOL ReadyToRunJitManager::JitCodeToMethodInfo(RangeSection * pRangeSection, // Save the raw entry PTR_RUNTIME_FUNCTION RawFunctionEntry = pRuntimeFunctions + MethodIndex; + // If the MethodIndex happen to be the cold code block, turn it into the associated hot code block + for (DWORD i = 0; i < pInfo->m_nScratch; i++) + { + if ((ULONG)MethodIndex == pInfo->m_pScratch[i]) + { + if (i % 2 == 0) + { + MethodIndex = pInfo->m_pScratch[i + 1]; + break; + } + } + } + MethodDesc *pMethodDesc; while ((pMethodDesc = pInfo->GetMethodDescForEntryPoint(ImageBase + RUNTIME_FUNCTION__BeginAddress(pRuntimeFunctions + MethodIndex))) == NULL) MethodIndex--; @@ -6132,6 +6143,8 @@ BOOL ReadyToRunJitManager::JitCodeToMethodInfo(RangeSection * pRangeSection, if (pCodeInfo) { + // Should EECodeInfo::GetRelOffset() respect hot-cold splitting? + // Certain debugger code path assume this respects hot-cold splitting, but others might not. pCodeInfo->m_relOffset = (DWORD) (RelativePc - RUNTIME_FUNCTION__BeginAddress(FunctionEntry)); @@ -6171,8 +6184,6 @@ TADDR ReadyToRunJitManager::GetFuncletStartAddress(EECodeInfo * pCodeInfo) { LIMITED_METHOD_DAC_CONTRACT; - // READYTORUN: FUTURE: Hot-cold spliting - return IJitManager::GetFuncletStartAddress(pCodeInfo); } @@ -6194,11 +6205,64 @@ DWORD ReadyToRunJitManager::GetFuncletStartOffsets(const METHODTOKEN& MethodToke pFirstFuncletFunctionEntry, moduleBase, &nFunclets, pStartFuncletOffsets, dwLength); - // READYTORUN: FUTURE: Hot/cold splitting + // Technically, the cold code is not a funclet, but it looks like the debugger wants it + if (regionInfo.coldSize > 0) + { + ReadyToRunInfo * pInfo = JitTokenToReadyToRunInfo(MethodToken); + PTR_RUNTIME_FUNCTION pRuntimeFunctions = pInfo->m_pRuntimeFunctions; + int i = 0; + while (true) + { + pFirstFuncletFunctionEntry = pRuntimeFunctions + i; + if (regionInfo.coldStartAddress == moduleBase + RUNTIME_FUNCTION__BeginAddress(pFirstFuncletFunctionEntry)) + { + break; + } + i++; + } + + GetFuncletStartOffsetsHelper(regionInfo.coldStartAddress, regionInfo.coldSize, 0, + pFirstFuncletFunctionEntry, moduleBase, + &nFunclets, pStartFuncletOffsets, dwLength); + } return nFunclets; } +BOOL ReadyToRunJitManager::IsFunclet(EECodeInfo* pCodeInfo) +{ + CONTRACTL { + NOTHROW; + GC_NOTRIGGER; + HOST_NOCALLS; + SUPPORTS_DAC; + } CONTRACTL_END; + + ReadyToRunInfo * pInfo = JitTokenToReadyToRunInfo(pCodeInfo->GetMethodToken()); + + COUNT_T nRuntimeFunctions = pInfo->m_nRuntimeFunctions; + PTR_RUNTIME_FUNCTION pRuntimeFunctions = pInfo->m_pRuntimeFunctions; + + ULONG methodIndex = (ULONG)(pCodeInfo->GetFunctionEntry() - pRuntimeFunctions); + + // If it is either the main hot-code or the cold code, then it is not a funclet + for (DWORD i = 0; i < pInfo->m_nScratch; i++) + { + if (methodIndex == pInfo->m_pScratch[i]) + { + return FALSE; + } + } + + // It could be a funclet, or it could be a function that is not split + // so fall back to existing logic + + TADDR funcletStartAddress = GetFuncletStartAddress(pCodeInfo); + TADDR methodStartAddress = pCodeInfo->GetStartAddress(); + + return (funcletStartAddress != methodStartAddress); +} + BOOL ReadyToRunJitManager::IsFilterFunclet(EECodeInfo * pCodeInfo) { CONTRACTL { @@ -6250,12 +6314,32 @@ void ReadyToRunJitManager::JitTokenToMethodRegionInfo(const METHODTOKEN& MethodT PRECONDITION(methodRegionInfo != NULL); } CONTRACTL_END; - // READYTORUN: FUTURE: Hot-cold spliting - methodRegionInfo->hotStartAddress = JitTokenToStartAddress(MethodToken); methodRegionInfo->hotSize = GetCodeManager()->GetFunctionSize(GetGCInfoToken(MethodToken)); methodRegionInfo->coldStartAddress = 0; methodRegionInfo->coldSize = 0; + + ReadyToRunInfo * pInfo = JitTokenToReadyToRunInfo(MethodToken); + COUNT_T nRuntimeFunctions = pInfo->m_nRuntimeFunctions; + PTR_RUNTIME_FUNCTION pRuntimeFunctions = pInfo->m_pRuntimeFunctions; + + PTR_RUNTIME_FUNCTION pRuntimeFunction = dac_cast(MethodToken.m_pCodeHeader); + + ULONG methodIndex = (ULONG)(pRuntimeFunction - pRuntimeFunctions); + + for (DWORD i = 0; i < pInfo->m_nScratch; i++) + { + if (methodIndex == pInfo->m_pScratch[i]) + { + _ASSERTE((i % 2) == 1); + ULONG coldMethodIndex = pInfo->m_pScratch[i - 1]; + PTR_RUNTIME_FUNCTION pColdRuntimeFunction = pRuntimeFunctions + coldMethodIndex; + methodRegionInfo->coldStartAddress = JitTokenToModuleBase(MethodToken) + RUNTIME_FUNCTION__BeginAddress(pColdRuntimeFunction); + methodRegionInfo->coldSize = RUNTIME_FUNCTION__EndAddress(pColdRuntimeFunction, 0) - RUNTIME_FUNCTION__BeginAddress(pColdRuntimeFunction); + methodRegionInfo->hotSize -= methodRegionInfo->coldSize; + break; + } + } } #ifdef DACCESS_COMPILE diff --git a/src/coreclr/vm/codeman.h b/src/coreclr/vm/codeman.h index 00a39598fd0d..589b18065618 100644 --- a/src/coreclr/vm/codeman.h +++ b/src/coreclr/vm/codeman.h @@ -787,7 +787,7 @@ class IJitManager virtual DWORD GetFuncletStartOffsets(const METHODTOKEN& MethodToken, DWORD* pStartFuncletOffsets, DWORD dwLength) = 0; - BOOL IsFunclet(EECodeInfo * pCodeInfo); + virtual BOOL IsFunclet(EECodeInfo * pCodeInfo); virtual BOOL IsFilterFunclet(EECodeInfo * pCodeInfo); #endif // FEATURE_EH_FUNCLETS @@ -1559,6 +1559,7 @@ class ReadyToRunJitManager final: public IJitManager virtual TADDR GetFuncletStartAddress(EECodeInfo * pCodeInfo); virtual DWORD GetFuncletStartOffsets(const METHODTOKEN& MethodToken, DWORD* pStartFuncletOffsets, DWORD dwLength); + virtual BOOL IsFunclet(EECodeInfo * pCodeInfo); virtual BOOL IsFilterFunclet(EECodeInfo * pCodeInfo); #endif // FEATURE_EH_FUNCLETS diff --git a/src/coreclr/vm/debugdebugger.cpp b/src/coreclr/vm/debugdebugger.cpp index 46d5b81631c7..598a5a83611f 100644 --- a/src/coreclr/vm/debugdebugger.cpp +++ b/src/coreclr/vm/debugdebugger.cpp @@ -1154,7 +1154,11 @@ void DebugStackTrace::GetStackFramesFromException(OBJECTREF * e, if (cur.ip) { - dwNativeOffset = (DWORD)(cur.ip - (UINT_PTR)pMD->GetNativeCode()); + EECodeInfo codeInfo(cur.ip); + // As of now, EECodeInfo::GetRelOffset does not account for the gap + // between hot and cold code, so it is still wrong to get the IL offset + // using this value as the native offset. + dwNativeOffset = codeInfo.GetRelOffset(); } else { diff --git a/src/coreclr/vm/readytoruninfo.cpp b/src/coreclr/vm/readytoruninfo.cpp index d2228aeba4bb..5dc37075ce4d 100644 --- a/src/coreclr/vm/readytoruninfo.cpp +++ b/src/coreclr/vm/readytoruninfo.cpp @@ -646,6 +646,17 @@ ReadyToRunInfo::ReadyToRunInfo(Module * pModule, LoaderAllocator* pLoaderAllocat m_nRuntimeFunctions = 0; } + IMAGE_DATA_DIRECTORY * pScratchDir = m_pComposite->FindSection(ReadyToRunSectionType::Scratch); + if (pScratchDir != NULL) + { + m_pScratch = (PTR_ULONG)m_pComposite->GetLayout()->GetDirectoryData(pScratchDir); + m_nScratch = pScratchDir->Size / sizeof(ULONG); + } + else + { + m_nScratch = 0; + } + IMAGE_DATA_DIRECTORY * pImportSectionsDir = m_pComposite->FindSection(ReadyToRunSectionType::ImportSections); if (pImportSectionsDir != NULL) { diff --git a/src/coreclr/vm/readytoruninfo.h b/src/coreclr/vm/readytoruninfo.h index 0264629e5b82..c6bb158231d3 100644 --- a/src/coreclr/vm/readytoruninfo.h +++ b/src/coreclr/vm/readytoruninfo.h @@ -61,6 +61,9 @@ class ReadyToRunInfo PTR_RUNTIME_FUNCTION m_pRuntimeFunctions; DWORD m_nRuntimeFunctions; + PTR_ULONG m_pScratch; + DWORD m_nScratch; + PTR_IMAGE_DATA_DIRECTORY m_pSectionDelayLoadMethodCallThunks; PTR_READYTORUN_IMPORT_SECTION m_pImportSections;