diff --git a/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S b/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S index 2bab323e65abc..79ffed2b05210 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S +++ b/src/coreclr/nativeaot/Runtime/arm64/AllocFast.S @@ -46,7 +46,7 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit = OFFSETOF__Thread__m_rgbAll add x2, x2, x12 ldr x13, [x1, #OFFSETOF__Thread__m_alloc_context__alloc_limit] cmp x2, x13 - bhi RhpNewFast_RarePath + bhi LOCAL_LABEL(RhpNewFast_RarePath) // Update the alloc pointer to account for the allocation. str x2, [x1, #OFFSETOF__Thread__m_alloc_context__alloc_ptr] @@ -57,7 +57,7 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit = OFFSETOF__Thread__m_rgbAll mov x0, x12 ret -RhpNewFast_RarePath: +LOCAL_LABEL(RhpNewFast_RarePath): mov x1, #0 b C_FUNC(RhpNewObject) LEAF_END RhpNewFast, _TEXT @@ -88,12 +88,12 @@ RhpNewFast_RarePath: bl C_FUNC(RhpGcAlloc) // Set the new objects MethodTable pointer on success. - cbz x0, NewOutOfMemory + cbz x0, LOCAL_LABEL(NewOutOfMemory) POP_COOP_PINVOKE_FRAME EPILOG_RETURN -NewOutOfMemory: +LOCAL_LABEL(NewOutOfMemory): // This is the OOM failure path. We are going to tail-call to a managed helper that will throw // an out of memory exception that the caller of this allocator understands. @@ -113,7 +113,7 @@ NewOutOfMemory: movz x2, MAX_STRING_LENGTH & 0xFFFF movk x2, MAX_STRING_LENGTH >> 16, lsl 16 cmp x1, x2 - bhi StringSizeOverflow + bhi LOCAL_LABEL(StringSizeOverflow) // Compute overall allocation size (align(base size + (element size * elements), 8)). mov w2, #STRING_COMPONENT_SIZE @@ -139,7 +139,7 @@ NewOutOfMemory: add x2, x2, x12 ldr x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_limit] cmp x2, x12 - bhi C_FUNC(RhpNewArrayRare) + bhi LOCAL_LABEL(RhNewString_Rare) // Reload new object address into r12. ldr x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_ptr] @@ -156,7 +156,7 @@ NewOutOfMemory: ret -StringSizeOverflow: +LOCAL_LABEL(StringSizeOverflow): // We get here if the length of the final string object can not be represented as an unsigned // 32-bit value. We are going to tail-call to a managed helper that will throw // an OOM exception that the caller of this allocator understands. @@ -164,6 +164,9 @@ StringSizeOverflow: // x0 holds MethodTable pointer already mov x1, #1 // Indicate that we should throw OverflowException b C_FUNC(RhExceptionHandling_FailedAllocation) + +LOCAL_LABEL(RhNewString_Rare): + b C_FUNC(RhpNewArrayRare) LEAF_END RhNewString, _Text // Allocate one dimensional, zero based array (SZARRAY). @@ -177,7 +180,7 @@ StringSizeOverflow: // case (32 dimensional MdArray) is less than 0xffff, and thus the product fits in 64 bits. mov x2, #0x7FFFFFFF cmp x1, x2 - bhi ArraySizeOverflow + bhi LOCAL_LABEL(ArraySizeOverflow) ldrh w2, [x0, #OFFSETOF__MethodTable__m_usComponentSize] umull x2, w1, w2 @@ -204,7 +207,7 @@ StringSizeOverflow: add x2, x2, x12 ldr x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_limit] cmp x2, x12 - bhi C_FUNC(RhpNewArrayRare) + bhi LOCAL_LABEL(RhpNewArray_Rare) // Reload new object address into x12. ldr x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_ptr] @@ -221,7 +224,7 @@ StringSizeOverflow: ret -ArraySizeOverflow: +LOCAL_LABEL(ArraySizeOverflow): // We get here if the size of the final array object can not be represented as an unsigned // 32-bit value. We are going to tail-call to a managed helper that will throw // an overflow exception that the caller of this allocator understands. @@ -229,6 +232,9 @@ ArraySizeOverflow: // x0 holds MethodTable pointer already mov x1, #1 // Indicate that we should throw OverflowException b C_FUNC(RhExceptionHandling_FailedAllocation) + +LOCAL_LABEL(RhpNewArray_Rare): + b C_FUNC(RhpNewArrayRare) LEAF_END RhpNewArray, _TEXT // Allocate one dimensional, zero based array (SZARRAY) using the slow path that calls a runtime helper. @@ -254,12 +260,12 @@ ArraySizeOverflow: bl C_FUNC(RhpGcAlloc) // Set the new objects MethodTable pointer and length on success. - cbz x0, ArrayOutOfMemory + cbz x0, LOCAL_LABEL(ArrayOutOfMemory) POP_COOP_PINVOKE_FRAME EPILOG_RETURN -ArrayOutOfMemory: +LOCAL_LABEL(ArrayOutOfMemory): // This is the OOM failure path. We are going to tail-call to a managed helper that will throw // an out of memory exception that the caller of this allocator understands. diff --git a/src/coreclr/nativeaot/Runtime/arm64/ExceptionHandling.S b/src/coreclr/nativeaot/Runtime/arm64/ExceptionHandling.S index d0425171e1d19..7c04f15ad3b85 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/ExceptionHandling.S +++ b/src/coreclr/nativeaot/Runtime/arm64/ExceptionHandling.S @@ -275,7 +275,7 @@ // where the tail-calling thread had saved LR, which may not match where we have saved LR. ldr x1, [x2, #OFFSETOF__Thread__m_pvHijackedReturnAddress] - cbz x1, NotHijacked + cbz x1, LOCAL_LABEL(NotHijacked) ldr x3, [x2, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation] @@ -286,13 +286,13 @@ add x12, sp, #(STACKSIZEOF_ExInfo + SIZEOF__PAL_LIMITED_CONTEXT) // re-compute SP at callsite cmp x3, x12 // if (m_ppvHijackedReturnAddressLocation < SP at callsite) - blo TailCallWasHijacked + blo LOCAL_LABEL(TailCallWasHijacked) // normal case where a valid return address location is hijacked str x1, [x3] - b ClearThreadState + b LOCAL_LABEL(ClearThreadState) -TailCallWasHijacked: +LOCAL_LABEL(TailCallWasHijacked): // Abnormal case where the return address location is now invalid because we ended up here via a tail // call. In this case, our hijacked return address should be the correct caller of this method. @@ -302,13 +302,13 @@ TailCallWasHijacked: str lr, [sp, #(rsp_offsetof_Context + OFFSETOF__PAL_LIMITED_CONTEXT__LR)] str lr, [sp, #(rsp_offsetof_Context + OFFSETOF__PAL_LIMITED_CONTEXT__IP)] -ClearThreadState: +LOCAL_LABEL(ClearThreadState): // clear the Thread's hijack state str xzr, [x2, #OFFSETOF__Thread__m_ppvHijackedReturnAddressLocation] str xzr, [x2, #OFFSETOF__Thread__m_pvHijackedReturnAddress] -NotHijacked: +LOCAL_LABEL(NotHijacked): add x1, sp, #rsp_offsetof_ExInfo // x1 <- ExInfo* str xzr, [x1, #OFFSETOF__ExInfo__m_exception] // pExInfo->m_exception = null @@ -429,13 +429,13 @@ NotHijacked: add x12, x5, #OFFSETOF__Thread__m_ThreadStateFlags -ClearRetry_Catch: +LOCAL_LABEL(ClearRetry_Catch): ldxr w4, [x12] bic w4, w4, #TSF_DoNotTriggerGc stxr w6, w4, [x12] - cbz w6, ClearSuccess_Catch - b ClearRetry_Catch -ClearSuccess_Catch: + cbz w6, LOCAL_LABEL(ClearSuccess_Catch) + b LOCAL_LABEL(ClearRetry_Catch) +LOCAL_LABEL(ClearSuccess_Catch): // // set preserved regs to the values expected by the funclet @@ -487,21 +487,21 @@ ClearSuccess_Catch: ldr x3, [sp, #rsp_offset_x3] // x3 <- current ExInfo* ldr x2, [x2, #OFFSETOF__REGDISPLAY__SP] // x2 <- resume SP value -PopExInfoLoop: +LOCAL_LABEL(PopExInfoLoop): ldr x3, [x3, #OFFSETOF__ExInfo__m_pPrevExInfo] // x3 <- next ExInfo - cbz x3, DonePopping // if (pExInfo == null) { we're done } + cbz x3, LOCAL_LABEL(DonePopping) // if (pExInfo == null) { we're done } cmp x3, x2 - blt PopExInfoLoop // if (pExInfo < resume SP} { keep going } + blt LOCAL_LABEL(PopExInfoLoop) // if (pExInfo < resume SP} { keep going } -DonePopping: +LOCAL_LABEL(DonePopping): str x3, [x1, #OFFSETOF__Thread__m_pExInfoStackHead] // store the new head on the Thread PREPARE_EXTERNAL_VAR_INDIRECT_W RhpTrapThreads, 3 - tbz x3, #TrapThreadsFlags_AbortInProgress_Bit, NoAbort + tbz x3, #TrapThreadsFlags_AbortInProgress_Bit, LOCAL_LABEL(NoAbort) ldr x3, [sp, #rsp_offset_is_not_handling_thread_abort] - cbnz x3, NoAbort + cbnz x3, LOCAL_LABEL(NoAbort) // It was the ThreadAbortException, so rethrow it // reset SP @@ -510,7 +510,7 @@ DonePopping: mov sp, x2 b C_FUNC(RhpThrowHwEx) -NoAbort: +LOCAL_LABEL(NoAbort): // reset SP and jump to continuation address mov sp, x2 br x0 @@ -564,13 +564,13 @@ NoAbort: add x12, x2, #OFFSETOF__Thread__m_ThreadStateFlags -ClearRetry: +LOCAL_LABEL(ClearRetry): ldxr w4, [x12] bic w4, w4, #TSF_DoNotTriggerGc stxr w3, w4, [x12] - cbz w3, ClearSuccess - b ClearRetry -ClearSuccess: + cbz w3, LOCAL_LABEL(ClearSuccess) + b LOCAL_LABEL(ClearRetry) +LOCAL_LABEL(ClearSuccess): // // set preserved regs to the values expected by the funclet @@ -602,13 +602,13 @@ ClearSuccess: ldr x2, [sp, rsp_FinallyFunclet_offset_thread] add x12, x2, #OFFSETOF__Thread__m_ThreadStateFlags -SetRetry: +LOCAL_LABEL(SetRetry): ldxr w1, [x12] orr w1, w1, #TSF_DoNotTriggerGc stxr w3, w1, [x12] - cbz w3, SetSuccess - b SetRetry -SetSuccess: + cbz w3, LOCAL_LABEL(SetSuccess) + b LOCAL_LABEL(SetRetry) +LOCAL_LABEL(SetSuccess): ldp d8, d9, [sp, #0x00] ldp d10, d11, [sp, #0x10] @@ -707,13 +707,13 @@ SetSuccess: add x12, x5, #OFFSETOF__Thread__m_ThreadStateFlags -ClearRetry_Propagate: +LOCAL_LABEL(ClearRetry_Propagate): ldxr w4, [x12] bic w4, w4, #TSF_DoNotTriggerGc stxr w6, w4, [x12] - cbz w6, ClearSuccess_Propagate - b ClearRetry_Propagate -ClearSuccess_Propagate: + cbz w6, LOCAL_LABEL(ClearSuccess_Propagate) + b LOCAL_LABEL(ClearRetry_Propagate) +LOCAL_LABEL(ClearSuccess_Propagate): // // set preserved regs to the values expected by the funclet @@ -749,13 +749,13 @@ ClearSuccess_Propagate: ldr x3, [sp, #rsp_offset_x3] // x3 <- current ExInfo* ldr x2, [x2, #OFFSETOF__REGDISPLAY__SP] // x2 <- resume SP value -Propagate_PopExInfoLoop: +LOCAL_LABEL(Propagate_PopExInfoLoop): ldr x3, [x3, #OFFSETOF__ExInfo__m_pPrevExInfo] // x3 <- next ExInfo - cbz x3, Propagate_DonePopping // if (pExInfo == null) { we're done } + cbz x3, LOCAL_LABEL(Propagate_DonePopping) // if (pExInfo == null) { we're done } cmp x3, x2 - blt Propagate_PopExInfoLoop // if (pExInfo < resume SP} { keep going } + blt LOCAL_LABEL(Propagate_PopExInfoLoop) // if (pExInfo < resume SP} { keep going } -Propagate_DonePopping: +LOCAL_LABEL(Propagate_DonePopping): str x3, [x1, #OFFSETOF__Thread__m_pExInfoStackHead] // store the new head on the Thread // restore preemptive mode diff --git a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S index e27834bae6fed..abe7555b76113 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S +++ b/src/coreclr/nativeaot/Runtime/arm64/GcProbe.S @@ -127,10 +127,10 @@ NESTED_ENTRY RhpGcProbeHijack, _TEXT, NoHandler FixupHijackedCallstack PREPARE_EXTERNAL_VAR_INDIRECT_W RhpTrapThreads, 3 - tbnz x3, #TrapThreadsFlags_TrapThreads_Bit, WaitForGC + tbnz x3, #TrapThreadsFlags_TrapThreads_Bit, LOCAL_LABEL(WaitForGC) ret -WaitForGC: +LOCAL_LABEL(WaitForGC): orr x12, x12, DEFAULT_FRAME_SAVE_FLAGS + PTFF_SAVE_X0 + PTFF_SAVE_X1 b C_FUNC(RhpWaitForGC) NESTED_END RhpGcProbeHijack @@ -144,11 +144,11 @@ NESTED_ENTRY RhpWaitForGC, _TEXT, NoHandler bl C_FUNC(RhpWaitForGC2) ldr x2, [sp, #OFFSETOF__PInvokeTransitionFrame__m_Flags] - tbnz x2, #PTFF_THREAD_ABORT_BIT, ThrowThreadAbort + tbnz x2, #PTFF_THREAD_ABORT_BIT, LOCAL_LABEL(ThrowThreadAbort) POP_PROBE_FRAME EPILOG_RETURN -ThrowThreadAbort: +LOCAL_LABEL(ThrowThreadAbort): POP_PROBE_FRAME mov w0, #STATUS_REDHAWK_THREAD_ABORT mov x1, lr // return address as exception PC @@ -159,8 +159,10 @@ NESTED_END RhpWaitForGC LEAF_ENTRY RhpGcPoll PREPARE_EXTERNAL_VAR_INDIRECT_W RhpTrapThreads, 0 - cbnz w0, C_FUNC(RhpGcPollRare) // TrapThreadsFlags_None = 0 + cbnz w0, LOCAL_LABEL(RhpGcPoll_Rare) // TrapThreadsFlags_None = 0 ret +LOCAL_LABEL(RhpGcPoll_Rare): + b C_FUNC(RhpGcPollRare) LEAF_END RhpGcPoll NESTED_ENTRY RhpGcPollRare, _TEXT, NoHandler diff --git a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S index d00ffb3a4a997..835466c3b9e7e 100644 --- a/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S +++ b/src/coreclr/nativeaot/Runtime/arm64/WriteBarriers.S @@ -224,9 +224,11 @@ LEAF_END RhpByRefAssignRefArm64, _TEXT PREPARE_EXTERNAL_VAR_INDIRECT g_highest_address, x12 ccmp x14, x12, #0x2, hs - blo C_FUNC(RhpAssignRefArm64) + bhs LOCAL_LABEL(NotInHeap) -NotInHeap: + b C_FUNC(RhpAssignRefArm64) + +LOCAL_LABEL(NotInHeap): ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation str x15, [x14], 8 ret @@ -293,44 +295,44 @@ LEAF_END RhpAssignRef, _TEXT #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT PREPARE_EXTERNAL_VAR_INDIRECT_W g_cpuFeatures, 16 - tbz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, CmpXchgRetry + tbz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, LOCAL_LABEL(CmpXchgRetry) #endif mov x10, x2 ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation casal x10, x1, [x0] // exchange cmp x2, x10 - bne CmpXchgNoUpdate + bne LOCAL_LABEL(CmpXchgNoUpdate) #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT - b DoCardsCmpXchg -CmpXchgRetry: + b LOCAL_LABEL(DoCardsCmpXchg) +LOCAL_LABEL(CmpXchgRetry): // Check location value is what we expect. ALTERNATE_ENTRY RhpCheckedLockCmpXchgAVLocation2 ldaxr x10, [x0] cmp x10, x2 - bne CmpXchgNoUpdate + bne LOCAL_LABEL(CmpXchgNoUpdate) // Current value matches comparand, attempt to update with the new value. stlxr w12, x1, [x0] - cbnz w12, CmpXchgRetry + cbnz w12, LOCAL_LABEL(CmpXchgRetry) #endif -DoCardsCmpXchg: +LOCAL_LABEL(DoCardsCmpXchg): // We have successfully updated the value of the objectref so now we need a GC write barrier. // The following barrier code takes the destination in x0 and the value in x1 so the arguments are // already correctly set up. INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1 -CmpXchgNoUpdate: +LOCAL_LABEL(CmpXchgNoUpdate): // x10 still contains the original value. mov x0, x10 #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT - tbnz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, NoBarrierCmpXchg + tbnz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, LOCAL_LABEL(NoBarrierCmpXchg) InterlockedOperationBarrier -NoBarrierCmpXchg: +LOCAL_LABEL(NoBarrierCmpXchg): #endif ret lr @@ -357,25 +359,25 @@ NoBarrierCmpXchg: #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT PREPARE_EXTERNAL_VAR_INDIRECT_W g_cpuFeatures, 16 - tbz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, ExchangeRetry + tbz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, LOCAL_LABEL(ExchangeRetry) #endif ALTERNATE_ENTRY RhpCheckedXchgAVLocation swpal x1, x10, [x0] // exchange #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT - b DoCardsXchg -ExchangeRetry: + b LOCAL_LABEL(DoCardsXchg) +LOCAL_LABEL(ExchangeRetry): // Read the existing memory location. ALTERNATE_ENTRY RhpCheckedXchgAVLocation2 ldaxr x10, [x0] // Attempt to update with the new value. stlxr w12, x1, [x0] - cbnz w12, ExchangeRetry + cbnz w12, LOCAL_LABEL(ExchangeRetry) #endif -DoCardsXchg: +LOCAL_LABEL(DoCardsXchg): // We have successfully updated the value of the objectref so now we need a GC write barrier. // The following barrier code takes the destination in x0 and the value in x1 so the arguments are // already correctly set up. @@ -386,9 +388,9 @@ DoCardsXchg: mov x0, x10 #ifndef LSE_INSTRUCTIONS_ENABLED_BY_DEFAULT - tbnz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, NoBarrierXchg + tbnz w16, #ARM64_ATOMICS_FEATURE_FLAG_BIT, LOCAL_LABEL(NoBarrierXchg) InterlockedOperationBarrier -NoBarrierXchg: +LOCAL_LABEL(NoBarrierXchg): #endif ret diff --git a/src/coreclr/nativeaot/Runtime/unix/unixasmmacros.inc b/src/coreclr/nativeaot/Runtime/unix/unixasmmacros.inc index ef6d393fd248b..bde1d517b7e82 100644 --- a/src/coreclr/nativeaot/Runtime/unix/unixasmmacros.inc +++ b/src/coreclr/nativeaot/Runtime/unix/unixasmmacros.inc @@ -3,6 +3,11 @@ #define INVALIDGCVALUE 0xCCCCCCCD +// Enforce subsections via symbols to workaround bugs in Xcode 15 linker. +#if defined(__APPLE__) +.subsections_via_symbols +#endif + #if defined(__APPLE__) #define C_FUNC(name) _##name #define EXTERNAL_C_FUNC(name) C_FUNC(name) diff --git a/src/coreclr/pal/inc/unixasmmacros.inc b/src/coreclr/pal/inc/unixasmmacros.inc index 658a65bb4b35a..120b26543e3fa 100644 --- a/src/coreclr/pal/inc/unixasmmacros.inc +++ b/src/coreclr/pal/inc/unixasmmacros.inc @@ -3,6 +3,11 @@ #define INVALIDGCVALUE 0xCCCCCCCD +// Enforce subsections via symbols to workaround bugs in Xcode 15 linker. +#if defined(__APPLE__) +.subsections_via_symbols +#endif + #if defined(__APPLE__) #define C_FUNC(name) _##name #define EXTERNAL_C_FUNC(name) C_FUNC(name) diff --git a/src/coreclr/vm/arm64/asmhelpers.S b/src/coreclr/vm/arm64/asmhelpers.S index cdbe24ec427a9..89dab80461c35 100644 --- a/src/coreclr/vm/arm64/asmhelpers.S +++ b/src/coreclr/vm/arm64/asmhelpers.S @@ -329,7 +329,9 @@ WRITE_BARRIER_ENTRY JIT_CheckedWriteBarrier // branch below is not taken. ccmp x14, x12, #0x2, hs - blo C_FUNC(JIT_WriteBarrier) + bhs LOCAL_LABEL(NotInHeap) + + b C_FUNC(JIT_WriteBarrier) LOCAL_LABEL(NotInHeap): str x15, [x14], 8