Support interpreter re-entry for CRIU debug support

runtime/jilgen/jilconsts.c

@@ -762,6 +762,7 @@ writeConstants(OMRPortLibrary *OMRPORTLIB, IDATA fd)
 			writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_enter_method_monitor", J9_BCLOOP_ENTER_METHOD_MONITOR) |
 			writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_report_method_enter", J9_BCLOOP_REPORT_METHOD_ENTER) |
 			writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_exit_interpreter", J9_BCLOOP_EXIT_INTERPRETER) |
+			writeConstant(OMRPORTLIB, fd, "J9TR_bcloop_reenter_interpreter", J9_BCLOOP_REENTER_INTERPRETER) |
 			writeConstant(OMRPORTLIB, fd, "J9TR_MethodNotCompiledBit", J9_STARTPC_NOT_TRANSLATED) |
 			writeConstant(OMRPORTLIB, fd, "J9TR_InterpVTableOffset", J9JIT_INTERP_VTABLE_OFFSET) |
 			writeConstant(OMRPORTLIB, fd, "J9TR_RequiredClassAlignment", J9_REQUIRED_CLASS_ALIGNMENT) |

runtime/oti/VMHelpers.hpp

@@ -803,7 +803,7 @@ class VM_VMHelpers
 	static VMINLINE bool
 	immediateAsyncPending(J9VMThread *currentThread)
 	{
-		return (0 != (currentThread->publicFlags & J9_PUBLIC_FLAGS_POP_FRAMES_INTERRUPT));
+		return J9_ARE_ANY_BITS_SET(currentThread->publicFlags, J9_PUBLIC_FLAGS_POP_FRAMES_INTERRUPT);
 	}
 
 	/**
@@ -2170,6 +2170,34 @@ class VM_VMHelpers
 		return result;
 	}
 #endif /* defined(J9VM_OPT_CRIU_SUPPORT) */
+
+	/**
+	 * Query and reset the current thread's interpreter re-entry flag.
+	 *
+	 * @param[in] currentThread the current J9VMThread
+	 *
+	 * @return true if re-entry requested, false if not
+	 */
+	static VMINLINE bool
+	interpreterReentryRequested(J9VMThread *currentThread)
+	{
+		bool rc = J9_ARE_ANY_BITS_SET(currentThread->privateFlags2, J9_PRIVATE_FLAGS2_REENTER_INTERPRETER);
+		currentThread->privateFlags2 &= ~(UDATA)J9_PRIVATE_FLAGS2_REENTER_INTERPRETER;
+		return rc;
+	}
+
+	/**
+	 * Request interpreter re-entry on a J9VMThread. The target thread must either be
+	 * the current thread or not have VM access (e.g. halted by exclusive VM access).
+	 *
+	 * @param[in] targetThread the target J9VMThread
+	 */
+	static VMINLINE void
+	requestInterpreterReentry(J9VMThread *targetThread)
+	{
+		targetThread->privateFlags2 |= J9_PRIVATE_FLAGS2_REENTER_INTERPRETER;
+		indicateAsyncMessagePending(targetThread);
+	}
 };
 
 #endif /* VMHELPERS_HPP_ */

runtime/oti/j9consts.h

@@ -181,6 +181,7 @@ extern "C" {
 #define J9_PRIVATE_FLAGS2_CHECK_PACKAGE_ACCESS 0x2
 #define J9_PRIVATE_FLAGS2_UNSAFE_HANDLE_SIGBUS 0x4
 #define J9_PRIVATE_FLAGS2_ASYNC_GET_CALL_TRACE 0x8 /* DO NOT set this anywhere but AsyncGetCallTrace */
+#define J9_PRIVATE_FLAGS2_REENTER_INTERPRETER 0x10
 
 #define J9_PUBLIC_FLAGS_HALT_THREAD_EXCLUSIVE 0x1
 #define J9_PUBLIC_FLAGS_DEBUG_VM_ACCESS 0x2
@@ -720,6 +721,7 @@ extern "C" {
 #if JAVA_SPEC_VERSION >= 16
 #define J9_BCLOOP_N2I_TRANSITION 0x18
 #endif /* JAVA_SPEC_VERSION >= 16 */
+#define J9_BCLOOP_REENTER_INTERPRETER 0x19
 
 #define J9_RAS_METHOD_UNSEEN 0x0
 #define J9_RAS_METHOD_SEEN 0x1

runtime/vm/BytecodeInterpreter.hpp

@@ -189,6 +189,25 @@ class INTERPRETER_CLASS
  * Function members
  */
 private:
+
+	VMINLINE bool
+	interpreterReentryRequested()
+	{
+#if defined(J9VM_OPT_CRIU_SUPPORT)
+		return VM_VMHelpers::interpreterReentryRequested(_currentThread);
+#else /* J9VM_OPT_CRIU_SUPPORT */
+		return false;
+#endif /* J9VM_OPT_CRIU_SUPPORT */
+	}
+
+	VMINLINE VM_BytecodeAction
+	reenterInterpreter(UDATA reentryAction)
+	{
+		_currentThread->returnValue = reentryAction;
+		_nextAction = J9_BCLOOP_REENTER_INTERPRETER;
+		return GOTO_DONE;
+	}
+
 #if defined(J9VM_OPT_METHOD_HANDLE)
 	/**
 	 * Run a methodHandle using the MethodHandle interpreter/
@@ -1447,66 +1466,74 @@ class INTERPRETER_CLASS
 	VMINLINE VM_BytecodeAction
 	checkAsync(REGISTER_ARGS_LIST)
 	{
-		/* The current stack frame may be one of the following:
-		 *
-		 * 1) Special frame
-		 *
-		 *	1a) INL frame (for an INL which returns void)
-		 *
-		 *		No frame build is required, and the restoreSpecialStackFrameAndDrop will remove the
-		 *		argument from the stack before resuming execution.
-		 *
-		 *	1b) An immediate async is pending
-		 *
-		 *		The stack is already in a walkable state, and the restore and executeCurrentBytecode
-		 *		will not take place.
-		 *
-		 * 2) Bytecode or JNI call-in frame
-		 *
-		 *	Build a generic special frame, tear it down when done.
-		 */
 		VM_BytecodeAction rc = EXECUTE_BYTECODE;
-		bool inlFrame = false;
-		UDATA *bp = NULL;
-		if ((UDATA)_pc <= J9SF_MAX_SPECIAL_FRAME_TYPE) {
-			if (J9SF_FRAME_TYPE_NATIVE_METHOD == (UDATA)_pc) {
-				inlFrame = true;
-				bp = ((UDATA*)(((J9SFNativeMethodFrame*)((UDATA)_sp + (UDATA)_literals)) + 1)) - 1;
-			}
+
+		/* Check for interpreter re-entry first because we will come back to checkAsync
+		 * immediately upon re-entry to the interpreter.
+		 */
+		if (interpreterReentryRequested()) {
+			rc = reenterInterpreter(J9_BCLOOP_CHECK_ASYNC);
 		} else {
-			buildGenericSpecialStackFrame(REGISTER_ARGS, 0);
-			bp = _arg0EA;
-		}
-		UDATA relativeBP = bp - _arg0EA;
-		updateVMStruct(REGISTER_ARGS);
-		UDATA action = javaCheckAsyncMessages(_currentThread, TRUE);
-		VMStructHasBeenUpdated(REGISTER_ARGS);
-		switch(action) {
-		case J9_CHECK_ASYNC_THROW_EXCEPTION:
-			rc = GOTO_THROW_CURRENT_EXCEPTION;
-			break;
-#if defined(DEBUG_VERSION)
-		case J9_CHECK_ASYNC_POP_FRAMES:
-			rc = HANDLE_POP_FRAMES;
-			break;
-#endif
-		default:
-			restoreSpecialStackFrameAndDrop(REGISTER_ARGS, _arg0EA + relativeBP);
-			if (inlFrame) {
-				_pc += 3;
+			/* The current stack frame may be one of the following:
+			 *
+			 * 1) Special frame
+			 *
+			 *	1a) INL frame (for an INL which returns void)
+			 *
+			 *		No frame build is required, and the restoreSpecialStackFrameAndDrop will remove the
+			 *		argument from the stack before resuming execution.
+			 *
+			 *	1b) An immediate async is pending
+			 *
+			 *		The stack is already in a walkable state, and the restore and executeCurrentBytecode
+			 *		will not take place.
+			 *
+			 * 2) Bytecode or JNI call-in frame
+			 *
+			 *	Build a generic special frame, tear it down when done.
+			 */
+			bool inlFrame = false;
+			UDATA *bp = NULL;
+			if ((UDATA)_pc <= J9SF_MAX_SPECIAL_FRAME_TYPE) {
+				if (J9SF_FRAME_TYPE_NATIVE_METHOD == (UDATA)_pc) {
+					inlFrame = true;
+					bp = ((UDATA*)(((J9SFNativeMethodFrame*)((UDATA)_sp + (UDATA)_literals)) + 1)) - 1;
+				}
+			} else {
+				buildGenericSpecialStackFrame(REGISTER_ARGS, 0);
+				bp = _arg0EA;
+			}
+			UDATA relativeBP = bp - _arg0EA;
+			updateVMStruct(REGISTER_ARGS);
+			UDATA action = javaCheckAsyncMessages(_currentThread, TRUE);
+			VMStructHasBeenUpdated(REGISTER_ARGS);
+			switch(action) {
+			case J9_CHECK_ASYNC_THROW_EXCEPTION:
+				rc = GOTO_THROW_CURRENT_EXCEPTION;
+				break;
+	#if defined(DEBUG_VERSION)
+			case J9_CHECK_ASYNC_POP_FRAMES:
+				rc = HANDLE_POP_FRAMES;
+				break;
+	#endif
+			default:
+				restoreSpecialStackFrameAndDrop(REGISTER_ARGS, _arg0EA + relativeBP);
+				if (inlFrame) {
+					_pc += 3;
+				}
+				break;
 			}
-			break;
 		}
 		return rc;
 	}
 
 	VMINLINE bool immediateAsyncPending()
 	{
-#if defined(DEBUG_VERSION)
+#if defined(DEBUG_VERSION) || defined(J9VM_OPT_CRIU_SUPPORT)
 		return VM_VMHelpers::immediateAsyncPending(_currentThread);
-#else
+#else /* defined(DEBUG_VERSION) || defined(J9VM_OPT_CRIU_SUPPORT) */
 		return false;
-#endif
+#endif /* defined(DEBUG_VERSION) || defined(J9VM_OPT_CRIU_SUPPORT) */
 	}
 
 	VMINLINE VM_BytecodeAction
@@ -1865,7 +1892,14 @@ class INTERPRETER_CLASS
 			}
 #if defined(DO_HOOKS)
 			rc = REPORT_METHOD_ENTER;
-#endif
+			if (interpreterReentryRequested()) {
+				rc = reenterInterpreter(J9_BCLOOP_REPORT_METHOD_ENTER);
+			}
+#else /* DO_HOOKS */
+			if (interpreterReentryRequested()) {
+				rc = reenterInterpreter(J9_BCLOOP_EXECUTE_BYTECODE);
+			}
+#endif /* DO_HOOKS */
 		}
 done:
 		return rc;
@@ -2341,6 +2375,9 @@ class INTERPRETER_CLASS
 			*(U_32 *)_sp = (U_32)_currentThread->returnValue;
 		}
 		rc = EXECUTE_BYTECODE;
+		if (interpreterReentryRequested()) {
+			rc = reenterInterpreter(J9_BCLOOP_EXECUTE_BYTECODE);
+		}
 done:
 		return rc;
 	}

runtime/vm/arm64cinterp.m4

@@ -49,6 +49,8 @@ FUNC_LABEL(cInterpreter):
 	blr x28
 	cmp x0, {#}J9TR_bcloop_exit_interpreter
 	beq .L_cInterpExit
+	cmp x0, {#}J9TR_bcloop_reenter_interpreter
+	beq FUNC_LABEL(cInterpreter
 	RESTORE_PRESERVED_REGS
 	RESTORE_FPLR
 	SWITCH_TO_JAVA_STACK

runtime/vm/armcinterp.m4

@@ -42,6 +42,8 @@ cInterpreter:
 	ldr r15,[r4,{#}J9TR_JavaVM_bytecodeLoop]
 	cmp r0,{#}J9TR_bcloop_exit_interpreter
 	beq .L_cInterpExit
+	cmp r0,{#}J9TR_bcloop_reenter_interpreter
+	beq cInterpreter
 	RESTORE_PRESERVED_REGS
 	RESTORE_LR
 	SWITCH_TO_JAVA_STACK

runtime/vm/pcinterp.m4

@@ -130,6 +130,8 @@ ifdef({ASM_J9VM_ENV_DATA64},{
 	CALL_INDIRECT
 	cmpliaddr r3,J9TR_bcloop_exit_interpreter
 	beq .L_cInterpExit
+	cmpliaddr r3,J9TR_bcloop_reenter_interpreter
+	beq .L_cInterpOnCStack
 	RESTORE_PRESERVED_REGS
 	RESTORE_LR
 	SWITCH_TO_JAVA_STACK

runtime/vm/riscvcinterp.m4

@@ -70,6 +70,8 @@ cInterpreter:
     jalr ra, s8, 0
     li s7, J9TR_bcloop_exit_interpreter
     beq a0, s7, .L_cInterpExit
+    li s7, J9TR_bcloop_reenter_interpreter
+    beq a0, s7, cInterpreter
     RESTORE_PRESERVED_REGS
     RESTORE_FPLR
     SWITCH_TO_JAVA_STACK

runtime/vm/xcinterp.m4

@@ -168,6 +168,8 @@ C_FUNCTION_SYMBOL(cInterpreter):
 	CALL_C_ADDR_WITH_VMTHREAD(uword ptr J9TR_JavaVM_bytecodeLoop[_rax],0)
 	cmp _rax,J9TR_bcloop_exit_interpreter
 	je SHORT_JMP cInterpExit
+	cmp _rax,J9TR_bcloop_reenter_interpreter
+	je SHORT_JMP C_FUNCTION_SYMBOL(cInterpreter)
 	RESTORE_PRESERVED_REGS
 	SWITCH_TO_JAVA_STACK
 	jmp uword ptr J9TR_VMThread_tempSlot[_rbp]

runtime/vm/zcinterp.m4

@@ -78,6 +78,9 @@ PLACE_LABEL(L_CINTERP)
     LHI_GPR r0,J9TR_bcloop_exit_interpreter
     CLR_GPR CRINT,r0
     je LABEL_NAME(L_EXIT)
+    LHI_GPR r0,J9TR_bcloop_reenter_interpreter
+    CLR_GPR CRINT,r0
+    je LABEL_NAME(L_CINTERP)
     RESTORE_LR
     RESTORE_PRESERVED_REGS_AND_SWITCH_TO_JAVA_STACK
     BRANCH_VIA_VMTHREAD(J9TR_VMThread_tempSlot)