Support transition to debug interpreter on restore

Support transition to debug interpreter on restore when
the transition is requested with an env var file or an
options file.

Issue: eclipse-openj9#17642
Co-authored-by: Tobi Ajila <atobia@ca.ibm.com>
Signed-off-by: Amarpreet Singh <Amarpreet.A.Singh@ibm.com>

runtime/nls/j9vm/j9vm.nls

@@ -2394,3 +2394,10 @@ J9NLS_VM_DEPRECATED_OPTION.explanation=The command line option is deprecated.
 J9NLS_VM_DEPRECATED_OPTION.system_action=The JVM will print a deprecation warning.
 J9NLS_VM_DEPRECATED_OPTION.user_response=Remove the command line option from the command line.
 # END NON-TRANSLATABLE
+
+J9NLS_VM_CRIU_CHECK_TRANSITION_TO_DEBUG_INTERPRETER_FAILED=The check for the transition to the debug interpreter failed
+# START NON-TRANSLATABLE
+J9NLS_VM_CRIU_CHECK_TRANSITION_TO_DEBUG_INTERPRETER_FAILED.explanation=checkTransitionToDebugInterpreter failed.
+J9NLS_VM_CRIU_CHECK_TRANSITION_TO_DEBUG_INTERPRETER_FAILED.system_action=The JVM will throw a JVMRestoreException.
+J9NLS_VM_CRIU_CHECK_TRANSITION_TO_DEBUG_INTERPRETER_FAILED.user_response=View CRIU documentation to determine how to resolve the error.
+# END NON-TRANSLATABLE

runtime/vm/CRIUHelpers.cpp

@@ -59,6 +59,12 @@ static BOOLEAN criuRestoreInitializeXrs(J9VMThread *currentThread, void *userDat
 static BOOLEAN criuRestoreDisableSharedClassCache(J9VMThread *currentThread, void *userData, const char **nlsMsgFormat);
 static BOOLEAN criuRestoreInitializeDump(J9VMThread *currentThread, void *userData, const char **nlsMsgFormat);
 static jvmtiIterationControl objectIteratorCallback(J9JavaVM *vm, J9MM_IterateObjectDescriptor *objectDesc, void *userData);
+#if defined(OMR_GC_FULL_POINTERS)
+UDATA debugBytecodeLoopFull(J9VMThread *currentThread);
+#endif /* defined(OMR_GC_FULL_POINTERS) */
+#if defined(OMR_GC_COMPRESSED_POINTERS)
+UDATA debugBytecodeLoopCompressed(J9VMThread *currentThread);
+#endif /* defined(OMR_GC_COMPRESSED_POINTERS) */
 
 #define STRING_BUFFER_SIZE 256
 #define ENV_FILE_BUFFER 1024
@@ -1443,6 +1449,49 @@ loadRestoreArguments(J9VMThread *currentThread, const char *optionsFile, char *e
 	return result;
 }
 
+static void
+transitionToDebugInterpreter(J9JavaVM *vm)
+{
+	if (J9JAVAVM_COMPRESS_OBJECT_REFERENCES(vm)) {
+#if defined(OMR_GC_COMPRESSED_POINTERS)
+		vm->bytecodeLoop = debugBytecodeLoopCompressed;
+#endif /* defined(OMR_GC_COMPRESSED_POINTERS) */
+	} else {
+#if defined(OMR_GC_FULL_POINTERS)
+		vm->bytecodeLoop = debugBytecodeLoopFull;
+#endif /* defined(OMR_GC_FULL_POINTERS) */
+	}
+	J9VMThread *walkThread = J9_LINKED_LIST_START_DO(vm->mainThread);
+	while (NULL != walkThread) {
+		VM_VMHelpers::requestInterpreterReentry(walkThread);
+		walkThread = J9_LINKED_LIST_NEXT_DO(vm->mainThread, walkThread);
+	}
+}
+
+static BOOLEAN
+checkTransitionToDebugInterpreter(J9VMThread *currentThread)
+{
+	BOOLEAN result = TRUE;
+	J9JavaVM *vm = currentThread->javaVM;
+	if (NULL != vm->checkpointState.restoreArgsList) {
+		J9VMInitArgs *restoreArgsList = vm->checkpointState.restoreArgsList;
+		IDATA debugOn = FIND_AND_CONSUME_ARG(restoreArgsList, EXACT_MATCH, VMOPT_XXDEBUGINTERPRETER, NULL);
+		IDATA debugOff = FIND_AND_CONSUME_ARG(restoreArgsList, EXACT_MATCH, VMOPT_XXNODEBUGINTERPRETER, NULL);
+		if (debugOn > debugOff) {
+			/*
+			 * The transition to the debug interpreter currently only works with -Xint,
+			 * and the null check for vm->jitConfig will be removed when the jit changes are completed.
+			 */
+			if (isDebugOnRestoreEnabled(currentThread) && (NULL == vm->jitConfig)) {
+				transitionToDebugInterpreter(vm);
+			} else {
+				result = FALSE;
+			}
+		}
+	}
+	return result;
+}
+
 void JNICALL
 criuCheckpointJVMImpl(JNIEnv *env,
 		jstring imagesDir,
@@ -1823,6 +1872,15 @@ criuCheckpointJVMImpl(JNIEnv *env,
 			case RESTORE_ARGS_RETURN_OK:
 				break;
 			}
+
+			if (!checkTransitionToDebugInterpreter(currentThread)) {
+				currentExceptionClass = vm->checkpointState.criuJVMRestoreExceptionClass;
+				nlsMsgFormat = j9nls_lookup_message(
+					J9NLS_DO_NOT_PRINT_MESSAGE_TAG | J9NLS_DO_NOT_APPEND_NEWLINE,
+					J9NLS_VM_CRIU_CHECK_TRANSITION_TO_DEBUG_INTERPRETER_FAILED,
+					NULL);
+				goto wakeJavaThreadsWithExclusiveVMAccess;
+			}
 		}
 
 		VM_VMHelpers::setVMState(currentThread, J9VMSTATE_CRIU_SUPPORT_RESTORE_PHASE_JAVA_HOOKS);

test/functional/cmdLineTests/criu/criu_nonPortable.xml

@@ -761,6 +761,24 @@
     <output type="failure" caseSensitive="yes" regex="no">org.eclipse.openj9.criu.JVMRestoreException</output>
     <output type="failure" caseSensitive="yes" regex="no">User requested Java dump using</output>
   </test>
+
+  <test id="Test checkTransitionToDebugInterpreter with env var file">
+    <!-- The transition to the debug interpreter currently only works with -Xint, which will be removed when the jit changes are completed. -->
+    <command>bash $SCRIPPATH$ $TEST_RESROOT$ $JAVA_COMMAND$ "$JVM_OPTIONS$ -XX:+DebugOnRestore -Xint" $MAINCLASS_ENVVAR_TEST$ testCheckTransitionToDebugInterpreterWithEnvVarFile 1 false false</command>
+    <output type="success" caseSensitive="no" regex="no">Killed</output>
+    <output type="required" caseSensitive="yes" regex="no">Pre-checkpoint</output>
+    <output type="success" caseSensitive="yes" regex="no">Post-checkpoint</output>
+    <output type="failure" caseSensitive="yes" regex="no">CRIU is not enabled</output>
+    <output type="failure" caseSensitive="yes" regex="no">Operation not permitted</output>
+    <!-- If CRIU can't acquire the original thread IDs, this test will fail. Nothing can be done about this failure. -->
+    <output type="success" caseSensitive="yes" regex="no">Thread pid mismatch</output>
+    <output type="success" caseSensitive="yes" regex="no">do not match expected</output>
+    <output type="success" caseSensitive="yes" regex="no">Unable to create a thread:</output>
+    <!-- In the past, the failure below was caused by an issue where CRIU can't be found on the PATH. -->
+    <output type="failure" caseSensitive="yes" regex="no">Could not dump the JVM processes, err=-70</output>
+    <output type="failure" caseSensitive="yes" regex="no">User requested Java dump using</output>
+  </test>
+
   <test id="Restore trace options test with no trace options specified before checkpoint - 1">
     <command>bash $SCRIPPATH$ $TEST_RESROOT$ $JAVA_COMMAND$ "$JVM_OPTIONS$" $MAINCLASS_OPTIONSFILE_TEST$ TraceOptionsTest1 1 false false</command>
     <output type="success" caseSensitive="no" regex="no">Killed</output>
@@ -906,6 +924,23 @@
     <output type="failure" caseSensitive="yes" regex="no">User requested Java dump using</output>
   </test>
 
+  <test id="Test checkTransitionToDebugInterpreter with options file">
+    <!-- The transition to the debug interpreter currently only works with -Xint, which will be removed when the jit changes are completed. -->
+    <command>bash $SCRIPPATH$ $TEST_RESROOT$ $JAVA_COMMAND$ "$JVM_OPTIONS$ -XX:+DebugOnRestore -Xint" $MAINCLASS_OPTIONSFILE_TEST$ testCheckTransitionToDebugInterpreterWithOptionsFile 1 false false</command>
+    <output type="success" caseSensitive="no" regex="no">Killed</output>
+    <output type="required" caseSensitive="yes" regex="no">Pre-checkpoint</output>
+    <output type="success" caseSensitive="yes" regex="no">Post-checkpoint</output>
+    <output type="failure" caseSensitive="yes" regex="no">CRIU is not enabled</output>
+    <output type="failure" caseSensitive="yes" regex="no">Operation not permitted</output>
+    <!-- If CRIU can't acquire the original thread IDs, this test will fail. Nothing can be done about this failure. -->
+    <output type="success" caseSensitive="yes" regex="no">Thread pid mismatch</output>
+    <output type="success" caseSensitive="yes" regex="no">do not match expected</output>
+    <output type="success" caseSensitive="yes" regex="no">Unable to create a thread:</output>
+    <!-- In the past, the failure below was caused by an issue where CRIU can't be found on the PATH. -->
+    <output type="failure" caseSensitive="yes" regex="no">Could not dump the JVM processes, err=-70</output>
+    <output type="failure" caseSensitive="yes" regex="no">User requested Java dump using</output>
+  </test>
+
   <test id="Username test">
     <command>bash $SCRIPPATH$ $TEST_RESROOT$ $JAVA_COMMAND$ "$JVM_OPTIONS$" $MAINCLASS_USERNAME_TEST$ unusedArgument 1 false false</command>
     <output type="success" caseSensitive="no" regex="no">Killed</output>

test/functional/cmdLineTests/criu/src/org/openj9/criu/EnvVarFileTest.java

@@ -83,6 +83,9 @@ public static void main(String[] args) {
 		case "EnvVarFileTest16":
 			envVarFileTest16();
 			break;
+		case "testCheckTransitionToDebugInterpreterWithEnvVarFile":
+			testCheckTransitionToDebugInterpreterWithEnvVarFile();
+			break;
 		default:
 			throw new RuntimeException("incorrect parameters");
 		}
@@ -404,4 +407,15 @@ static void envVarFileTest16() {
 		System.out.println("Post-checkpoint");
 	}
 
+	static void testCheckTransitionToDebugInterpreterWithEnvVarFile() {
+		String optionsContents = RESTORE_ENV_VAR + "=-XX:+DebugInterpreter";
+		Path optionsFilePath = CRIUTestUtils.createOptionsFile("options", optionsContents);
+		Path imagePath = Paths.get("cpData");
+		CRIUTestUtils.createCheckpointDirectory(imagePath);
+		CRIUSupport criuSupport = new CRIUSupport(imagePath);
+		criuSupport.registerRestoreEnvFile(optionsFilePath);
+		System.out.println("Pre-checkpoint");
+		CRIUTestUtils.checkPointJVM(criuSupport, imagePath, true);
+		System.out.println("Post-checkpoint");
+	}
 }

test/functional/cmdLineTests/criu/src/org/openj9/criu/OptionsFileTest.java

@@ -71,6 +71,9 @@ public static void main(String[] args) {
 		case "JitOptionsTest":
 			jitOptionsTest(args);
 			break;
+		case "testCheckTransitionToDebugInterpreterWithOptionsFile":
+			testCheckTransitionToDebugInterpreterWithOptionsFile();
+			break;
 		default:
 			throw new RuntimeException("incorrect parameters");
 		}
@@ -332,4 +335,15 @@ static void jitOptionsTest(String[] args) {
 		}
 	}
 
+	static void testCheckTransitionToDebugInterpreterWithOptionsFile() {
+		String optionsContents = "-XX:+DebugInterpreter";
+		Path optionsFilePath = CRIUTestUtils.createOptionsFile("options", optionsContents);
+		Path imagePath = Paths.get("cpData");
+		CRIUTestUtils.createCheckpointDirectory(imagePath);
+		CRIUSupport criuSupport = new CRIUSupport(imagePath);
+		criuSupport.registerRestoreOptionsFile(optionsFilePath);
+		System.out.println("Pre-checkpoint");
+		CRIUTestUtils.checkPointJVM(criuSupport, imagePath, true);
+		System.out.println("Post-checkpoint");
+	}
 }