8193879: Java debugger hangs on method invocation

Reviewed-by: sspitsyn

src/share/classes/com/sun/tools/jdi/InvokableTypeImpl.java

 /*
- * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -236,15 +236,7 @@ abstract class InvokableTypeImpl extends ReferenceTypeImpl {
                                            final MethodImpl method,
                                            final ValueImpl[] args,
                                            final int options) {
-        /*
-         * Cache the values of args when TRACE_SENDS is enabled, for later printing.
-         * If not cached, printing causes a remote call while synchronized, and deadlock.
-         */
-        if ((vm.traceFlags & VirtualMachineImpl.TRACE_SENDS) != 0) {
-           for (ValueImpl arg: args) {
-              arg.toString();
-           }
-        }
+
         CommandSender sender = getInvokeMethodSender(thread, method, args, options);
         PacketStream stream;
         if ((options & ClassType.INVOKE_SINGLE_THREADED) != 0) {

src/share/classes/com/sun/tools/jdi/VMState.java

 /*
- * Copyright (c) 1999, 2011, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,12 +39,10 @@ class VMState {
 
     /*
      * Certain information can be cached only when the entire VM is
-     * suspended and there are no pending resumes. The fields below
-     * are used to track whether there are pending resumes. (There
-     * is an assumption that JDWP command ids are increasing over time.)
+     * suspended and there are no pending resumes. The field below
+     * is used to track whether there are pending resumes.
      */
-    private int lastCompletedCommandId = 0;   // synchronized (this)
-    private int lastResumeCommandId = 0;      // synchronized (this)
+    private final Set<Integer> pendingResumeCommands = Collections.synchronizedSet(new HashSet<>());
 
     // This is cached only while the VM is suspended
     private static class Cache {
@@ -92,12 +90,12 @@ class VMState {
      * A JDWP command has been completed (reply has been received).
      * Update data that tracks pending resume commands.
      */
-    synchronized void notifyCommandComplete(int id) {
-        lastCompletedCommandId = id;
+    void notifyCommandComplete(int id) {
+        pendingResumeCommands.remove(id);
     }
 
     synchronized void freeze() {
-        if (cache == null && (lastCompletedCommandId >= lastResumeCommandId)) {
+        if (cache == null && (pendingResumeCommands.isEmpty())) {
             /*
              * No pending resumes to worry about. The VM is suspended
              * and additional state can be cached. Notify all
@@ -110,7 +108,7 @@ class VMState {
 
     synchronized PacketStream thawCommand(CommandSender sender) {
         PacketStream stream = sender.send();
-        lastResumeCommandId = stream.id();
+        pendingResumeCommands.add(stream.id());
         thaw();
         return stream;
     }

test/com/sun/jdi/MethodInvokeWithTraceOnTest.java

+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8193879 8193801 8129348
+ * @summary Invokes static and instance methods when debugger trace
+ * mode is on.
+ * @library /lib/testlibrary
+ * @run build TestScaffold VMConnection TargetListener TargetAdapter
+ * @run compile -g MethodInvokeWithTraceOnTest.java
+ * @run driver MethodInvokeWithTraceOnTest
+ */
+
+import com.sun.jdi.*;
+import com.sun.jdi.event.*;
+import com.sun.jdi.request.*;
+
+import java.util.*;
+
+import jdk.testlibrary.Utils;
+
+/********** target program **********/
+class MethodInvokeWithTraceOnTestTarg {
+    public static void main(String[] args) {
+        new MethodInvokeWithTraceOnTestTarg().test();
+    }
+
+    private void test() {
+        Thread thread = Thread.currentThread();
+        print(thread); // @1 breakpoint
+        String str = "test";
+        printStatic(str); // @2 breakpoint
+
+    }
+
+    public void print(Object obj) {
+        System.out.println(obj);
+    }
+
+    public static void printStatic(Object obj) {
+        System.out.println(obj);
+    }
+
+}
+
+
+/********** test program **********/
+
+public class MethodInvokeWithTraceOnTest extends TestScaffold {
+
+    MethodInvokeWithTraceOnTest(String args[]) {
+        super(args);
+    }
+
+    public static void main(String[] args)
+            throws Exception {
+        new MethodInvokeWithTraceOnTest(args).startTests();
+    }
+
+    /********** test core **********/
+
+    protected void runTests() throws Exception {
+        init();
+
+        // Test with suspend policy set to SUSPEND_EVENT_THREAD
+        BreakpointEvent be = resumeToBreakpoint(true, 1);
+        System.out.println("Breakpoint 1 is hit, suspendPolicy:" + be.request().suspendPolicy());
+        testMethods(be);
+
+        // Test with suspend policy set to SUSPEND_ALL
+        be = resumeToBreakpoint(false, 2);
+        System.out.println("Breakpoint 2 is hit, suspendPolicy:" + be.request().suspendPolicy());
+        testMethods(be);
+
+        listenUntilVMDisconnect();
+    }
+
+    private void init() throws Exception {
+        startToMain("MethodInvokeWithTraceOnTestTarg");
+        vm().setDebugTraceMode(VirtualMachine.TRACE_ALL);
+    }
+
+    private BreakpointEvent resumeToBreakpoint(boolean suspendThread, int breakpointId) throws Exception {
+        int bkpLine = Utils.parseBreakpoints(Utils.getTestSourcePath("MethodInvokeWithTraceOnTest.java"), breakpointId).get(0);
+        System.out.println("Running to line: " + bkpLine);
+        return resumeTo("MethodInvokeWithTraceOnTestTarg", bkpLine, suspendThread);
+    }
+
+    private void testMethods(BreakpointEvent be) throws Exception {
+        System.out.println("Testing  methods...");
+        ThreadReference thread = be.thread();
+        StackFrame frame = thread.frame(0);
+        ObjectReference thisObj = frame.thisObject();
+        LocalVariable threadVar = frame.visibleVariableByName("thread");
+        ThreadReference threadObj = (ThreadReference) frame.getValue(threadVar);
+        StringReference stringObj = vm().mirrorOf("test string");
+        int invokeOptions = getMethodInvokeOptions(be);
+
+        testInstanceMethod1(thread, thisObj, stringObj, threadObj, invokeOptions);
+        testStaticMethod1(thread, thisObj, stringObj, threadObj, invokeOptions);
+        testStaticMethod2(thread, invokeOptions);
+    }
+
+    private void testInstanceMethod1(ThreadReference thread, ObjectReference thisObj, StringReference stringObj,
+                                     ThreadReference threadObj, int invokeOptions) throws Exception {
+        ClassType classType = (ClassType) thisObj.referenceType();
+        Method printMethod = classType.methodsByName("print",
+                "(Ljava/lang/Object;)V").get(0);
+
+        System.out.println("Passing StringReference to instance method...");
+        thisObj.invokeMethod(thread, printMethod, Collections.singletonList(stringObj), invokeOptions);
+
+        System.out.println("Passing ThreadReference to instance method...");
+        thisObj.invokeMethod(thread, printMethod, Collections.singletonList(threadObj), invokeOptions);
+    }
+
+    private void testStaticMethod1(ThreadReference thread, ObjectReference thisObj, StringReference stringObj,
+                                   ThreadReference threadObj, int invokeOptions) throws Exception {
+        ClassType classType = (ClassType) thisObj.referenceType();
+        Method printMethod = classType.methodsByName("printStatic",
+                "(Ljava/lang/Object;)V").get(0);
+
+        System.out.println("Passing StringReference to static method...");
+        classType.invokeMethod(thread, printMethod, Collections.singletonList(stringObj), invokeOptions);
+
+        System.out.println("Passing ThreadReference to static method...");
+        classType.invokeMethod(thread, printMethod, Collections.singletonList(threadObj), invokeOptions);
+    }
+
+    private void testStaticMethod2(ThreadReference thread, int invokeOptions) throws Exception {
+        ClassType classType = getClassType("java.lang.Class");
+        Method forNameMethod = classType.methodsByName("forName",
+                "(Ljava/lang/String;)Ljava/lang/Class;").get(0);
+        StringReference classNameParam = vm().mirrorOf("java.lang.String");
+        classType.invokeMethod(thread, forNameMethod, Collections.singletonList(classNameParam), invokeOptions);
+    }
+
+    private ClassType getClassType(String className) {
+        List classes = vm().classesByName(className);
+        return (ClassType) classes.get(0);
+    }
+
+    private int getMethodInvokeOptions(BreakpointEvent be) {
+        return be.request().suspendPolicy() == EventRequest.SUSPEND_EVENT_THREAD ?
+                ObjectReference.INVOKE_SINGLE_THREADED : 0;
+    }
+}

test/com/sun/jdi/TestScaffold.java

 /*
- * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -780,9 +780,16 @@ abstract public class TestScaffold extends TargetAdapter {
     }
 
     public BreakpointEvent resumeTo(Location loc) {
+        return resumeTo(loc, false);
+    }
+
+    public BreakpointEvent resumeTo(Location loc, boolean suspendThread) {
         final BreakpointRequest request =
             requestManager.createBreakpointRequest(loc);
         request.addCountFilter(1);
+        if (suspendThread) {
+            request.setSuspendPolicy(EventRequest.SUSPEND_EVENT_THREAD);
+        }
         request.enable();
         return (BreakpointEvent)waitForRequestedEvent(request);
     }
@@ -841,12 +848,16 @@ abstract public class TestScaffold extends TargetAdapter {
     }
 
     public BreakpointEvent resumeTo(String clsName, int lineNumber) throws AbsentInformationException {
+        return resumeTo(clsName, lineNumber, false);
+    }
+
+    public BreakpointEvent resumeTo(String clsName, int lineNumber, boolean suspendThread) throws AbsentInformationException {
         ReferenceType rt = findReferenceType(clsName);
         if (rt == null) {
             rt = resumeToPrepareOf(clsName).referenceType();
         }
 
-        return resumeTo(findLocation(rt, lineNumber));
+        return resumeTo(findLocation(rt, lineNumber), suspendThread);
     }
 
     public ClassPrepareEvent resumeToPrepareOf(String className) {

test/lib/testlibrary/jdk/testlibrary/Utils.java

 /*
- * Copyright (c) 2013, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,12 +37,15 @@ import java.net.UnknownHostException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Arrays;
+import java.util.LinkedList;
+import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.Objects;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
+import java.nio.file.Files;
 
 /**
  * Common library for various test helper functions.
@@ -411,4 +414,30 @@ public final class Utils {
         }
         return transferred;
     }
+
+    // Parses the specified source file for "@{id} breakpoint" tags and returns
+    // list of the line numbers containing the tag.
+    // Example:
+    //   System.out.println("BP is here");  // @1 breakpoint
+    public static List<Integer> parseBreakpoints(String filePath, int id) {
+        final String pattern = "@" + id + " breakpoint";
+        int lineNum = 1;
+        List<Integer> result = new LinkedList<>();
+        try {
+            for (String line: Files.readAllLines(Paths.get(filePath))) {
+                if (line.contains(pattern)) {
+                    result.add(lineNum);
+                }
+                lineNum++;
+            }
+        } catch (IOException ex) {
+            throw new RuntimeException("failed to parse " + filePath, ex);
+        }
+        return result;
+    }
+
+    // gets full test source path for the given test filename
+    public static String getTestSourcePath(String fileName) {
+        return Paths.get(System.getProperty("test.src")).resolve(fileName).toString();
+    }
 }