6440846: (cl) Deadlock between AppClassLoader and ExtClassLoader

Summary: Fixed a deadlock between the two class loaders
Reviewed-by: alanb

src/share/classes/sun/security/jca/ProviderConfig.java

 /*
- * Copyright 2003-2006 Sun Microsystems, Inc.  All Rights Reserved.
+ * Copyright 2003-2009 Sun Microsystems, Inc.  All Rights Reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -60,37 +60,6 @@ final class ProviderConfig {
     // use by doLoadProvider()
     private final static Class[] CL_STRING = { String.class };
 
-    // lock to use while loading a provider. it ensures that each provider
-    // is loaded only once and that we can detect recursion.
-    // NOTE that because of 4944382 we use the system classloader as lock.
-    // By using the same lock to load classes as to load providers we avoid
-    // deadlock due to lock ordering. However, this class may be initialized
-    // early in the startup when the system classloader has not yet been set
-    // up. Use a temporary lock object if that is the case.
-    // Any of this may break if the class loading implementation is changed.
-    private static volatile Object LOCK = new Object();
-
-    private static Object getLock() {
-        Object o = LOCK;
-        // check if lock is already set to the class loader
-        if (o instanceof ClassLoader) {
-            return o;
-        }
-        Object cl = AccessController.doPrivileged(
-                                new PrivilegedAction<Object>() {
-            public Object run() {
-                return ClassLoader.getSystemClassLoader();
-            }
-        });
-        // check if class loader initialized now (non-null)
-        if (cl != null) {
-            LOCK = cl;
-            o = cl;
-        }
-        return o;
-    }
-
-
     // name of the provider class
     private final String className;
 
@@ -194,7 +163,7 @@ final class ProviderConfig {
     /**
      * Get the provider object. Loads the provider if it is not already loaded.
      */
-    Provider getProvider() {
+    synchronized Provider getProvider() {
         // volatile variable load
         Provider p = provider;
         if (p != null) {
@@ -203,30 +172,23 @@ final class ProviderConfig {
         if (shouldLoad() == false) {
             return null;
         }
-        synchronized (getLock()) {
-            p = provider;
-            if (p != null) {
-                // loaded by another thread while we were blocked on lock
-                return p;
-            }
-            if (isLoading) {
-                // because this method is synchronized, this can only
-                // happen if there is recursion.
-                if (debug != null) {
-                    debug.println("Recursion loading provider: " + this);
-                    new Exception("Call trace").printStackTrace();
-                }
-                return null;
+        if (isLoading) {
+            // because this method is synchronized, this can only
+            // happen if there is recursion.
+            if (debug != null) {
+                debug.println("Recursion loading provider: " + this);
+                new Exception("Call trace").printStackTrace();
             }
-            try {
-                isLoading = true;
-                tries++;
-                p = doLoadProvider();
-            } finally {
-                isLoading = false;
-            }
-            provider = p;
+            return null;
+        }
+        try {
+            isLoading = true;
+            tries++;
+            p = doLoadProvider();
+        } finally {
+            isLoading = false;
         }
+        provider = p;
         return p;
     }
 

test/java/security/Security/ClassLoaderDeadlock/CreateSerialized.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+import java.io.*;
+import sun.misc.*;
+
+public class CreateSerialized {
+    public static void main(String[] args) throws Exception {
+        Object o = new com.sun.crypto.provider.SunJCE();
+
+        FileOutputStream fos = new FileOutputStream("object.tmp");
+        ObjectOutputStream objectOutputStream = new ObjectOutputStream(fos);
+        objectOutputStream.writeObject(o);
+        fos.close();
+    }
+}

test/java/security/Security/ClassLoaderDeadlock/Deadlock2.java

+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+import java.io.*;
+import javax.xml.parsers.DocumentBuilderFactory;
+import java.security.*;
+
+public class Deadlock2 {
+    public static void main(String[] args) throws Exception {
+        File file = new File("object.tmp");
+        final byte[] bytes = new byte[(int) file.length()];
+        FileInputStream fileInputStream = new FileInputStream(file);
+        int read = fileInputStream.read(bytes);
+        if (read != file.length()) {
+            throw new Exception("Didn't read all");
+        }
+        Thread.sleep(1000);
+
+        Runnable xmlRunnable = new Runnable() {
+                public void run() {
+                    try {
+                        DocumentBuilderFactory.newInstance();
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+            };
+
+        Runnable readObjectRunnable = new Runnable() {
+                public void run() {
+                    try {
+                        ObjectInputStream objectInputStream =
+                            new ObjectInputStream(new ByteArrayInputStream(bytes));
+                        Object o = objectInputStream.readObject();
+                        System.out.println(o.getClass());
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                }
+            }
+
+        Thread thread1 = new Thread(readObjectRunnable, "Read Object");
+        Thread thread2 = new Thread(xmlRunnable, "XML");
+
+        thread1.start();
+        thread2.start();
+
+        thread1.join();
+        thread2.join();
+    }
+}

test/java/security/Security/ClassLoaderDeadlock/Deadlock2.sh

+#!/bin/sh
+
+#
+# Copyright 2009 Sun Microsystems, Inc.  All Rights Reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+# CA 95054 USA or visit www.sun.com if you need additional information or
+# have any questions.
+#
+
+
+# @test
+# @bug 6440846
+# @summary make sure we do not deadlock between ExtClassLoader and AppClassLoader
+# @author Valerie Peng
+# @run shell/timeout=20 Deadlock2.sh
+
+# set a few environment variables so that the shell-script can run stand-alone
+# in the source directory
+
+if [ "${TESTSRC}" = "" ] ; then
+   TESTSRC="."
+fi
+
+if [ "${TESTCLASSES}" = "" ] ; then
+   TESTCLASSES="."
+fi
+
+if [ "${TESTJAVA}" = "" ] ; then
+   echo "TESTJAVA not set.  Test cannot execute."
+   echo "FAILED!!!"
+   exit 1
+fi
+
+# set platform-dependent variables
+OS=`uname -s`
+case "$OS" in
+  SunOS )
+    PATHSEP=":"
+    FILESEP="/"
+    ;;
+  Linux )
+    PATHSEP=":"
+    FILESEP="/"
+    ;;
+  Windows* )
+    PATHSEP=";"
+    FILESEP="\\"
+    ;;
+  * )
+    echo "Unrecognized system!"
+    exit 1;
+    ;;
+esac
+
+# remove old class files
+cd ${TESTCLASSES}
+rm -f Deadlock2*.class
+if [ -d testlib ] ; then
+    rm -rf testlib
+fi
+cp -r ${TESTJAVA}${FILESEP}lib${FILESEP}ext testlib
+
+# compile and package the test program
+${TESTJAVA}${FILESEP}bin${FILESEP}javac \
+    -d ${TESTCLASSES} \
+    ${TESTSRC}${FILESEP}CreateSerialized.java \
+    ${TESTSRC}${FILESEP}Deadlock2.java
+
+${TESTJAVA}${FILESEP}bin${FILESEP}jar \
+    -cvf testlib${FILESEP}Deadlock2.jar \
+    Deadlock2*.class
+
+rm Deadlock2*.class
+
+# create serialized object and run the test
+${TESTJAVA}${FILESEP}bin${FILESEP}java CreateSerialized
+${TESTJAVA}${FILESEP}bin${FILESEP}java -Djava.ext.dirs=${TESTCLASSES}${FILESEP}testlib Deadlock2
+STATUS=$?
+
+# clean up
+rm object.tmp CreateSerialized.class
+rm -rf testlib
+exit ${STATUS}