8163958: Improved garbage collection

Reviewed-by: rriggs

make/GenerateClasses.gmk

@@ -79,8 +79,7 @@ $(eval $(call SetupRMICompilation,RMI_12, \
 GENCLASSES += $(RMI_12)
 
 $(eval $(call SetupRMICompilation,RMI_11, \
-    CLASSES := sun.rmi.registry.RegistryImpl \
-        sun.rmi.transport.DGCImpl, \
+    CLASSES := sun.rmi.registry.RegistryImpl, \
     CLASSES_DIR := $(CLASSES_DIR), \
     STUB_CLASSES_DIR := $(STUB_CLASSES_DIR), \
     RUN_V11 := true))

src/share/classes/sun/rmi/server/UnicastRef.java

 /*
- * Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,6 @@ import java.lang.reflect.Method;
 import java.rmi.MarshalException;
 import java.rmi.Remote;
 import java.rmi.RemoteException;
-import java.rmi.ServerException;
 import java.rmi.UnmarshalException;
 import java.rmi.server.Operation;
 import java.rmi.server.RemoteCall;
@@ -187,14 +186,11 @@ public class UnicastRef implements RemoteRef {
 
                 return returnValue;
 
-            } catch (IOException e) {
-                clientRefLog.log(Log.BRIEF,
-                                 "IOException unmarshalling return: ", e);
-                throw new UnmarshalException("error unmarshalling return", e);
-            } catch (ClassNotFoundException e) {
+            } catch (IOException | ClassNotFoundException e) {
+                // disable saving any refs in the inputStream for GC
+                ((StreamRemoteCall)call).discardPendingRefs();
                 clientRefLog.log(Log.BRIEF,
-                    "ClassNotFoundException unmarshalling return: ", e);
-
+                                 e.getClass().getName() + " unmarshalling return: ", e);
                 throw new UnmarshalException("error unmarshalling return", e);
             } finally {
                 try {

src/share/classes/sun/rmi/server/UnicastServerRef.java

 /*
- * Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -57,6 +57,7 @@ import java.util.concurrent.atomic.AtomicInteger;
 import sun.misc.ObjectInputFilter;
 import sun.rmi.runtime.Log;
 import sun.rmi.transport.LiveRef;
+import sun.rmi.transport.StreamRemoteCall;
 import sun.rmi.transport.Target;
 import sun.rmi.transport.tcp.TCPTransport;
 import sun.security.action.GetBooleanAction;
@@ -330,10 +331,9 @@ public class UnicastServerRef extends UnicastRef
             try {
                 unmarshalCustomCallData(in);
                 params = unmarshalParameters(obj, method, marshalStream);
-            } catch (java.io.IOException e) {
-                throw new UnmarshalException(
-                    "error unmarshalling arguments", e);
-            } catch (ClassNotFoundException e) {
+            } catch (java.io.IOException | ClassNotFoundException e) {
+                // disable saving any refs in the inputStream for GC
+                ((StreamRemoteCall) call).discardPendingRefs();
                 throw new UnmarshalException(
                     "error unmarshalling arguments", e);
             } finally {

src/share/classes/sun/rmi/transport/ConnectionInputStream.java

 /*
- * Copyright (c) 1996, 2012, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,6 +81,14 @@ class ConnectionInputStream extends MarshalInputStream {
         refList.add(ref);
     }
 
+    /**
+     * Discard the saved incoming refs so there is nothing to register
+     * when {@code registerRefs} is called.
+     */
+    void discardRefs() {
+        incomingRefTable.clear();
+    }
+
     /**
      * Add references to DGC table (and possibly send dirty call).
      * RegisterRefs now calls DGCClient.referenced on all

src/share/classes/sun/rmi/transport/DGCClient.java

@@ -24,9 +24,11 @@
  */
 package sun.rmi.transport;
 
+import java.io.InvalidClassException;
 import java.lang.ref.PhantomReference;
 import java.lang.ref.ReferenceQueue;
 import java.net.SocketPermission;
+import java.rmi.UnmarshalException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.HashMap;
@@ -41,7 +43,9 @@ import java.rmi.dgc.DGC;
 import java.rmi.dgc.Lease;
 import java.rmi.dgc.VMID;
 import java.rmi.server.ObjID;
+
 import sun.misc.GC;
+import sun.rmi.runtime.Log;
 import sun.rmi.runtime.NewThreadAction;
 import sun.rmi.server.UnicastRef;
 import sun.rmi.server.Util;
@@ -393,6 +397,12 @@ final class DGCClient {
                 synchronized (this) {
                     dirtyFailures++;
 
+                    if (e instanceof UnmarshalException
+                            && e.getCause() instanceof InvalidClassException) {
+                        DGCImpl.dgcLog.log(Log.BRIEF, "InvalidClassException exception in DGC dirty call", e);
+                        return;             // protocol error, do not register these refs
+                    }
+
                     if (dirtyFailures == 1) {
                         /*
                          * If this was the first recent failed dirty call,

src/share/classes/sun/rmi/transport/DGCImpl_Skel.java

+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.rmi.transport;
+
+/**
+ * Skeleton to dispatch DGC methods.
+ * Originally generated by RMIC but frozen to match the stubs.
+ */
+@SuppressWarnings({"deprecation", "serial"})
+public final class DGCImpl_Skel
+        implements java.rmi.server.Skeleton {
+    private static final java.rmi.server.Operation[] operations = {
+            new java.rmi.server.Operation("void clean(java.rmi.server.ObjID[], long, java.rmi.dgc.VMID, boolean)"),
+            new java.rmi.server.Operation("java.rmi.dgc.Lease dirty(java.rmi.server.ObjID[], long, java.rmi.dgc.Lease)")
+    };
+
+    private static final long interfaceHash = -669196253586618813L;
+
+    public java.rmi.server.Operation[] getOperations() {
+        return operations.clone();
+    }
+
+    public void dispatch(java.rmi.Remote obj, java.rmi.server.RemoteCall call, int opnum, long hash)
+            throws java.lang.Exception {
+        if (hash != interfaceHash)
+            throw new java.rmi.server.SkeletonMismatchException("interface hash mismatch");
+
+        sun.rmi.transport.DGCImpl server = (sun.rmi.transport.DGCImpl) obj;
+        switch (opnum) {
+            case 0: // clean(ObjID[], long, VMID, boolean)
+            {
+                java.rmi.server.ObjID[] $param_arrayOf_ObjID_1;
+                long $param_long_2;
+                java.rmi.dgc.VMID $param_VMID_3;
+                boolean $param_boolean_4;
+                try {
+                    java.io.ObjectInput in = call.getInputStream();
+                    $param_arrayOf_ObjID_1 = (java.rmi.server.ObjID[]) in.readObject();
+                    $param_long_2 = in.readLong();
+                    $param_VMID_3 = (java.rmi.dgc.VMID) in.readObject();
+                    $param_boolean_4 = in.readBoolean();
+                } catch (java.io.IOException e) {
+                    throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
+                } catch (java.lang.ClassNotFoundException e) {
+                    throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
+                } finally {
+                    call.releaseInputStream();
+                }
+                server.clean($param_arrayOf_ObjID_1, $param_long_2, $param_VMID_3, $param_boolean_4);
+                try {
+                    call.getResultStream(true);
+                } catch (java.io.IOException e) {
+                    throw new java.rmi.MarshalException("error marshalling return", e);
+                }
+                break;
+            }
+
+            case 1: // dirty(ObjID[], long, Lease)
+            {
+                java.rmi.server.ObjID[] $param_arrayOf_ObjID_1;
+                long $param_long_2;
+                java.rmi.dgc.Lease $param_Lease_3;
+                try {
+                    java.io.ObjectInput in = call.getInputStream();
+                    $param_arrayOf_ObjID_1 = (java.rmi.server.ObjID[]) in.readObject();
+                    $param_long_2 = in.readLong();
+                    $param_Lease_3 = (java.rmi.dgc.Lease) in.readObject();
+                } catch (java.io.IOException e) {
+                    throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
+                } catch (java.lang.ClassNotFoundException e) {
+                    throw new java.rmi.UnmarshalException("error unmarshalling arguments", e);
+                } finally {
+                    call.releaseInputStream();
+                }
+                java.rmi.dgc.Lease $result = server.dirty($param_arrayOf_ObjID_1, $param_long_2, $param_Lease_3);
+                try {
+                    java.io.ObjectOutput out = call.getResultStream(true);
+                    out.writeObject($result);
+                } catch (java.io.IOException e) {
+                    throw new java.rmi.MarshalException("error marshalling return", e);
+                }
+                break;
+            }
+
+            default:
+                throw new java.rmi.UnmarshalException("invalid method number");
+        }
+    }
+}

src/share/classes/sun/rmi/transport/DGCImpl_Stub.java

+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package sun.rmi.transport;
+
+import java.io.ObjectInputStream;
+import java.rmi.dgc.Lease;
+import java.rmi.dgc.VMID;
+import java.rmi.server.UID;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+
+import sun.misc.ObjectInputFilter;
+import sun.rmi.server.UnicastRef;
+import sun.rmi.transport.tcp.TCPConnection;
+
+/**
+ * Stubs to invoke DGC remote methods.
+ * Originally generated from RMIC but frozen to insert serialFilter.
+ */
+@SuppressWarnings({"deprecation", "serial"})
+public final class DGCImpl_Stub
+        extends java.rmi.server.RemoteStub
+        implements java.rmi.dgc.DGC {
+    private static final java.rmi.server.Operation[] operations = {
+            new java.rmi.server.Operation("void clean(java.rmi.server.ObjID[], long, java.rmi.dgc.VMID, boolean)"),
+            new java.rmi.server.Operation("java.rmi.dgc.Lease dirty(java.rmi.server.ObjID[], long, java.rmi.dgc.Lease)")
+    };
+
+    private static final long interfaceHash = -669196253586618813L;
+
+    /** Registry max depth of remote invocations. **/
+    private static int DGCCLIENT_MAX_DEPTH = 6;
+
+    /** Registry maximum array size in remote invocations. **/
+    private static int DGCCLIENT_MAX_ARRAY_SIZE = 10000;
+
+    // constructors
+    public DGCImpl_Stub() {
+        super();
+    }
+
+    public DGCImpl_Stub(java.rmi.server.RemoteRef ref) {
+        super(ref);
+    }
+
+    // methods from remote interfaces
+
+    // implementation of clean(ObjID[], long, VMID, boolean)
+    public void clean(java.rmi.server.ObjID[] $param_arrayOf_ObjID_1, long $param_long_2, java.rmi.dgc.VMID $param_VMID_3, boolean $param_boolean_4)
+            throws java.rmi.RemoteException {
+        try {
+            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 0, interfaceHash);
+            try {
+                java.io.ObjectOutput out = call.getOutputStream();
+                out.writeObject($param_arrayOf_ObjID_1);
+                out.writeLong($param_long_2);
+                out.writeObject($param_VMID_3);
+                out.writeBoolean($param_boolean_4);
+            } catch (java.io.IOException e) {
+                throw new java.rmi.MarshalException("error marshalling arguments", e);
+            }
+            ref.invoke(call);
+            ref.done(call);
+        } catch (java.lang.RuntimeException e) {
+            throw e;
+        } catch (java.rmi.RemoteException e) {
+            throw e;
+        } catch (java.lang.Exception e) {
+            throw new java.rmi.UnexpectedException("undeclared checked exception", e);
+        }
+    }
+
+    // implementation of dirty(ObjID[], long, Lease)
+    public java.rmi.dgc.Lease dirty(java.rmi.server.ObjID[] $param_arrayOf_ObjID_1, long $param_long_2, java.rmi.dgc.Lease $param_Lease_3)
+            throws java.rmi.RemoteException {
+        try {
+            java.rmi.server.RemoteCall call = ref.newCall((java.rmi.server.RemoteObject) this, operations, 1, interfaceHash);
+            try {
+                java.io.ObjectOutput out = call.getOutputStream();
+                out.writeObject($param_arrayOf_ObjID_1);
+                out.writeLong($param_long_2);
+                out.writeObject($param_Lease_3);
+            } catch (java.io.IOException e) {
+                throw new java.rmi.MarshalException("error marshalling arguments", e);
+            }
+            ref.invoke(call);
+            java.rmi.dgc.Lease $result;
+            Connection connection = ((StreamRemoteCall) call).getConnection();
+            try {
+                java.io.ObjectInput in = call.getInputStream();
+
+                if (in instanceof ObjectInputStream) {
+                    /**
+                     * Set a filter on the stream for the return value.
+                     */
+                    ObjectInputStream ois = (ObjectInputStream) in;
+                    AccessController.doPrivileged((PrivilegedAction<Void>)() -> {
+                        ObjectInputFilter.Config.setObjectInputFilter(ois, DGCImpl_Stub::leaseFilter);
+                        return null;
+                    });
+                }
+                $result = (java.rmi.dgc.Lease) in.readObject();
+            } catch (java.io.IOException | java.lang.ClassNotFoundException e) {
+                if (connection instanceof TCPConnection) {
+                    // Modified to prevent re-use of the connection after an exception
+                    ((TCPConnection) connection).getChannel().free(connection, false);
+                }
+                throw new java.rmi.UnmarshalException("error unmarshalling return", e);
+            } finally {
+                ref.done(call);
+            }
+            return $result;
+        } catch (java.lang.RuntimeException e) {
+            throw e;
+        } catch (java.rmi.RemoteException e) {
+            throw e;
+        } catch (java.lang.Exception e) {
+            throw new java.rmi.UnexpectedException("undeclared checked exception", e);
+        }
+    }
+
+    /**
+     * ObjectInputFilter to filter DGCClient return value (a Lease).
+     * The list of acceptable classes is very short and explicit.
+     * The depth and array sizes are limited.
+     *
+     * @param filterInfo access to class, arrayLength, etc.
+     * @return  {@link ObjectInputFilter.Status#ALLOWED} if allowed,
+     *          {@link ObjectInputFilter.Status#REJECTED} if rejected,
+     *          otherwise {@link ObjectInputFilter.Status#UNDECIDED}
+     */
+    private static ObjectInputFilter.Status leaseFilter(ObjectInputFilter.FilterInfo filterInfo) {
+
+        if (filterInfo.depth() > DGCCLIENT_MAX_DEPTH) {
+            return ObjectInputFilter.Status.REJECTED;
+        }
+        Class<?> clazz = filterInfo.serialClass();
+        if (clazz != null) {
+            while (clazz.isArray()) {
+                if (filterInfo.arrayLength() >= 0 && filterInfo.arrayLength() > DGCCLIENT_MAX_ARRAY_SIZE) {
+                    return ObjectInputFilter.Status.REJECTED;
+                }
+                // Arrays are allowed depending on the component type
+                clazz = clazz.getComponentType();
+            }
+            if (clazz.isPrimitive()) {
+                // Arrays of primitives are allowed
+                return ObjectInputFilter.Status.ALLOWED;
+            }
+            return (clazz == UID.class ||
+                    clazz == VMID.class ||
+                    clazz == Lease.class)
+                    ? ObjectInputFilter.Status.ALLOWED
+                    : ObjectInputFilter.Status.REJECTED;
+        }
+        // Not a class, not size limited
+        return ObjectInputFilter.Status.UNDECIDED;
+    }
+
+}

src/share/classes/sun/rmi/transport/StreamRemoteCall.java

 /*
- * Copyright (c) 1996, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -167,6 +167,13 @@ public class StreamRemoteCall implements RemoteCall {
         }
     }
 
+    /**
+     * Discard any post-processing of refs the InputStream.
+     */
+    public void discardPendingRefs() {
+        in.discardRefs();
+    }
+
     /**
      * Returns an output stream (may put out header information
      * relating to the success of the call).