8241522: Manifest improved jar headers redux

Reviewed-by: sspitsyn, jwilhelm, mschoene, rhalade

src/java.instrument/share/native/libinstrument/EncodingSupport.c

@@ -38,7 +38,11 @@ modifiedUtf8LengthOfUtf8(char* string, int length) {
     int i;
 
     new_length = 0;
-    for ( i = 0 ; i < length ; i++ ) {
+    /*
+     * if length < 0 or new_length becomes < 0 => string is too big
+     * (handled as error after the cycle).
+     */
+    for ( i = 0 ; i < length && new_length >= 0 ; i++ ) {
         unsigned byte;
 
         byte = (unsigned char)string[i];

src/java.instrument/share/native/libinstrument/InvocationAdapter.c

@@ -203,8 +203,10 @@ DEF_Agent_OnLoad(JavaVM *vm, char *tail, void * reserved) {
         /*
          * According to JVMS class name is represented as CONSTANT_Utf8_info,
          * so its length is u2 (i.e. must be <= 0xFFFF).
+         * Negative oldLen or newLen means we got signed integer overflow
+         * (modifiedUtf8LengthOfUtf8 returns negative value if oldLen is negative).
          */
-        if (newLen > 0xFFFF) {
+        if (oldLen < 0 || newLen < 0 || newLen > 0xFFFF) {
             fprintf(stderr, "-javaagent: Premain-Class value is too big\n");
             free(jarfile);
             if (options != NULL) free(options);
@@ -372,8 +374,10 @@ DEF_Agent_OnAttach(JavaVM* vm, char *args, void * reserved) {
         /*
          * According to JVMS class name is represented as CONSTANT_Utf8_info,
          * so its length is u2 (i.e. must be <= 0xFFFF).
+         * Negative oldLen or newLen means we got signed integer overflow
+         * (modifiedUtf8LengthOfUtf8 returns negative value if oldLen is negative).
          */
-        if (newLen > 0xFFFF) {
+        if (oldLen < 0 || newLen < 0 || newLen > 0xFFFF) {
             fprintf(stderr, "Agent-Class value is too big\n");
             free(jarfile);
             if (options != NULL) free(options);
@@ -508,8 +512,10 @@ jint loadAgent(JNIEnv* env, jstring path) {
     /*
      * According to JVMS class name is represented as CONSTANT_Utf8_info,
      * so its length is u2 (i.e. must be <= 0xFFFF).
+     * Negative oldLen or newLen means we got signed integer overflow
+     * (modifiedUtf8LengthOfUtf8 returns negative value if oldLen is negative).
      */
-    if (newLen > 0xFFFF) {
+    if (oldLen < 0 || newLen < 0 || newLen > 0xFFFF) {
         goto releaseAndReturn;
     }
     if (newLen == oldLen) {
@@ -554,16 +560,16 @@ jint loadAgent(JNIEnv* env, jstring path) {
     // initialization complete
     result = JNI_OK;
 
-    releaseAndReturn:
-        if (agentClass != NULL) {
-            free(agentClass);
-        }
-        if (attributes != NULL) {
-            freeAttributes(attributes);
-        }
-        if (jarfile != NULL) {
-            (*env)->ReleaseStringUTFChars(env, path, jarfile);
-        }
+releaseAndReturn:
+    if (agentClass != NULL) {
+        free(agentClass);
+    }
+    if (attributes != NULL) {
+        freeAttributes(attributes);
+    }
+    if (jarfile != NULL) {
+        (*env)->ReleaseStringUTFChars(env, path, jarfile);
+    }
 
     return result;
 }