8144732: VM_HeapDumper hits assert with bad dump_len

Reviewed-by: dsamersoff

src/share/vm/runtime/arguments.cpp

@@ -305,6 +305,7 @@ static ObsoleteFlag obsolete_jvm_flags[] = {
   { "UseOldInlining",                JDK_Version::jdk_update(8, 20), JDK_Version::jdk(10) },
   { "AutoShutdownNMT",               JDK_Version::jdk_update(8, 40), JDK_Version::jdk(10) },
   { "CompilationRepeat",             JDK_Version::jdk(8), JDK_Version::jdk(9) },
+  { "SegmentedHeapDumpThreshold",    JDK_Version::jdk_update(8, 252), JDK_Version::jdk(10) },
 #ifdef PRODUCT
   { "DesiredMethodLimit",
                            JDK_Version::jdk_update(7, 2), JDK_Version::jdk(8) },

src/share/vm/services/heapDumper.cpp

@@ -50,8 +50,7 @@
  *   src/share/demo/jvmti/hprof/hprof_io.c
  *
  *
- *  header    "JAVA PROFILE 1.0.1" or "JAVA PROFILE 1.0.2"
- *            (0-terminated)
+ *  header    "JAVA PROFILE 1.0.2" (0-terminated)
  *
  *  u4        size of identifiers. Identifiers are used to represent
  *            UTF8 strings, objects, stack traces, etc. They usually
@@ -382,6 +381,8 @@ class DumpWriter : public StackObj {
   size_t _size;
   size_t _pos;
 
+  jlong _dump_start;
+
   char* _error;   // error message when I/O fails
 
   void set_file_descriptor(int fd)              { _fd = fd; }
@@ -405,6 +406,10 @@ class DumpWriter : public StackObj {
   bool is_open() const                  { return file_descriptor() >= 0; }
   void flush();
 
+  jlong dump_start() const                      { return _dump_start; }
+  void set_dump_start(jlong pos);
+  julong current_record_length();
+
   // total number of bytes written to the disk
   julong bytes_written() const          { return _bytes_written; }
 
@@ -446,6 +451,7 @@ DumpWriter::DumpWriter(const char* path) {
   _pos = 0;
   _error = NULL;
   _bytes_written = 0L;
+  _dump_start = (jlong)-1;
   _fd = os::create_binary_file(path, false);    // don't replace existing file
 
   // if the open failed we record the error
@@ -473,6 +479,22 @@ void DumpWriter::close() {
   }
 }
 
+// sets the dump starting position
+void DumpWriter::set_dump_start(jlong pos) {
+  _dump_start = pos;
+}
+
+julong DumpWriter::current_record_length() {
+  if (is_open()) {
+    // calculate the size of the dump record
+    julong dump_end = bytes_written() + bytes_unwritten();
+    assert(dump_end == (size_t)current_offset(), "checking");
+    julong dump_len = dump_end - dump_start() - 4;
+    return dump_len;
+  }
+  return 0;
+}
+
 // write directly to the file
 void DumpWriter::write_internal(void* s, size_t len) {
   if (is_open()) {
@@ -641,6 +663,18 @@ class DumperSupport : AllStatic {
   static void dump_prim_array(DumpWriter* writer, typeArrayOop array);
   // create HPROF_FRAME record for the given method and bci
   static void dump_stack_frame(DumpWriter* writer, int frame_serial_num, int class_serial_num, Method* m, int bci);
+
+  // check if we need to truncate an array
+  static int calculate_array_max_length(DumpWriter* writer, arrayOop array, short header_size);
+
+  // writes a HPROF_HEAP_DUMP_SEGMENT record
+  static void write_dump_header(DumpWriter* writer);
+
+  // fixes up the length of the current dump record
+  static void write_current_dump_record_length(DumpWriter* writer);
+
+  // fixes up the current dump record and writes HPROF_HEAP_DUMP_END record
+  static void end_of_dump(DumpWriter* writer);
 };
 
 // write a header of the given type
@@ -1047,50 +1081,103 @@ void DumperSupport::dump_basic_type_array_class(DumpWriter* writer, Klass* k) {
   }
 }
 
+// Hprof uses an u4 as record length field,
+// which means we need to truncate arrays that are too long.
+int DumperSupport::calculate_array_max_length(DumpWriter* writer, arrayOop array, short header_size) {
+  BasicType type = ArrayKlass::cast(array->klass())->element_type();
+  assert(type >= T_BOOLEAN && type <= T_OBJECT, "invalid array element type");
+
+  int length = array->length();
+
+  int type_size;
+  if (type == T_OBJECT) {
+    type_size = sizeof(address);
+  } else {
+    type_size = type2aelembytes(type);
+  }
+
+  size_t length_in_bytes = (size_t)length * type_size;
+
+  // Create a new record if the current record is non-empty and the array can't fit.
+  julong current_record_length = writer->current_record_length();
+  if (current_record_length > 0 &&
+      (current_record_length + header_size + length_in_bytes) > max_juint) {
+    write_current_dump_record_length(writer);
+    write_dump_header(writer);
+
+    // We now have an empty record.
+    current_record_length = 0;
+  }
+
+  // Calculate max bytes we can use.
+  uint max_bytes = max_juint - (header_size + current_record_length);
+
+  // Array too long for the record?
+  // Calculate max length and return it.
+  if (length_in_bytes > max_bytes) {
+    length = max_bytes / type_size;
+    length_in_bytes = (size_t)length * type_size;
+
+    warning("cannot dump array of type %s[] with length %d; truncating to length %d",
+            type2name_tab[type], array->length(), length);
+  }
+  return length;
+}
+
 // creates HPROF_GC_OBJ_ARRAY_DUMP record for the given object array
 void DumperSupport::dump_object_array(DumpWriter* writer, objArrayOop array) {
+  // sizeof(u1) + 2 * sizeof(u4) + sizeof(objectID) + sizeof(classID)
+  short header_size = 1 + 2 * 4 + 2 * sizeof(address);
+
+  int length = calculate_array_max_length(writer, array, header_size);
 
   writer->write_u1(HPROF_GC_OBJ_ARRAY_DUMP);
   writer->write_objectID(array);
   writer->write_u4(STACK_TRACE_ID);
-  writer->write_u4((u4)array->length());
+  writer->write_u4(length);
+
 
   // array class ID
   writer->write_classID(array->klass());
 
   // [id]* elements
-  for (int index=0; index<array->length(); index++) {
+  for (int index = 0; index < length; index++) {
     oop o = array->obj_at(index);
     writer->write_objectID(o);
   }
 }
 
-#define WRITE_ARRAY(Array, Type, Size) \
-  for (int i=0; i<Array->length(); i++) { writer->write_##Size((Size)array->Type##_at(i)); }
-
+#define WRITE_ARRAY(Array, Type, Size, Length) \
+  for (int i = 0; i < Length; i++) { writer->write_##Size((Size)array->Type##_at(i)); }
 
 // creates HPROF_GC_PRIM_ARRAY_DUMP record for the given type array
 void DumperSupport::dump_prim_array(DumpWriter* writer, typeArrayOop array) {
   BasicType type = TypeArrayKlass::cast(array->klass())->element_type();
 
+  // 2 * sizeof(u1) + 2 * sizeof(u4) + sizeof(objectID)
+  short header_size = 2 * 1 + 2 * 4 + sizeof(address);
+
+  int length = calculate_array_max_length(writer, array, header_size);
+  int type_size = type2aelembytes(type);
+  u4 length_in_bytes = (u4)length * type_size;
+
   writer->write_u1(HPROF_GC_PRIM_ARRAY_DUMP);
   writer->write_objectID(array);
   writer->write_u4(STACK_TRACE_ID);
-  writer->write_u4((u4)array->length());
+  writer->write_u4(length);
   writer->write_u1(type2tag(type));
 
   // nothing to copy
-  if (array->length() == 0) {
+  if (length == 0) {
     return;
   }
 
   // If the byte ordering is big endian then we can copy most types directly
-  u4 length_in_bytes = (u4)array->length() * type2aelembytes(type);
 
   switch (type) {
     case T_INT : {
       if (Bytes::is_Java_byte_ordering_different()) {
-        WRITE_ARRAY(array, int, u4);
+        WRITE_ARRAY(array, int, u4, length);
       } else {
         writer->write_raw((void*)(array->int_at_addr(0)), length_in_bytes);
       }
@@ -1102,7 +1189,7 @@ void DumperSupport::dump_prim_array(DumpWriter* writer, typeArrayOop array) {
     }
     case T_CHAR : {
       if (Bytes::is_Java_byte_ordering_different()) {
-        WRITE_ARRAY(array, char, u2);
+        WRITE_ARRAY(array, char, u2, length);
       } else {
         writer->write_raw((void*)(array->char_at_addr(0)), length_in_bytes);
       }
@@ -1110,7 +1197,7 @@ void DumperSupport::dump_prim_array(DumpWriter* writer, typeArrayOop array) {
     }
     case T_SHORT : {
       if (Bytes::is_Java_byte_ordering_different()) {
-        WRITE_ARRAY(array, short, u2);
+        WRITE_ARRAY(array, short, u2, length);
       } else {
         writer->write_raw((void*)(array->short_at_addr(0)), length_in_bytes);
       }
@@ -1118,7 +1205,7 @@ void DumperSupport::dump_prim_array(DumpWriter* writer, typeArrayOop array) {
     }
     case T_BOOLEAN : {
       if (Bytes::is_Java_byte_ordering_different()) {
-        WRITE_ARRAY(array, bool, u1);
+        WRITE_ARRAY(array, bool, u1, length);
       } else {
         writer->write_raw((void*)(array->bool_at_addr(0)), length_in_bytes);
       }
@@ -1126,7 +1213,7 @@ void DumperSupport::dump_prim_array(DumpWriter* writer, typeArrayOop array) {
     }
     case T_LONG : {
       if (Bytes::is_Java_byte_ordering_different()) {
-        WRITE_ARRAY(array, long, u8);
+        WRITE_ARRAY(array, long, u8, length);
       } else {
         writer->write_raw((void*)(array->long_at_addr(0)), length_in_bytes);
       }
@@ -1138,14 +1225,14 @@ void DumperSupport::dump_prim_array(DumpWriter* writer, typeArrayOop array) {
     // use IEEE 754.
 
     case T_FLOAT : {
-      for (int i=0; i<array->length(); i++) {
-        dump_float( writer, array->float_at(i) );
+      for (int i = 0; i < length; i++) {
+        dump_float(writer, array->float_at(i));
       }
       break;
     }
     case T_DOUBLE : {
-      for (int i=0; i<array->length(); i++) {
-        dump_double( writer, array->double_at(i) );
+      for (int i = 0; i < length; i++) {
+        dump_double(writer, array->double_at(i));
       }
       break;
     }
@@ -1362,8 +1449,6 @@ class VM_HeapDumper : public VM_GC_Operation {
   JavaThread*           _oome_thread;
   Method*               _oome_constructor;
   bool _gc_before_heap_dump;
-  bool _is_segmented_dump;
-  jlong _dump_start;
   GrowableArray<Klass*>* _klass_map;
   ThreadStackTrace** _stack_traces;
   int _num_threads;
@@ -1382,11 +1467,6 @@ class VM_HeapDumper : public VM_GC_Operation {
   void clear_global_dumper() { _global_dumper = NULL; }
   void clear_global_writer() { _global_writer = NULL; }
 
-  bool is_segmented_dump() const                { return _is_segmented_dump; }
-  void set_segmented_dump()                     { _is_segmented_dump = true; }
-  jlong dump_start() const                      { return _dump_start; }
-  void set_dump_start(jlong pos);
-
   bool skip_operation() const;
 
   // writes a HPROF_LOAD_CLASS record
@@ -1411,16 +1491,6 @@ class VM_HeapDumper : public VM_GC_Operation {
   // HPROF_TRACE and HPROF_FRAME records
   void dump_stack_traces();
 
-  // writes a HPROF_HEAP_DUMP or HPROF_HEAP_DUMP_SEGMENT record
-  void write_dump_header();
-
-  // fixes up the length of the current dump record
-  void write_current_dump_record_length();
-
-  // fixes up the current dump record )and writes HPROF_HEAP_DUMP_END
-  // record in the case of a segmented heap dump)
-  void end_of_dump();
-
  public:
   VM_HeapDumper(DumpWriter* writer, bool gc_before_heap_dump, bool oome) :
     VM_GC_Operation(0 /* total collections,      dummy, ignored */,
@@ -1429,8 +1499,6 @@ class VM_HeapDumper : public VM_GC_Operation {
                     gc_before_heap_dump) {
     _local_writer = writer;
     _gc_before_heap_dump = gc_before_heap_dump;
-    _is_segmented_dump = false;
-    _dump_start = (jlong)-1;
     _klass_map = new (ResourceObj::C_HEAP, mtInternal) GrowableArray<Klass*>(INITIAL_CLASS_COUNT, true);
     _stack_traces = NULL;
     _num_threads = 0;
@@ -1470,35 +1538,23 @@ bool VM_HeapDumper::skip_operation() const {
   return false;
 }
 
-// sets the dump starting position
-void VM_HeapDumper::set_dump_start(jlong pos) {
-  _dump_start = pos;
-}
-
- // writes a HPROF_HEAP_DUMP or HPROF_HEAP_DUMP_SEGMENT record
-void VM_HeapDumper::write_dump_header() {
-  if (writer()->is_open()) {
-    if (is_segmented_dump()) {
-      writer()->write_u1(HPROF_HEAP_DUMP_SEGMENT);
-    } else {
-      writer()->write_u1(HPROF_HEAP_DUMP);
-    }
-    writer()->write_u4(0); // current ticks
+ // writes a HPROF_HEAP_DUMP_SEGMENT record
+void DumperSupport::write_dump_header(DumpWriter* writer) {
+  if (writer->is_open()) {
+    writer->write_u1(HPROF_HEAP_DUMP_SEGMENT);
+    writer->write_u4(0); // current ticks
 
     // record the starting position for the dump (its length will be fixed up later)
-    set_dump_start(writer()->current_offset());
-    writer()->write_u4(0);
+    writer->set_dump_start(writer->current_offset());
+    writer->write_u4(0);
   }
 }
 
 // fixes up the length of the current dump record
-void VM_HeapDumper::write_current_dump_record_length() {
-  if (writer()->is_open()) {
-    assert(dump_start() >= 0, "no dump start recorded");
-
-    // calculate the size of the dump record
-    julong dump_end = writer()->current_offset();
-    julong dump_len = (dump_end - dump_start() - 4);
+void DumperSupport::write_current_dump_record_length(DumpWriter* writer) {
+  if (writer->is_open()) {
+    julong dump_end = writer->bytes_written() + writer->bytes_unwritten();
+    julong dump_len = writer->current_record_length();
 
     // record length must fit in a u4
     if (dump_len > max_juint) {
@@ -1506,17 +1562,18 @@ void VM_HeapDumper::write_current_dump_record_length() {
     }
 
     // seek to the dump start and fix-up the length
-    writer()->seek_to_offset(dump_start());
-    writer()->write_u4((u4)dump_len);
+    assert(writer->dump_start() >= 0, "no dump start recorded");
+    writer->seek_to_offset(writer->dump_start());
+    writer->write_u4((u4)dump_len);
 
     // adjust the total size written to keep the bytes written correct.
-    writer()->adjust_bytes_written(-((jlong) sizeof(u4)));
+    writer->adjust_bytes_written(-((jlong) sizeof(u4)));
 
     // seek to dump end so we can continue
-    writer()->seek_to_offset(dump_end);
+    writer->seek_to_offset(dump_end);
 
     // no current dump record
-    set_dump_start((jlong)-1);
+    writer->set_dump_start((jlong)-1);
   }
 }
 
@@ -1524,33 +1581,23 @@ void VM_HeapDumper::write_current_dump_record_length() {
 // new segment.
 void VM_HeapDumper::check_segment_length() {
   if (writer()->is_open()) {
-    if (is_segmented_dump()) {
-      // don't use current_offset that would be too expensive on a per record basis
-      julong dump_end = writer()->bytes_written() + writer()->bytes_unwritten();
-      assert(dump_end == (julong)writer()->current_offset(), "checking");
-      julong dump_len = (dump_end - dump_start() - 4);
-      assert(dump_len <= max_juint, "bad dump length");
-
-      if (dump_len > HeapDumpSegmentSize) {
-        write_current_dump_record_length();
-        write_dump_header();
-      }
+    julong dump_len = writer()->current_record_length();
+
+    if (dump_len > 2UL*G) {
+      DumperSupport::write_current_dump_record_length(writer());
+      DumperSupport::write_dump_header(writer());
     }
   }
 }
 
-// fixes up the current dump record )and writes HPROF_HEAP_DUMP_END
-// record in the case of a segmented heap dump)
-void VM_HeapDumper::end_of_dump() {
-  if (writer()->is_open()) {
-    write_current_dump_record_length();
+// fixes up the current dump record and writes HPROF_HEAP_DUMP_END record
+void DumperSupport::end_of_dump(DumpWriter* writer) {
+  if (writer->is_open()) {
+    write_current_dump_record_length(writer);
 
-    // for segmented dump we write the end record
-    if (is_segmented_dump()) {
-      writer()->write_u1(HPROF_HEAP_DUMP_END);
-      writer()->write_u4(0);
-      writer()->write_u4(0);
-    }
+    writer->write_u1(HPROF_HEAP_DUMP_END);
+    writer->write_u4(0);
+    writer->write_u4(0);
   }
 }
 
@@ -1716,16 +1763,17 @@ void VM_HeapDumper::do_threads() {
 //  [HPROF_LOAD_CLASS]*
 //  [[HPROF_FRAME]*|HPROF_TRACE]*
 //  [HPROF_GC_CLASS_DUMP]*
-//  HPROF_HEAP_DUMP
+//  [HPROF_HEAP_DUMP_SEGMENT]*
+//  HPROF_HEAP_DUMP_END
 //
 // The HPROF_TRACE records represent the stack traces where the heap dump
 // is generated and a "dummy trace" record which does not include
 // any frames. The dummy trace record is used to be referenced as the
 // unknown object alloc site.
 //
-// The HPROF_HEAP_DUMP record has a length following by sub-records. To allow
-// the heap dump be generated in a single pass we remember the position of
-// the dump length and fix it up after all sub-records have been written.
+// Each HPROF_HEAP_DUMP_SEGMENT record has a length followed by sub-records.
+// To allow the heap dump be generated in a single pass we remember the position
+// of the dump length and fix it up after all sub-records have been written.
 // To generate the sub-records we iterate over the heap, writing
 // HPROF_GC_INSTANCE_DUMP, HPROF_GC_OBJ_ARRAY_DUMP, and HPROF_GC_PRIM_ARRAY_DUMP
 // records as we go. Once that is done we write records for some of the GC
@@ -1752,15 +1800,9 @@ void VM_HeapDumper::doit() {
   set_global_dumper();
   set_global_writer();
 
-  // Write the file header - use 1.0.2 for large heaps, otherwise 1.0.1
+  // Write the file header - we always use 1.0.2
   size_t used = ch->used();
-  const char* header;
-  if (used > (size_t)SegmentedHeapDumpThreshold) {
-    set_segmented_dump();
-    header = "JAVA PROFILE 1.0.2";
-  } else {
-    header = "JAVA PROFILE 1.0.1";
-  }
+  const char* header = "JAVA PROFILE 1.0.2";
 
   // header is few bytes long - no chance to overflow int
   writer()->write_raw((void*)header, (int)strlen(header));
@@ -1780,8 +1822,8 @@ void VM_HeapDumper::doit() {
   // this must be called after _klass_map is built when iterating the classes above.
   dump_stack_traces();
 
-  // write HPROF_HEAP_DUMP or HPROF_HEAP_DUMP_SEGMENT
-  write_dump_header();
+  // write HPROF_HEAP_DUMP_SEGMENT
+  DumperSupport::write_dump_header(writer());
 
   // Writes HPROF_GC_CLASS_DUMP records
   ClassLoaderDataGraph::classes_do(&do_class_dump);
@@ -1789,9 +1831,9 @@ void VM_HeapDumper::doit() {
   check_segment_length();
 
   // writes HPROF_GC_INSTANCE_DUMP records.
-  // After each sub-record is written check_segment_length will be invoked. When
-  // generated a segmented heap dump this allows us to check if the current
-  // segment exceeds a threshold and if so, then a new segment is started.
+  // After each sub-record is written check_segment_length will be invoked
+  // to check if the current segment exceeds a threshold. If so, a new
+  // segment is started.
   // The HPROF_GC_CLASS_DUMP and HPROF_GC_INSTANCE_DUMP are the vast bulk
   // of the heap dump.
   HeapObjectDumper obj_dumper(this, writer());
@@ -1817,9 +1859,8 @@ void VM_HeapDumper::doit() {
   StickyClassDumper class_dumper(writer());
   SystemDictionary::always_strong_classes_do(&class_dumper);
 
-  // fixes up the length of the dump record. In the case of a segmented
-  // heap then the HPROF_HEAP_DUMP_END record is also written.
-  end_of_dump();
+  // fixes up the length of the dump record and writes the HPROF_HEAP_DUMP_END record.
+  DumperSupport::end_of_dump(writer());
 
   // Now we clear the global variables, so that a future dumper might run.
   clear_global_dumper();