Add support for oops embedded in nullable flat fields

src/hotspot/share/classfile/classFileParser.cpp

@@ -1560,9 +1560,21 @@ void ClassFileParser::parse_fields(const ClassFileStream* const cfs,
                  (u2)vmSymbols::as_int(VM_SYMBOL_ENUM_NAME(object_signature)),
                  0,
                  fflags);
-      int idx = _temp_field_info->append(fi);
-      _temp_field_info->adr_at(idx)->set_index(idx);
-      _static_oop_count++;
+    int idx = _temp_field_info->append(fi);
+    _temp_field_info->adr_at(idx)->set_index(idx);
+    _static_oop_count++;
+    // Inject static ".reset" field
+    FieldInfo::FieldFlags fflags2(0);
+    fflags2.update_injected(true);
+    AccessFlags aflags2(JVM_ACC_STATIC);
+    FieldInfo fi2(aflags2,
+                 (u2)vmSymbols::as_int(VM_SYMBOL_ENUM_NAME(reset_value_name)),
+                 (u2)vmSymbols::as_int(VM_SYMBOL_ENUM_NAME(object_signature)),
+                 0,
+                 fflags2);
+    int idx2 = _temp_field_info->append(fi2);
+    _temp_field_info->adr_at(idx2)->set_index(idx2);
+    _static_oop_count++;
   }
 
   if (_need_verify && length > 1) {
@@ -5608,6 +5620,7 @@ void ClassFileParser::fill_instance_klass(InstanceKlass* ik,
     vk->set_nullable_size_in_bytes(_layout_info->_nullable_layout_size_in_bytes);
     vk->set_null_marker_offset(_layout_info->_null_marker_offset);
     vk->set_default_value_offset(_layout_info->_default_value_offset);
+    vk->set_reset_value_offset(_layout_info->_reset_value_offset);
     if (_layout_info->_is_empty_inline_klass) vk->set_is_empty_inline_type();
     vk->initialize_calling_convention(CHECK);
   }

src/hotspot/share/classfile/classFileParser.hpp

@@ -83,6 +83,7 @@ class FieldLayoutInfo : public ResourceObj {
   int _nullable_layout_size_in_bytes;
   int _null_marker_offset;
   int _default_value_offset;
+  int _reset_value_offset;
   bool _has_nonstatic_fields;
   bool _is_naturally_atomic;
   bool _must_be_atomic;

src/hotspot/share/classfile/fieldLayoutBuilder.cpp

@@ -199,6 +199,7 @@ FieldLayout::FieldLayout(GrowableArray<FieldInfo>* field_info, Array<InlineLayou
   _super_alignment(-1),
   _super_min_align_required(-1),
   _default_value_offset(-1),
+  _reset_value_offset(-1),
   _super_has_fields(false),
   _has_inherited_fields(false) {}
 
@@ -391,6 +392,9 @@ LayoutRawBlock* FieldLayout::insert_field_block(LayoutRawBlock* slot, LayoutRawB
     if (_field_info->adr_at(block->field_index())->name(_cp) == vmSymbols::default_value_name()) {
       _default_value_offset = block->offset();
     }
+    if (_field_info->adr_at(block->field_index())->name(_cp) == vmSymbols::reset_value_name()) {
+      _reset_value_offset = block->offset();
+    }
   }
   if (block->block_kind() == LayoutRawBlock::FLAT && block->layout_kind() == LayoutKind::NULLABLE_FLAT) {
     int nm_offset = block->inline_klass()->null_marker_offset() - block->inline_klass()->first_field_offset() + block->offset();
@@ -1133,8 +1137,7 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
     if (AtomicFieldFlattening) {
       int atomic_size = _payload_size_in_bytes == 0 ? 0 : round_up_power_of_2(_payload_size_in_bytes);
       if (  atomic_size <= (int)MAX_ATOMIC_OP_SIZE
-          && (InlineFieldMaxFlatSize < 0 || atomic_size * BitsPerByte <= InlineFieldMaxFlatSize)
-          && _nonstatic_oopmap_count == 0) {
+          && (InlineFieldMaxFlatSize < 0 || atomic_size * BitsPerByte <= InlineFieldMaxFlatSize)) {
         _atomic_layout_size_in_bytes = atomic_size;
       }
     }
@@ -1176,8 +1179,7 @@ void FieldLayoutBuilder::compute_inline_class_layout() {
       int new_raw_size = _layout->last_block()->offset() - _layout->first_field_block()->offset();
       int nullable_size = round_up_power_of_2(new_raw_size);
       if (nullable_size <= (int)MAX_ATOMIC_OP_SIZE
-        && (InlineFieldMaxFlatSize < 0 || nullable_size * BitsPerByte <= InlineFieldMaxFlatSize)
-        && _nonstatic_oopmap_count == 0) {
+        && (InlineFieldMaxFlatSize < 0 || nullable_size * BitsPerByte <= InlineFieldMaxFlatSize)) {
         _nullable_layout_size_in_bytes = nullable_size;
         _null_marker_offset = null_marker_offset;
       } else {
@@ -1321,6 +1323,7 @@ void FieldLayoutBuilder::epilogue() {
     _info->_nullable_layout_size_in_bytes = _nullable_layout_size_in_bytes;
     _info->_null_marker_offset = _null_marker_offset;
     _info->_default_value_offset = _static_layout->default_value_offset();
+    _info->_reset_value_offset = _static_layout->reset_value_offset();
     _info->_is_empty_inline_klass = _is_empty_inline_class;
   }
 

src/hotspot/share/classfile/fieldLayoutBuilder.hpp

@@ -203,6 +203,7 @@ class FieldLayout : public ResourceObj {
   int _super_alignment;
   int _super_min_align_required;
   int _default_value_offset;  // offset of the default value in class mirror, only for static layout of inline classes
+  int _reset_value_offset;    // offset of the reset value in class mirror, only for static layout of inline classes
   bool _super_has_fields;
   bool _has_inherited_fields;
 
@@ -231,6 +232,10 @@ class FieldLayout : public ResourceObj {
     assert(_default_value_offset != -1, "Must have been set");
     return _default_value_offset;
   }
+  int reset_value_offset() const {
+    assert(_reset_value_offset != -1, "Must have been set");
+    return _reset_value_offset;
+  }
   bool super_has_fields() const { return _super_has_fields; }
   bool has_inherited_fields() const { return _has_inherited_fields; }
 
@@ -341,7 +346,6 @@ class FieldLayoutBuilder : public ResourceObj {
 
   int null_marker_offset() const { return _null_marker_offset; }
   bool is_empty_inline_class() const { return _is_empty_inline_class; }
-  int default_value_offset() const { return _static_layout->default_value_offset(); }
 
   void build_layout();
   void compute_regular_layout();

src/hotspot/share/classfile/vmSymbols.hpp

@@ -541,6 +541,7 @@ class SerializeClosure;
   template(resolved_references_name,                  "<resolved_references>")                    \
   template(init_lock_name,                            "<init_lock>")                              \
   template(default_value_name,                        ".default")                                 \
+  template(reset_value_name,                          ".reset")                                   \
   template(empty_marker_name,                         ".empty")                                   \
   template(address_size_name,                         "ADDRESS_SIZE0")                            \
   template(page_size_name,                            "PAGE_SIZE")                                \

src/hotspot/share/interpreter/interpreterRuntime.cpp

@@ -347,16 +347,23 @@ JRT_ENTRY(void, InterpreterRuntime::write_nullable_flat_field(JavaThread* curren
 
   InstanceKlass* holder = entry->field_holder();
   InlineLayoutInfo* li = holder->inline_layout_info_adr(entry->field_index());
+  InlineKlass* vk = li->klass();
   assert(li->kind() == LayoutKind::NULLABLE_FLAT, "Must be");
   int nm_offset = li->null_marker_offset();
 
   if (val_h() == nullptr) {
-    assert(li->klass()->nonstatic_oop_count() == 0, "Code below is valid only if flat field contains no oops");
-    obj_h()->byte_field_put(nm_offset, (jbyte)0);
+    if(li->klass()->nonstatic_oop_count() == 0) {
+      // No embedded oops, just reset the null marker
+      obj_h()->byte_field_put(nm_offset, (jbyte)0);
+    } else {
+      // Has embedded oops, using the reset value to rewrite all fields to null/zeros
+      assert(li->klass()->reset_value()->byte_field(vk->null_marker_offset()) == 0, "reset value must always have a null marker set to 0");
+      vk->inline_copy_oop_to_payload(vk->reset_value(), ((char*)(oopDesc*)obj_h()) + entry->field_offset(), li->kind());
+    }
     return;
   }
 
-  InlineKlass* vk = InlineKlass::cast(val_h()->klass());
+  assert(val_h()->klass() == vk, "Must match because flat fields are monomorphic");
   // The interpreter copies values with a bulk operation
   // To avoid accidentally setting the null marker to "null" during
   // the copying, the null marker is set to non zero in the source object

src/hotspot/share/oops/inlineKlass.cpp

@@ -73,6 +73,7 @@ void InlineKlass::init_fixed_block() {
   *((address*)adr_unpack_handler()) = nullptr;
   assert(pack_handler() == nullptr, "pack handler not null");
   *((int*)adr_default_value_offset()) = 0;
+  *((int*)adr_reset_value_offset()) = 0;
   *((address*)adr_value_array_klasses()) = nullptr;
   *((int*)adr_first_field_offset()) = -1;
   *((int*)adr_payload_size_in_bytes()) = -1;
@@ -84,28 +85,20 @@ void InlineKlass::init_fixed_block() {
   *((int*)adr_null_marker_offset()) = -1;
 }
 
-oop InlineKlass::default_value() {
-  assert(is_initialized() || is_being_initialized() || is_in_error_state(), "default value is set at the beginning of initialization");
-  oop val = java_mirror()->obj_field_acquire(default_value_offset());
+void InlineKlass::set_default_value(oop val) {
   assert(val != nullptr, "Sanity check");
   assert(oopDesc::is_oop(val), "Sanity check");
   assert(val->is_inline_type(), "Sanity check");
   assert(val->klass() == this, "sanity check");
-  return val;
+  java_mirror()->obj_field_put(default_value_offset(), val);
 }
 
-int InlineKlass::first_field_offset_old() {
-#ifdef ASSERT
-  int first_offset = INT_MAX;
-  for (AllFieldStream fs(this); !fs.done(); fs.next()) {
-    if (fs.offset() < first_offset) first_offset= fs.offset();
-  }
-#endif
-  int base_offset = instanceOopDesc::base_offset_in_bytes();
-  // The first field of line types is aligned on a long boundary
-  base_offset = align_up(base_offset, BytesPerLong);
-  assert(base_offset == first_offset, "inconsistent offsets");
-  return base_offset;
+void InlineKlass::set_reset_value(oop val) {
+  assert(val != nullptr, "Sanity check");
+  assert(oopDesc::is_oop(val), "Sanity check");
+  assert(val->is_inline_type(), "Sanity check");
+  assert(val->klass() == this, "sanity check");
+  java_mirror()->obj_field_put(reset_value_offset(), val);
 }
 
 instanceOop InlineKlass::allocate_instance(TRAPS) {

src/hotspot/share/oops/inlineKlass.hpp

@@ -80,6 +80,11 @@ class InlineKlass: public InstanceKlass {
     return ((address)_adr_inlineklass_fixed_block) + in_bytes(default_value_offset_offset());
   }
 
+  address adr_reset_value_offset() const {
+    assert(_adr_inlineklass_fixed_block != nullptr, "Should have been initialized");
+    return ((address)_adr_inlineklass_fixed_block) + in_bytes(reset_value_offset_offset());
+  }
+
   ArrayKlass* volatile* adr_value_array_klasses() const {
     assert(_adr_inlineklass_fixed_block != nullptr, "Should have been initialized");
     return (ArrayKlass* volatile*) ((address)_adr_inlineklass_fixed_block) + in_bytes(byte_offset_of(InlineKlassFixedBlock, _null_free_inline_array_klasses));
@@ -167,8 +172,6 @@ class InlineKlass: public InstanceKlass {
   int layout_alignment(LayoutKind kind) const;
   int layout_size_in_bytes(LayoutKind kind) const;
 
-  int first_field_offset_old();
-
   virtual void remove_unshareable_info();
   virtual void remove_java_mirror();
   virtual void restore_unshareable_info(ClassLoaderData* loader_data, Handle protection_domain, PackageEntry* pkg_entry, TRAPS);
@@ -291,6 +294,10 @@ class InlineKlass: public InstanceKlass {
     return byte_offset_of(InlineKlassFixedBlock, _default_value_offset);
   }
 
+  static ByteSize reset_value_offset_offset() {
+    return byte_offset_of(InlineKlassFixedBlock, _reset_value_offset);
+  }
+
   static ByteSize first_field_offset_offset() {
     return byte_offset_of(InlineKlassFixedBlock, _first_field_offset);
   }
@@ -309,11 +316,34 @@ class InlineKlass: public InstanceKlass {
     return offset;
   }
 
-  void set_default_value(oop val) {
-    java_mirror()->obj_field_put(default_value_offset(), val);
+  void set_default_value(oop val);
+
+  oop default_value() {
+    assert(is_initialized() || is_being_initialized() || is_in_error_state(), "default value is set at the beginning of initialization");
+    oop val = java_mirror()->obj_field_acquire(default_value_offset());
+    assert(val != nullptr, "Sanity check");
+    return val;
+  }
+
+  void set_reset_value_offset(int offset) {
+    *((int*)adr_reset_value_offset()) = offset;
+  }
+
+  int reset_value_offset() {
+    int offset = *((int*)adr_reset_value_offset());
+    assert(offset != 0, "must not be called if not initialized");
+    return offset;
+  }
+
+  void set_reset_value(oop val);
+
+  oop reset_value() {
+    assert(is_initialized() || is_being_initialized() || is_in_error_state(), "reset value is set at the beginning of initialization");
+    oop val = java_mirror()->obj_field_acquire(reset_value_offset());
+    assert(val != nullptr, "Sanity check");
+    return val;
   }
 
-  oop default_value();
   void deallocate_contents(ClassLoaderData* loader_data);
   static void cleanup(InlineKlass* ik) ;
 

src/hotspot/share/oops/instanceKlass.cpp

@@ -1360,6 +1360,22 @@ void InstanceKlass::initialize_impl(TRAPS) {
           THROW_OOP(e());
       }
       vk->set_default_value(val);
+      if (vk->has_nullable_layout()) {
+        val = vk->allocate_instance(THREAD);
+        if (HAS_PENDING_EXCEPTION) {
+            Handle e(THREAD, PENDING_EXCEPTION);
+            CLEAR_PENDING_EXCEPTION;
+            {
+                EXCEPTION_MARK;
+                add_initialization_error(THREAD, e);
+                // Locks object, set state, and notify all waiting threads
+                set_initialization_state_and_notify(initialization_error, THREAD);
+                CLEAR_PENDING_EXCEPTION;
+            }
+            THROW_OOP(e());
+        }
+        vk->set_reset_value(val);
+      }
   }
 
   // Step 7

src/hotspot/share/oops/instanceKlass.hpp

@@ -144,6 +144,7 @@ class InlineKlassFixedBlock {
   address* _pack_handler_jobject;
   address* _unpack_handler;
   int* _default_value_offset;
+  int* _reset_value_offset;
   ArrayKlass** _null_free_inline_array_klasses;
   int _first_field_offset;
   int _payload_size_in_bytes;   // size of payload layout