8345995: [lworld] Flat arrays need to be redesigned to support new flat layouts

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.cpp

@@ -138,29 +138,16 @@ void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators
   }
 }
 
-void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators,
-                                     Register src, Register dst, Register value_klass) {
-  // value_copy implementation is fairly complex, and there are not any
-  // "short-cuts" to be made from asm. What there is, appears to have the same
-  // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
-  // of hand-rolled instructions...
-  if (decorators & IS_DEST_UNINITIALIZED) {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass);
-  } else {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass);
-  }
-}
-
 void BarrierSetAssembler::flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
                                      Register src, Register dst, Register inline_layout_info) {
   // flat_field_copy implementation is fairly complex, and there are not any
   // "short-cuts" to be made from asm. What there is, appears to have the same
   // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
   // of hand-rolled instructions...
   if (decorators & IS_DEST_UNINITIALIZED) {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized2), src, dst, inline_layout_info);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, inline_layout_info);
   } else {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy2), src, dst, inline_layout_info);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, inline_layout_info);
   }
 }
 

src/hotspot/cpu/aarch64/gc/shared/barrierSetAssembler_aarch64.hpp

@@ -100,8 +100,6 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                         Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
 
-  virtual void value_copy(MacroAssembler* masm, DecoratorSet decorators,
-                          Register src, Register dst, Register value_klass);
   virtual void flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
                           Register src, Register dst, Register inline_layout_info);
 

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

@@ -5410,12 +5410,6 @@ void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
   }
 }
 
-void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
-                                       Register inline_klass) {
-  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
-  bs->value_copy(this, decorators, src, dst, inline_klass);
-}
-
 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst,
                                      Register inline_layout_info) {
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
@@ -5453,7 +5447,7 @@ void MacroAssembler::data_for_value_array_index(Register array, Register array_k
   lslv(index, index, rscratch1);
 
   add(data, array, index);
-  add(data, data, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT));
+  add(data, data, arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT));
 }
 
 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,

src/hotspot/cpu/aarch64/macroAssembler_aarch64.hpp

@@ -926,7 +926,6 @@ class MacroAssembler: public Assembler {
   void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
                        Register tmp1, Register tmp2, Register tmp3);
 
-  void access_value_copy(DecoratorSet decorators, Register src, Register dst, Register inline_klass);
   void flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info);
 
   // inline type data payload offsets...

src/hotspot/cpu/aarch64/templateTable_aarch64.cpp

@@ -828,7 +828,7 @@ void TemplateTable::aaload()
 
     __ b(done);
     __ bind(is_flat_array);
-    __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), r0, r1);
+    __ call_VM(r0, CAST_FROM_FN_PTR(address, InterpreterRuntime::flat_array_load), r0, r1);
     // Ensure the stores to copy the inline field contents are visible
     // before any subsequent store that publishes this reference.
     __ membar(Assembler::StoreStore);
@@ -1182,7 +1182,11 @@ void TemplateTable::aastore() {
   if (EnableValhalla) {
     Label is_null_into_value_array_npe, store_null;
 
-    // No way to store null in flat null-free array
+    if (UseFlatArray) {
+      __ test_flat_array_oop(r3, r8, is_flat_array);
+    }
+
+    // No way to store null in a null-free array
     __ test_null_free_array_oop(r3, r8, is_null_into_value_array_npe);
     __ b(store_null);
 
@@ -1200,40 +1204,10 @@ void TemplateTable::aastore() {
      Label is_type_ok;
     __ bind(is_flat_array); // Store non-null value to flat
 
-    // Simplistic type check...
-    // r0 - value, r2 - index, r3 - array.
-
-    // Profile the not-null value's klass.
-    // Load value class
-     __ load_klass(r1, r0);
-
-    // Move element klass into r7
-     __ ldr(r7, Address(r5, ArrayKlass::element_klass_offset()));
-
-    // flat value array needs exact type match
-    // is "r1 == r7" (value subclass == array element superclass)
-
-     __ cmp(r7, r1);
-     __ br(Assembler::EQ, is_type_ok);
-
-     __ b(ExternalAddress(Interpreter::_throw_ArrayStoreException_entry));
-
-     __ bind(is_type_ok);
-    // r1: value's klass
-    // r3: array
-    // r5: array klass
-    __ test_klass_is_empty_inline_type(r1, r7, done);
-
-    // calc dst for copy
-    __ ldrw(r7, at_tos_p1()); // index
-    __ data_for_value_array_index(r3, r5, r7, r7);
-
-    // ...and src for copy
-    __ ldr(r6, at_tos());  // value
-    __ data_for_oop(r6, r6, r1);
-
-    __ mov(r4, r1);  // Shuffle arguments to avoid conflict with c_rarg1
-    __ access_value_copy(IN_HEAP, r6, r7, r4);
+    __ ldr(r0, at_tos());    // value
+    __ ldr(r3, at_tos_p1()); // index
+    __ ldr(r2, at_tos_p2()); // array
+    __ call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::flat_array_store), r0, r2, r3);
   }
 
   // Pop stack arguments

src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.cpp

@@ -200,29 +200,16 @@ void BarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators
   }
 }
 
-void BarrierSetAssembler::value_copy(MacroAssembler* masm, DecoratorSet decorators,
-                                     Register src, Register dst, Register value_klass) {
-  // value_copy implementation is fairly complex, and there are not any
-  // "short-cuts" to be made from asm. What there is, appears to have the same
-  // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
-  // of hand-rolled instructions...
-  if (decorators & IS_DEST_UNINITIALIZED) {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, value_klass);
-  } else {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, value_klass);
-  }
-}
-
 void BarrierSetAssembler::flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
                                      Register src, Register dst, Register inline_layout_info) {
   // flat_field_copy implementation is fairly complex, and there are not any
   // "short-cuts" to be made from asm. What there is, appears to have the same
   // cost in C++, so just "call_VM_leaf" for now rather than maintain hundreds
   // of hand-rolled instructions...
   if (decorators & IS_DEST_UNINITIALIZED) {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized2), src, dst, inline_layout_info);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy_is_dest_uninitialized), src, dst, inline_layout_info);
   } else {
-    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy2), src, dst, inline_layout_info);
+    __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSetRuntime::value_copy), src, dst, inline_layout_info);
   }
 }
 

src/hotspot/cpu/x86/gc/shared/barrierSetAssembler_x86.hpp

@@ -48,8 +48,6 @@ class BarrierSetAssembler: public CHeapObj<mtGC> {
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                         Address dst, Register val, Register tmp1, Register tmp2, Register tmp3);
 
-  virtual void value_copy(MacroAssembler* masm, DecoratorSet decorators,
-                          Register src, Register dst, Register value_klass);
   virtual void flat_field_copy(MacroAssembler* masm, DecoratorSet decorators,
                           Register src, Register dst, Register inline_layout_info);
 

src/hotspot/cpu/x86/interp_masm_x86.cpp

@@ -1247,53 +1247,6 @@ void InterpreterMacroAssembler::read_flat_field(Register entry, Register tmp1, R
   bind(done);
 }
 
-void InterpreterMacroAssembler::read_flat_element(Register array, Register index,
-                                                  Register t1, Register t2,
-                                                  Register obj) {
-  assert_different_registers(array, index, t1, t2);
-  Label alloc_failed, empty_value, done;
-  const Register array_klass = t2;
-  const Register elem_klass = t1;
-  const Register alloc_temp = LP64_ONLY(rscratch1) NOT_LP64(rsi);
-  const Register dst_temp   = LP64_ONLY(rscratch2) NOT_LP64(rdi);
-
-  // load in array->klass()->element_klass()
-  Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
-  load_klass(array_klass, array, tmp_load_klass);
-  movptr(elem_klass, Address(array_klass, ArrayKlass::element_klass_offset()));
-
-  //check for empty value klass
-  test_klass_is_empty_inline_type(elem_klass, dst_temp, empty_value);
-
-  // calc source into "array_klass" and free up some regs
-  const Register src = array_klass;
-  push(index); // preserve index reg in case alloc_failed
-  data_for_value_array_index(array, array_klass, index, src);
-
-  allocate_instance(elem_klass, obj, alloc_temp, dst_temp, false, alloc_failed);
-  // Have an oop instance buffer, copy into it
-  store_ptr(0, obj); // preserve obj (overwrite index, no longer needed)
-  data_for_oop(obj, dst_temp, elem_klass);
-  access_value_copy(IS_DEST_UNINITIALIZED, src, dst_temp, elem_klass);
-  pop(obj);
-  jmp(done);
-
-  bind(empty_value);
-  get_empty_inline_type_oop(elem_klass, dst_temp, obj);
-  jmp(done);
-
-  bind(alloc_failed);
-  pop(index);
-  if (array == c_rarg2) {
-    mov(elem_klass, array);
-    array = elem_klass;
-  }
-  call_VM(obj, CAST_FROM_FN_PTR(address, InterpreterRuntime::value_array_load), array, index);
-
-  bind(done);
-}
-
-
 // Lock object
 //
 // Args:

src/hotspot/cpu/x86/interp_masm_x86.hpp

@@ -226,15 +226,6 @@ class InterpreterMacroAssembler: public MacroAssembler {
                        Register tmp1, Register tmp2,
                        Register obj = rax);
 
-  // Allocate value buffer in "obj" and read in flat element at the given index
-  // NOTES:
-  //   - Return via "obj" must be rax
-  //   - kills all given regs
-  //   - 32 bits: kills rdi and rsi
-  void read_flat_element(Register array, Register index,
-                         Register t1, Register t2,
-                         Register obj = rax);
-
   // Object locking
   void lock_object  (Register lock_reg);
   void unlock_object(Register lock_reg);

src/hotspot/cpu/x86/macroAssembler_x86.cpp

@@ -6067,12 +6067,6 @@ void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Ad
   }
 }
 
-void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
-                                       Register inline_klass) {
-  BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
-  bs->value_copy(this, decorators, src, dst, inline_klass);
-}
-
 void MacroAssembler::flat_field_copy(DecoratorSet decorators, Register src, Register dst,
                                      Register inline_layout_info) {
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
@@ -6110,7 +6104,7 @@ void MacroAssembler::data_for_value_array_index(Register array, Register array_k
   andl(rcx, Klass::_lh_log2_element_size_mask);
   shlptr(index); // index << rcx
 
-  lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT)));
+  lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_FLAT_ELEMENT)));
 }
 
 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,

src/hotspot/cpu/x86/macroAssembler_x86.hpp

@@ -405,10 +405,7 @@ class MacroAssembler: public Assembler {
   void access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
                        Register tmp1, Register tmp2, Register tmp3);
 
-  void access_value_copy(DecoratorSet decorators, Register src, Register dst, Register inline_klass);
   void flat_field_copy(DecoratorSet decorators, Register src, Register dst, Register inline_layout_info);
-  // We probably need the following for arrays:    TODO FIXME
-  // void flat_element_copy(DecoratorSet decorators, Register src, Register dst, Register array);
 
   // inline type data payload offsets...
   void first_field_offset(Register inline_klass, Register offset);

src/hotspot/cpu/x86/templateTable_x86.cpp

@@ -848,7 +848,8 @@ void TemplateTable::aaload() {
                 IS_ARRAY);
     __ jmp(done);
     __ bind(is_flat_array);
-    __ read_flat_element(array, index, rbx, rcx, rax);
+    __ movptr(rcx, array);
+    call_VM(rax, CAST_FROM_FN_PTR(address, InterpreterRuntime::flat_array_load), rcx, index);
     __ bind(done);
   } else {
     do_oop_load(_masm,
@@ -1200,6 +1201,13 @@ void TemplateTable::aastore() {
   if (EnableValhalla) {
     Label is_null_into_value_array_npe, store_null;
 
+      // Move array class to rdi
+    __ load_klass(rdi, rdx, rscratch1);
+    if (UseFlatArray) {
+      __ movl(rbx, Address(rdi, Klass::layout_helper_offset()));
+      __ test_flat_array_layout(rbx, is_flat_array);
+    }
+
     // No way to store null in null-free array
     __ test_null_free_array_oop(rdx, rbx, is_null_into_value_array_npe);
     __ jmp(store_null);
@@ -1217,34 +1225,11 @@ void TemplateTable::aastore() {
     Label is_type_ok;
     __ bind(is_flat_array); // Store non-null value to flat
 
-    // Simplistic type check...
-
-    // Profile the not-null value's klass.
-    __ load_klass(rbx, rax, rscratch1);
-    // Move element klass into rax
-    __ movptr(rax, Address(rdi, ArrayKlass::element_klass_offset()));
-    // flat value array needs exact type match
-    // is "rax == rbx" (value subclass == array element superclass)
-    __ cmpptr(rax, rbx);
-    __ jccb(Assembler::equal, is_type_ok);
-
-    __ jump(RuntimeAddress(Interpreter::_throw_ArrayStoreException_entry));
-
-    __ bind(is_type_ok);
-    // rbx: value's klass
-    // rdx: array
-    // rdi: array klass
-    __ test_klass_is_empty_inline_type(rbx, rax, done);
-
-    // calc dst for copy
-    __ movl(rax, at_tos_p1()); // index
-    __ data_for_value_array_index(rdx, rdi, rax, rax);
-
-    // ...and src for copy
-    __ movptr(rcx, at_tos());  // value
-    __ data_for_oop(rcx, rcx, rbx);
+    __ movptr(rax, at_tos());
+    __ movl(rcx, at_tos_p1()); // index
+    __ movptr(rdx, at_tos_p2()); // array
 
-    __ access_value_copy(IN_HEAP, rcx, rax, rbx);
+    call_VM(noreg, CAST_FROM_FN_PTR(address, InterpreterRuntime::flat_array_store), rax, rdx, rcx);
   }
   // Pop stack arguments
   __ bind(done);

src/hotspot/share/c1/c1_Runtime1.cpp

@@ -446,9 +446,16 @@ JRT_ENTRY(void, Runtime1::new_null_free_array(JavaThread* current, Klass* array_
   Handle holder(THREAD, array_klass->klass_holder()); // keep the klass alive
   Klass* elem_klass = ArrayKlass::cast(array_klass)->element_klass();
   assert(elem_klass->is_inline_klass(), "must be");
+  InlineKlass* vk = InlineKlass::cast(elem_klass);
   // Logically creates elements, ensure klass init
   elem_klass->initialize(CHECK);
-  arrayOop obj = oopFactory::new_valueArray(elem_klass, length, CHECK);
+  arrayOop obj= nullptr;
+  //  Limitation here, only non-atomic layouts are supported
+  if (UseFlatArray && vk->has_non_atomic_layout()) {
+    obj = oopFactory::new_flatArray(elem_klass, length, LayoutKind::NON_ATOMIC_FLAT, CHECK);
+  } else {
+    obj = oopFactory::new_null_free_objArray(elem_klass, length, CHECK);
+  }
   current->set_vm_result(obj);
   // This is pretty rare but this runtime patch is stressful to deoptimization
   // if we deoptimize here so force a deopt to stress the path.
@@ -509,7 +516,7 @@ JRT_ENTRY(void, Runtime1::load_flat_array(JavaThread* current, flatArrayOopDesc*
   NOT_PRODUCT(_load_flat_array_slowcase_cnt++;)
   assert(array->length() > 0 && index < array->length(), "already checked");
   flatArrayHandle vah(current, array);
-  oop obj = flatArrayOopDesc::value_alloc_copy_from_index(vah, index, CHECK);
+  oop obj = array->read_value_from_flat_array(index, CHECK);
   current->set_vm_result(obj);
 JRT_END
 
@@ -525,7 +532,7 @@ JRT_ENTRY(void, Runtime1::store_flat_array(JavaThread* current, flatArrayOopDesc
     SharedRuntime::throw_and_post_jvmti_exception(current, vmSymbols::java_lang_NullPointerException());
   } else {
     assert(array->klass()->is_flatArray_klass(), "should not be called");
-    array->value_copy_to_index(value, index, LayoutKind::PAYLOAD); // Non atomic is currently the only layout supported by flat arrays
+    array->write_value_to_flat_array(value, index, CHECK);
   }
 JRT_END
 

src/hotspot/share/cds/archiveHeapWriter.cpp

@@ -214,7 +214,7 @@ void ArchiveHeapWriter::copy_roots_to_buffer(GrowableArrayCHeap<oop, mtClassShar
   assert((_buffer_used % MIN_GC_REGION_ALIGNMENT) == 0,
          "Pre-condition: Roots start at aligned boundary: " SIZE_FORMAT, _buffer_used);
 
-  int max_elem_count = ((MIN_GC_REGION_ALIGNMENT - arrayOopDesc::header_size_in_bytes()) / heapOopSize);
+  int max_elem_count = ((MIN_GC_REGION_ALIGNMENT - arrayOopDesc::header_size_in_bytes(T_INT)) / heapOopSize);
   assert(objArrayOopDesc::object_size(max_elem_count)*HeapWordSize == MIN_GC_REGION_ALIGNMENT,
          "Should match exactly");