From 7f24e1bcd2f35ece7f6140520cede99ef7f651e4 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 1 May 2024 22:57:27 +0200 Subject: [PATCH 1/5] Move to lower --- src/coreclr/inc/jiteeversionguid.h | 10 +-- src/coreclr/inc/jithelpers.h | 3 +- src/coreclr/inc/readytorun.h | 4 +- src/coreclr/inc/readytorunhelpers.h | 1 + src/coreclr/jit/compiler.h | 1 + src/coreclr/jit/lower.cpp | 85 +++++++++++++++++++ src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerxarch.cpp | 5 ++ .../nativeaot/Runtime/inc/ModuleHeaders.h | 2 +- .../Common/Internal/Runtime/ModuleHeaders.cs | 2 +- .../Internal/Runtime/ReadyToRunConstants.cs | 1 + .../ILCompiler.Compiler/Compiler/JitHelper.cs | 3 + .../JitInterface/CorInfoImpl.ReadyToRun.cs | 3 + .../ReadyToRunSignature.cs | 4 + .../JitInterface/CorInfoImpl.RyuJit.cs | 3 + src/coreclr/vm/corelib.h | 1 + src/coreclr/vm/ecall.cpp | 4 + src/coreclr/vm/jithelpers.cpp | 17 +--- .../src/System/Buffer.cs | 3 + 19 files changed, 127 insertions(+), 26 deletions(-) diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 79b6397a9111e..122906341d00b 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -43,11 +43,11 @@ typedef const GUID *LPCGUID; #define GUID_DEFINED #endif // !GUID_DEFINED -constexpr GUID JITEEVersionIdentifier = { /* 32d71f8e-c1f5-41cb-88cc-4e8504cabf40 */ - 0x32d71f8e, - 0xc1f5, - 0x41cb, - {0x88, 0xcc, 0x4e, 0x85, 0x04, 0xca, 0xbf, 0x40} +constexpr GUID JITEEVersionIdentifier = { /* bd8c41d4-8531-49c1-a600-0ae9bfe05de1 */ + 0xbd8c41d4, + 0x8531, + 0x49c1, + {0xa6, 0x00, 0x0a, 0xe9, 0xbf, 0xe0, 0x5d, 0xe1} }; ////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index f1711a9acfd9b..71d685d16087d 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -153,8 +153,7 @@ JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,CORINFO_HELP_SIG_REG_ONLY) DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,CORINFO_HELP_SIG_NO_ALIGN_STUB) - - JITHELPER(CORINFO_HELP_ASSIGN_STRUCT, JIT_StructWriteBarrier,CORINFO_HELP_SIG_4_STACK) + DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_STRUCT, NULL, CORINFO_HELP_SIG_REG_ONLY) // Accessing fields JITHELPER(CORINFO_HELP_GETFIELD8, JIT_GetField8,CORINFO_HELP_SIG_REG_ONLY) diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index 88219146a123a..d04ab19e1e844 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -20,7 +20,7 @@ // If you update this, ensure you run `git grep MINIMUM_READYTORUN_MAJOR_VERSION` // and handle pending work. #define READYTORUN_MAJOR_VERSION 0x0009 -#define READYTORUN_MINOR_VERSION 0x0002 +#define READYTORUN_MINOR_VERSION 0x0003 #define MINIMUM_READYTORUN_MAJOR_VERSION 0x009 @@ -34,6 +34,7 @@ // R2R Version 9.0 adds support for the Vector512 type // R2R Version 9.1 adds new helpers to allocate objects on frozen segments // R2R Version 9.2 adds MemZero and NativeMemSet helpers +// R2R Version 9.3 adds AssignStruct helper struct READYTORUN_CORE_HEADER @@ -321,6 +322,7 @@ enum ReadyToRunHelper READYTORUN_HELPER_WriteBarrier = 0x30, READYTORUN_HELPER_CheckedWriteBarrier = 0x31, READYTORUN_HELPER_ByRefWriteBarrier = 0x32, + READYTORUN_HELPER_AssignStruct = 0x33, // Array helpers READYTORUN_HELPER_Stelem_Ref = 0x38, diff --git a/src/coreclr/inc/readytorunhelpers.h b/src/coreclr/inc/readytorunhelpers.h index bbb586e8eb4a3..04bff3ef7ae5d 100644 --- a/src/coreclr/inc/readytorunhelpers.h +++ b/src/coreclr/inc/readytorunhelpers.h @@ -24,6 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero, CORINFO_HELP_THROWDIVZERO, HELPER(READYTORUN_HELPER_WriteBarrier, CORINFO_HELP_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_CheckedWriteBarrier, CORINFO_HELP_CHECKED_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_ByRefWriteBarrier, CORINFO_HELP_ASSIGN_BYREF, ) +HELPER(READYTORUN_HELPER_AssignStruct, CORINFO_HELP_ASSIGN_STRUCT, ) HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST, ) HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, ) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index deb7b97943dc8..6ed5f067173b8 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -7803,6 +7803,7 @@ class Compiler GenTree* optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree); GenTree* optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree); GenTree* optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call); + GenTree* optVNBasedFoldExpr_Blk(BasicBlock* block, GenTree* parent, GenTreeBlk* blk); GenTree* optExtractSideEffListFromConst(GenTree* tree); AssertionIndex GetAssertionCount() diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index cb8eb5d6a97a0..c2bc30d00cd5a 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8212,6 +8212,91 @@ void Lowering::ContainCheckBitCast(GenTree* node) } } +bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) +{ + if (comp->opts.OptimizationDisabled() || !ISMETHOD("Test")) + { + return false; + } + + // Replace STORE_BLK (struct copy) with CORINFO_HELP_ASSIGN_STRUCT which performs + // bulk copy for byrefs. + const unsigned bulkCopyThreshold = 4; + if (blk->OperIs(GT_STORE_BLK) && !blk->OperIsInitBlkOp() && + (blk->GetLayout()->GetGCPtrCount() >= bulkCopyThreshold)) + { + GenTree* addr = blk->Addr(); + GenTree* data = blk->Data(); + + const unsigned gcPtrs = blk->GetLayout()->GetGCPtrCount(); + if (!CheckedOps::MulOverflows((int)gcPtrs, TARGET_POINTER_SIZE, true)) + { + if (data->OperIs(GT_IND)) + { + // Drop GT_IND nodes + BlockRange().Remove(data); + data = data->AsIndir()->Addr(); + } + else + { + assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); + + // Convert local to LCL_ADDR + unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); + data->ChangeOper(GT_LCL_ADDR); + data->ChangeType(TYP_I_IMPL); + data->AsLclFld()->SetLclOffs(lclOffset); + data->ClearContained(); + } + + // Size is a constant + GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)gcPtrs * TARGET_POINTER_SIZE, TYP_I_IMPL); + BlockRange().InsertBefore(data, size); + + // A hacky way to safely call fgMorphTree in Lower + GenTree* destPlaceholder = comp->gtNewZeroConNode(addr->TypeGet()); + GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); + GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); + + GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder); + comp->fgMorphArgs(call); + + LIR::Range range = LIR::SeqTree(comp, call); + GenTree* rangeStart = range.FirstNode(); + GenTree* rangeEnd = range.LastNode(); + + BlockRange().InsertBefore(blk, std::move(range)); + blk->gtBashToNOP(); + + LIR::Use destUse; + LIR::Use sizeUse; + BlockRange().TryGetUse(destPlaceholder, &destUse); + BlockRange().TryGetUse(sizePlaceholder, &sizeUse); + destUse.ReplaceWith(addr); + sizeUse.ReplaceWith(size); + destPlaceholder->SetUnusedValue(); + sizePlaceholder->SetUnusedValue(); + + LIR::Use dataUse; + BlockRange().TryGetUse(dataPlaceholder, &dataUse); + dataUse.ReplaceWith(data); + dataPlaceholder->SetUnusedValue(); + + LowerRange(rangeStart, rangeEnd); + + // Finally move all GT_PUTARG_* nodes + // Re-use the existing logic for CFG call args here + MoveCFGCallArgs(call); + + BlockRange().Remove(destPlaceholder); + BlockRange().Remove(sizePlaceholder); + BlockRange().Remove(dataPlaceholder); + return true; + } + } + return false; +} + //------------------------------------------------------------------------ // LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call // diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 055f425b66456..339c1a82f78a7 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -334,6 +334,7 @@ class Lowering final : public Phase void LowerBlockStore(GenTreeBlk* blkNode); void LowerBlockStoreCommon(GenTreeBlk* blkNode); void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode); + bool TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode); void LowerLclHeap(GenTree* node); void ContainBlockStoreAddress(GenTreeBlk* blkNode, unsigned size, GenTree* addr, GenTree* addrParent); void LowerPutArgStkOrSplit(GenTreePutArgStk* putArgNode); diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0e18c5685066f..45ea924530180 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -457,6 +457,11 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); // If we have a long enough sequence of slots that do not require write barriers then diff --git a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h index 6a3b24a394487..d4360273d28ac 100644 --- a/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h +++ b/src/coreclr/nativeaot/Runtime/inc/ModuleHeaders.h @@ -12,7 +12,7 @@ struct ReadyToRunHeaderConstants static const uint32_t Signature = 0x00525452; // 'RTR' static const uint32_t CurrentMajorVersion = 9; - static const uint32_t CurrentMinorVersion = 2; + static const uint32_t CurrentMinorVersion = 3; }; struct ReadyToRunHeader diff --git a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs index 6fc5d9542e160..8420c9a4803b2 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ModuleHeaders.cs @@ -16,7 +16,7 @@ internal struct ReadyToRunHeaderConstants public const uint Signature = 0x00525452; // 'RTR' public const ushort CurrentMajorVersion = 9; - public const ushort CurrentMinorVersion = 2; + public const ushort CurrentMinorVersion = 3; } #if READYTORUN #pragma warning disable 0169 diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs index 0ca8b74c85229..d98ae8ef722ce 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs @@ -233,6 +233,7 @@ public enum ReadyToRunHelper WriteBarrier = 0x30, CheckedWriteBarrier = 0x31, ByRefWriteBarrier = 0x32, + AssignStruct = 0x33, // Array helpers Stelem_Ref = 0x38, diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs index 8faf4cf3fcfe7..37e42ef894d28 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs @@ -67,6 +67,9 @@ public static void GetEntryPoint(TypeSystemContext context, ReadyToRunHelper id, case ReadyToRunHelper.CheckedWriteBarrier: mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpCheckedAssignRefArm64" : "RhpCheckedAssignRef"; break; + case ReadyToRunHelper.AssignStruct: + mangledName = "RhBuffer_BulkMoveWithWriteBarrier"; + break; case ReadyToRunHelper.ByRefWriteBarrier: mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpByRefAssignRefArm64" : "RhpByRefAssignRef"; break; diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs index e1b1b359c0878..1ae9f245fd0d5 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs @@ -1000,6 +1000,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF: id = ReadyToRunHelper.ByRefWriteBarrier; break; + case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_STRUCT: + id = ReadyToRunHelper.AssignStruct; + break; case CorInfoHelpFunc.CORINFO_HELP_ARRADDR_ST: id = ReadyToRunHelper.Stelem_Ref; diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs index 0eae2f10cb8f0..f9c5582d0fe70 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs @@ -1672,6 +1672,10 @@ private void ParseHelper(StringBuilder builder) builder.Append("BYREF_WRITE_BARRIER"); break; + case ReadyToRunHelper.AssignStruct: + builder.Append("ASSIGN_STRUCT"); + break; + // Array helpers case ReadyToRunHelper.Stelem_Ref: builder.Append("STELEM_REF"); diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs index 2dfa5d9d88f67..4610fdeaf0d52 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs @@ -497,6 +497,9 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_CHECKED_ASSIGN_REF: id = ReadyToRunHelper.CheckedWriteBarrier; break; + case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_STRUCT: + id = ReadyToRunHelper.AssignStruct; + break; case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF: id = ReadyToRunHelper.ByRefWriteBarrier; break; diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index d5af8bdb3cb64..2f9049aea8d71 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -891,6 +891,7 @@ DEFINE_METHOD(DEBUGGER, BREAK, Break, DEFINE_CLASS(BUFFER, System, Buffer) DEFINE_METHOD(BUFFER, MEMCPY_PTRBYTE_ARRBYTE, Memcpy, SM_PtrByte_Int_ArrByte_Int_Int_RetVoid) DEFINE_METHOD(BUFFER, MEMCPY, Memcpy, SM_PtrByte_PtrByte_Int_RetVoid) +DEFINE_METHOD(BUFFER, MEMCOPYGC, BulkMoveWithWriteBarrier, SM_RefByte_RefByte_UIntPtr_RetVoid) DEFINE_CLASS(STUBHELPERS, StubHelpers, StubHelpers) DEFINE_METHOD(STUBHELPERS, GET_DELEGATE_TARGET, GetDelegateTarget, SM_Delegate_RetIntPtr) diff --git a/src/coreclr/vm/ecall.cpp b/src/coreclr/vm/ecall.cpp index 35a5d36eae4d7..59c1e8c6f421b 100644 --- a/src/coreclr/vm/ecall.cpp +++ b/src/coreclr/vm/ecall.cpp @@ -157,6 +157,10 @@ void ECall::PopulateManagedHelpers() pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_MEMCPY, pDest); + pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__BUFFER__MEMCOPYGC)); + pDest = pMD->GetMultiCallableAddrOfCode(); + SetJitHelperFunction(CORINFO_HELP_ASSIGN_STRUCT, pDest); + pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__MATH__ROUND)); pDest = pMD->GetMultiCallableAddrOfCode(); SetJitHelperFunction(CORINFO_HELP_DBLROUND, pDest); diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp index 98d6780258a6c..05d2ad18b5bd9 100644 --- a/src/coreclr/vm/jithelpers.cpp +++ b/src/coreclr/vm/jithelpers.cpp @@ -55,8 +55,8 @@ #ifndef FEATURE_EH_FUNCLETS #include "excep.h" #endif - #include "exinfo.h" +#include "arraynative.inl" using std::isfinite; using std::isnan; @@ -4779,21 +4779,6 @@ HCIMPLEND // //======================================================================== -/*************************************************************/ -HCIMPL3(VOID, JIT_StructWriteBarrier, void *dest, void* src, CORINFO_CLASS_HANDLE typeHnd_) -{ - FCALL_CONTRACT; - - TypeHandle typeHnd(typeHnd_); - MethodTable *pMT = typeHnd.AsMethodTable(); - - HELPER_METHOD_FRAME_BEGIN_NOPOLL(); // Set up a frame - CopyValueClass(dest, src, pMT); - HELPER_METHOD_FRAME_END_POLL(); - -} -HCIMPLEND - /*************************************************************/ // Slow helper to tailcall from the fast one NOINLINE HCIMPL0(void, JIT_PollGC_Framed) diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs index 24f8794d852af..d07df58bb2376 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffer.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffer.cs @@ -177,6 +177,9 @@ ref Unsafe.As(ref source), private const uint BulkMoveWithWriteBarrierChunk = 0x4000; #endif +#if NATIVEAOT + [System.Runtime.RuntimeExport("RhBuffer_BulkMoveWithWriteBarrier")] +#endif internal static void BulkMoveWithWriteBarrier(ref byte destination, ref byte source, nuint byteCount) { if (byteCount <= BulkMoveWithWriteBarrierChunk) From efa0292da4e78741d26b78af24d67a808f508ed8 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 2 May 2024 01:02:01 +0200 Subject: [PATCH 2/5] fix all platforms --- src/coreclr/jit/lower.cpp | 142 +++++++++++++++------------ src/coreclr/jit/lowerarmarch.cpp | 6 ++ src/coreclr/jit/lowerloongarch64.cpp | 6 ++ src/coreclr/jit/lowerriscv64.cpp | 6 ++ src/coreclr/jit/lowerxarch.cpp | 1 + 5 files changed, 100 insertions(+), 61 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index c2bc30d00cd5a..a95d275b5f2ad 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8212,9 +8212,15 @@ void Lowering::ContainCheckBitCast(GenTree* node) } } +//------------------------------------------------------------------------ +// TryLowerBlockStoreAsGcBulkCopyCall: Lower a block store node as a CORINFO_HELP_ASSIGN_STRUCT call +// +// Arguments: +// blkNode - The block store node to lower +// bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) { - if (comp->opts.OptimizationDisabled() || !ISMETHOD("Test")) + if (comp->opts.OptimizationDisabled()) { return false; } @@ -8222,79 +8228,93 @@ bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) // Replace STORE_BLK (struct copy) with CORINFO_HELP_ASSIGN_STRUCT which performs // bulk copy for byrefs. const unsigned bulkCopyThreshold = 4; - if (blk->OperIs(GT_STORE_BLK) && !blk->OperIsInitBlkOp() && - (blk->GetLayout()->GetGCPtrCount() >= bulkCopyThreshold)) + if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || (blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold)) { - GenTree* addr = blk->Addr(); - GenTree* data = blk->Data(); + return false; + } - const unsigned gcPtrs = blk->GetLayout()->GetGCPtrCount(); - if (!CheckedOps::MulOverflows((int)gcPtrs, TARGET_POINTER_SIZE, true)) - { - if (data->OperIs(GT_IND)) - { - // Drop GT_IND nodes - BlockRange().Remove(data); - data = data->AsIndir()->Addr(); - } - else - { - assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); - - // Convert local to LCL_ADDR - unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); - data->ChangeOper(GT_LCL_ADDR); - data->ChangeType(TYP_I_IMPL); - data->AsLclFld()->SetLclOffs(lclOffset); - data->ClearContained(); - } + GenTree* dest = blk->Addr(); + GenTree* data = blk->Data(); + + if (data->OperIs(GT_IND)) + { + // Drop GT_IND nodes + BlockRange().Remove(data); + data = data->AsIndir()->Addr(); + } + else + { + assert(data->OperIs(GT_LCL_VAR, GT_LCL_FLD)); + + // Convert local to LCL_ADDR + unsigned lclOffset = data->AsLclVarCommon()->GetLclOffs(); + data->ChangeOper(GT_LCL_ADDR); + data->ChangeType(TYP_I_IMPL); + data->AsLclFld()->SetLclOffs(lclOffset); + data->ClearContained(); + } - // Size is a constant - GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)gcPtrs * TARGET_POINTER_SIZE, TYP_I_IMPL); - BlockRange().InsertBefore(data, size); + // Size is a constant + GenTreeIntCon* size = comp->gtNewIconNode((ssize_t)blk->GetLayout()->GetSize(), TYP_I_IMPL); + BlockRange().InsertBefore(data, size); + + // A hacky way to safely call fgMorphTree in Lower + GenTree* destPlaceholder = comp->gtNewZeroConNode(dest->TypeGet()); + GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); + GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); - // A hacky way to safely call fgMorphTree in Lower - GenTree* destPlaceholder = comp->gtNewZeroConNode(addr->TypeGet()); - GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); - GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); + GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, + dataPlaceholder, sizePlaceholder); + comp->fgMorphArgs(call); - GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder); - comp->fgMorphArgs(call); + LIR::Range range = LIR::SeqTree(comp, call); + GenTree* rangeStart = range.FirstNode(); + GenTree* rangeEnd = range.LastNode(); - LIR::Range range = LIR::SeqTree(comp, call); - GenTree* rangeStart = range.FirstNode(); - GenTree* rangeEnd = range.LastNode(); + BlockRange().InsertBefore(blk, std::move(range)); + blk->gtBashToNOP(); - BlockRange().InsertBefore(blk, std::move(range)); - blk->gtBashToNOP(); + LIR::Use destUse; + LIR::Use sizeUse; + BlockRange().TryGetUse(destPlaceholder, &destUse); + BlockRange().TryGetUse(sizePlaceholder, &sizeUse); + destUse.ReplaceWith(dest); + sizeUse.ReplaceWith(size); + destPlaceholder->SetUnusedValue(); + sizePlaceholder->SetUnusedValue(); - LIR::Use destUse; - LIR::Use sizeUse; - BlockRange().TryGetUse(destPlaceholder, &destUse); - BlockRange().TryGetUse(sizePlaceholder, &sizeUse); - destUse.ReplaceWith(addr); - sizeUse.ReplaceWith(size); - destPlaceholder->SetUnusedValue(); - sizePlaceholder->SetUnusedValue(); + LIR::Use dataUse; + BlockRange().TryGetUse(dataPlaceholder, &dataUse); + dataUse.ReplaceWith(data); + dataPlaceholder->SetUnusedValue(); - LIR::Use dataUse; - BlockRange().TryGetUse(dataPlaceholder, &dataUse); - dataUse.ReplaceWith(data); - dataPlaceholder->SetUnusedValue(); + LowerRange(rangeStart, rangeEnd); - LowerRange(rangeStart, rangeEnd); + // Finally move all GT_PUTARG_* nodes + // Re-use the existing logic for CFG call args here + MoveCFGCallArgs(call); - // Finally move all GT_PUTARG_* nodes - // Re-use the existing logic for CFG call args here - MoveCFGCallArgs(call); + BlockRange().Remove(destPlaceholder); + BlockRange().Remove(sizePlaceholder); + BlockRange().Remove(dataPlaceholder); - BlockRange().Remove(destPlaceholder); - BlockRange().Remove(sizePlaceholder); - BlockRange().Remove(dataPlaceholder); - return true; + // Add implicit nullchecks for dest and data if needed: + // + auto wrapWithNullcheck = [&](GenTree* node) { + if (comp->fgAddrCouldBeNull(node)) + { + LIR::Use nodeUse; + BlockRange().TryGetUse(node, &nodeUse); + GenTree* nodeClone = comp->gtNewLclvNode(nodeUse.ReplaceWithLclVar(comp), genActualType(node)); + GenTree* nullcheck = comp->gtNewNullCheck(nodeClone, comp->compCurBB); + BlockRange().InsertAfter(nodeUse.Def(), nodeClone, nullcheck); + LowerNode(nullcheck); } - } - return false; + }; + wrapWithNullcheck(dest); + wrapWithNullcheck(data); + + return true; } //------------------------------------------------------------------------ diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 6456b2b972ea9..36b5e08f0afd6 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -737,6 +737,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerloongarch64.cpp b/src/coreclr/jit/lowerloongarch64.cpp index 4e826be0b2257..d3552e478fdfe 100644 --- a/src/coreclr/jit/lowerloongarch64.cpp +++ b/src/coreclr/jit/lowerloongarch64.cpp @@ -370,6 +370,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj or CopyBlk if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index aa8342ee1af5a..5b0bd99df484f 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -319,6 +319,12 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) // CopyObj or CopyBlk if (doCpObj) { + // Try to use bulk copy helper + if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) + { + return; + } + assert((dstAddr->TypeGet() == TYP_BYREF) || (dstAddr->TypeGet() == TYP_I_IMPL)); blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindCpObjUnroll; } diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 45ea924530180..156992b89e9d4 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -457,6 +457,7 @@ void Lowering::LowerBlockStore(GenTreeBlk* blkNode) if (doCpObj) { + // Try to use bulk copy helper if (TryLowerBlockStoreAsGcBulkCopyCall(blkNode)) { return; From 5dd3d40f18431c4bc2634d12dad6d682e7c05a44 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 2 May 2024 12:17:32 +0200 Subject: [PATCH 3/5] rename helper --- src/coreclr/inc/corinfo.h | 2 +- src/coreclr/inc/jithelpers.h | 2 +- src/coreclr/inc/readytorun.h | 4 ++-- src/coreclr/inc/readytorunhelpers.h | 2 +- src/coreclr/jit/codegencommon.cpp | 2 +- src/coreclr/jit/lower.cpp | 6 +++--- src/coreclr/jit/utils.cpp | 2 +- .../tools/Common/Internal/Runtime/ReadyToRunConstants.cs | 2 +- src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs | 2 +- .../tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs | 2 +- .../JitInterface/CorInfoImpl.ReadyToRun.cs | 4 ++-- .../ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs | 4 ++-- .../ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs | 4 ++-- src/coreclr/vm/ecall.cpp | 2 +- 14 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index ff82759f6aab6..2d526105164b6 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -496,7 +496,7 @@ enum CorInfoHelpFunc CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, // Do the store, and ensure that the target was not in the heap. CORINFO_HELP_ASSIGN_BYREF, - CORINFO_HELP_ASSIGN_STRUCT, + CORINFO_HELP_BULK_WRITEBARRIER, /* Accessing fields */ diff --git a/src/coreclr/inc/jithelpers.h b/src/coreclr/inc/jithelpers.h index 71d685d16087d..2ee4999453859 100644 --- a/src/coreclr/inc/jithelpers.h +++ b/src/coreclr/inc/jithelpers.h @@ -153,7 +153,7 @@ JITHELPER(CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, JIT_WriteBarrierEnsureNonHeapTarget,CORINFO_HELP_SIG_REG_ONLY) DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, JIT_ByRefWriteBarrier,CORINFO_HELP_SIG_NO_ALIGN_STUB) - DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_STRUCT, NULL, CORINFO_HELP_SIG_REG_ONLY) + DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, CORINFO_HELP_SIG_REG_ONLY) // Accessing fields JITHELPER(CORINFO_HELP_GETFIELD8, JIT_GetField8,CORINFO_HELP_SIG_REG_ONLY) diff --git a/src/coreclr/inc/readytorun.h b/src/coreclr/inc/readytorun.h index d04ab19e1e844..1c3ce8237ef7f 100644 --- a/src/coreclr/inc/readytorun.h +++ b/src/coreclr/inc/readytorun.h @@ -34,7 +34,7 @@ // R2R Version 9.0 adds support for the Vector512 type // R2R Version 9.1 adds new helpers to allocate objects on frozen segments // R2R Version 9.2 adds MemZero and NativeMemSet helpers -// R2R Version 9.3 adds AssignStruct helper +// R2R Version 9.3 adds BulkWriteBarrier helper struct READYTORUN_CORE_HEADER @@ -322,7 +322,7 @@ enum ReadyToRunHelper READYTORUN_HELPER_WriteBarrier = 0x30, READYTORUN_HELPER_CheckedWriteBarrier = 0x31, READYTORUN_HELPER_ByRefWriteBarrier = 0x32, - READYTORUN_HELPER_AssignStruct = 0x33, + READYTORUN_HELPER_BulkWriteBarrier = 0x33, // Array helpers READYTORUN_HELPER_Stelem_Ref = 0x38, diff --git a/src/coreclr/inc/readytorunhelpers.h b/src/coreclr/inc/readytorunhelpers.h index 04bff3ef7ae5d..a8d55725d3dec 100644 --- a/src/coreclr/inc/readytorunhelpers.h +++ b/src/coreclr/inc/readytorunhelpers.h @@ -24,7 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero, CORINFO_HELP_THROWDIVZERO, HELPER(READYTORUN_HELPER_WriteBarrier, CORINFO_HELP_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_CheckedWriteBarrier, CORINFO_HELP_CHECKED_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_ByRefWriteBarrier, CORINFO_HELP_ASSIGN_BYREF, ) -HELPER(READYTORUN_HELPER_AssignStruct, CORINFO_HELP_ASSIGN_STRUCT, ) +HELPER(READYTORUN_HELPER_BulkWriteBarrier, CORINFO_HELP_BULK_WRITEBARRIER, ) HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST, ) HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, ) diff --git a/src/coreclr/jit/codegencommon.cpp b/src/coreclr/jit/codegencommon.cpp index 6ec296a1391bb..6dcbba13b4853 100644 --- a/src/coreclr/jit/codegencommon.cpp +++ b/src/coreclr/jit/codegencommon.cpp @@ -1772,7 +1772,7 @@ void CodeGen::genGenerateCode(void** codePtr, uint32_t* nativeSizeOfCode) if (genWriteBarrierUsed && JitConfig.EnableExtraSuperPmiQueries() && !compiler->opts.IsReadyToRun()) { void* ignored; - for (int i = CORINFO_HELP_ASSIGN_REF; i <= CORINFO_HELP_ASSIGN_STRUCT; i++) + for (int i = CORINFO_HELP_ASSIGN_REF; i <= CORINFO_HELP_BULK_WRITEBARRIER; i++) { compiler->compGetHelperFtn((CorInfoHelpFunc)i, &ignored); } diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index a95d275b5f2ad..e31971884b157 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8213,7 +8213,7 @@ void Lowering::ContainCheckBitCast(GenTree* node) } //------------------------------------------------------------------------ -// TryLowerBlockStoreAsGcBulkCopyCall: Lower a block store node as a CORINFO_HELP_ASSIGN_STRUCT call +// TryLowerBlockStoreAsGcBulkCopyCall: Lower a block store node as a CORINFO_HELP_BULK_WRITEBARRIER call // // Arguments: // blkNode - The block store node to lower @@ -8225,7 +8225,7 @@ bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) return false; } - // Replace STORE_BLK (struct copy) with CORINFO_HELP_ASSIGN_STRUCT which performs + // Replace STORE_BLK (struct copy) with CORINFO_HELP_BULK_WRITEBARRIER which performs // bulk copy for byrefs. const unsigned bulkCopyThreshold = 4; if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || (blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold)) @@ -8263,7 +8263,7 @@ bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) GenTree* dataPlaceholder = comp->gtNewZeroConNode(genActualType(data)); GenTree* sizePlaceholder = comp->gtNewZeroConNode(genActualType(size)); - GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_ASSIGN_STRUCT, TYP_VOID, destPlaceholder, + GenTreeCall* call = comp->gtNewHelperCallNode(CORINFO_HELP_BULK_WRITEBARRIER, TYP_VOID, destPlaceholder, dataPlaceholder, sizePlaceholder); comp->fgMorphArgs(call); diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index e3b99463b466d..07ca1fb6dc09c 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -1746,7 +1746,7 @@ void HelperCallProperties::init() case CORINFO_HELP_CHECKED_ASSIGN_REF: case CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP: case CORINFO_HELP_ASSIGN_BYREF: - case CORINFO_HELP_ASSIGN_STRUCT: + case CORINFO_HELP_BULK_WRITEBARRIER: mutatesHeap = true; break; diff --git a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs index d98ae8ef722ce..af86b107e5b31 100644 --- a/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs +++ b/src/coreclr/tools/Common/Internal/Runtime/ReadyToRunConstants.cs @@ -233,7 +233,7 @@ public enum ReadyToRunHelper WriteBarrier = 0x30, CheckedWriteBarrier = 0x31, ByRefWriteBarrier = 0x32, - AssignStruct = 0x33, + BulkWriteBarrier = 0x33, // Array helpers Stelem_Ref = 0x38, diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs index 5346806c1aff6..14bf90a9aaaed 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoHelpFunc.cs @@ -138,7 +138,7 @@ which is the right helper to use to allocate an object of a given type. */ CORINFO_HELP_ASSIGN_REF_ENSURE_NONHEAP, // Do the store, and ensure that the target was not in the heap. CORINFO_HELP_ASSIGN_BYREF, - CORINFO_HELP_ASSIGN_STRUCT, + CORINFO_HELP_BULK_WRITEBARRIER, /* Accessing fields */ diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs index 37e42ef894d28..67cfd6cbe4f7a 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/JitHelper.cs @@ -67,7 +67,7 @@ public static void GetEntryPoint(TypeSystemContext context, ReadyToRunHelper id, case ReadyToRunHelper.CheckedWriteBarrier: mangledName = context.Target.Architecture == TargetArchitecture.ARM64 ? "RhpCheckedAssignRefArm64" : "RhpCheckedAssignRef"; break; - case ReadyToRunHelper.AssignStruct: + case ReadyToRunHelper.BulkWriteBarrier: mangledName = "RhBuffer_BulkMoveWithWriteBarrier"; break; case ReadyToRunHelper.ByRefWriteBarrier: diff --git a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs index 1ae9f245fd0d5..7f702fd6b764a 100644 --- a/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs +++ b/src/coreclr/tools/aot/ILCompiler.ReadyToRun/JitInterface/CorInfoImpl.ReadyToRun.cs @@ -1000,8 +1000,8 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF: id = ReadyToRunHelper.ByRefWriteBarrier; break; - case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_STRUCT: - id = ReadyToRunHelper.AssignStruct; + case CorInfoHelpFunc.CORINFO_HELP_BULK_WRITEBARRIER: + id = ReadyToRunHelper.BulkWriteBarrier; break; case CorInfoHelpFunc.CORINFO_HELP_ARRADDR_ST: diff --git a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs index f9c5582d0fe70..329b95f5d3c4e 100644 --- a/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs +++ b/src/coreclr/tools/aot/ILCompiler.Reflection.ReadyToRun/ReadyToRunSignature.cs @@ -1672,8 +1672,8 @@ private void ParseHelper(StringBuilder builder) builder.Append("BYREF_WRITE_BARRIER"); break; - case ReadyToRunHelper.AssignStruct: - builder.Append("ASSIGN_STRUCT"); + case ReadyToRunHelper.BulkWriteBarrier: + builder.Append("BULK_WRITE_BARRIER"); break; // Array helpers diff --git a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs index 4610fdeaf0d52..d344dd431fb55 100644 --- a/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs +++ b/src/coreclr/tools/aot/ILCompiler.RyuJit/JitInterface/CorInfoImpl.RyuJit.cs @@ -497,8 +497,8 @@ private ISymbolNode GetHelperFtnUncached(CorInfoHelpFunc ftnNum) case CorInfoHelpFunc.CORINFO_HELP_CHECKED_ASSIGN_REF: id = ReadyToRunHelper.CheckedWriteBarrier; break; - case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_STRUCT: - id = ReadyToRunHelper.AssignStruct; + case CorInfoHelpFunc.CORINFO_HELP_BULK_WRITEBARRIER: + id = ReadyToRunHelper.BulkWriteBarrier; break; case CorInfoHelpFunc.CORINFO_HELP_ASSIGN_BYREF: id = ReadyToRunHelper.ByRefWriteBarrier; diff --git a/src/coreclr/vm/ecall.cpp b/src/coreclr/vm/ecall.cpp index 59c1e8c6f421b..56a28a7a57ddb 100644 --- a/src/coreclr/vm/ecall.cpp +++ b/src/coreclr/vm/ecall.cpp @@ -159,7 +159,7 @@ void ECall::PopulateManagedHelpers() pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__BUFFER__MEMCOPYGC)); pDest = pMD->GetMultiCallableAddrOfCode(); - SetJitHelperFunction(CORINFO_HELP_ASSIGN_STRUCT, pDest); + SetJitHelperFunction(CORINFO_HELP_BULK_WRITEBARRIER, pDest); pMD = CoreLibBinder::GetMethod((BinderMethodID)(METHOD__MATH__ROUND)); pDest = pMD->GetMultiCallableAddrOfCode(); From a1d78c76e9fb97d3e1e26d6c653949404991d0c1 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 2 May 2024 12:19:55 +0200 Subject: [PATCH 4/5] clean up --- src/coreclr/jit/compiler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 6ed5f067173b8..deb7b97943dc8 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -7803,7 +7803,6 @@ class Compiler GenTree* optVNBasedFoldConstExpr(BasicBlock* block, GenTree* parent, GenTree* tree); GenTree* optVNBasedFoldExpr(BasicBlock* block, GenTree* parent, GenTree* tree); GenTree* optVNBasedFoldExpr_Call(BasicBlock* block, GenTree* parent, GenTreeCall* call); - GenTree* optVNBasedFoldExpr_Blk(BasicBlock* block, GenTree* parent, GenTreeBlk* blk); GenTree* optExtractSideEffListFromConst(GenTree* tree); AssertionIndex GetAssertionCount() From 9dcab81d5ffcc8e3accefa0f84fb1c7eac52bd93 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Fri, 3 May 2024 15:39:49 +0200 Subject: [PATCH 5/5] Conservatively, give up on volatile loads/stores --- src/coreclr/inc/readytorunhelpers.h | 2 +- src/coreclr/jit/lower.cpp | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/coreclr/inc/readytorunhelpers.h b/src/coreclr/inc/readytorunhelpers.h index a8d55725d3dec..a1fcef8fbaf83 100644 --- a/src/coreclr/inc/readytorunhelpers.h +++ b/src/coreclr/inc/readytorunhelpers.h @@ -24,7 +24,7 @@ HELPER(READYTORUN_HELPER_ThrowDivZero, CORINFO_HELP_THROWDIVZERO, HELPER(READYTORUN_HELPER_WriteBarrier, CORINFO_HELP_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_CheckedWriteBarrier, CORINFO_HELP_CHECKED_ASSIGN_REF, ) HELPER(READYTORUN_HELPER_ByRefWriteBarrier, CORINFO_HELP_ASSIGN_BYREF, ) -HELPER(READYTORUN_HELPER_BulkWriteBarrier, CORINFO_HELP_BULK_WRITEBARRIER, ) +HELPER(READYTORUN_HELPER_BulkWriteBarrier, CORINFO_HELP_BULK_WRITEBARRIER, ) HELPER(READYTORUN_HELPER_Stelem_Ref, CORINFO_HELP_ARRADDR_ST, ) HELPER(READYTORUN_HELPER_Ldelema_Ref, CORINFO_HELP_LDELEMA_REF, ) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index e31971884b157..b5591ad97cade 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -8228,7 +8228,8 @@ bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) // Replace STORE_BLK (struct copy) with CORINFO_HELP_BULK_WRITEBARRIER which performs // bulk copy for byrefs. const unsigned bulkCopyThreshold = 4; - if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || (blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold)) + if (!blk->OperIs(GT_STORE_BLK) || blk->OperIsInitBlkOp() || blk->IsVolatile() || + (blk->GetLayout()->GetGCPtrCount() < bulkCopyThreshold)) { return false; } @@ -8238,6 +8239,11 @@ bool Lowering::TryLowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blk) if (data->OperIs(GT_IND)) { + if (data->AsIndir()->IsVolatile()) + { + return false; + } + // Drop GT_IND nodes BlockRange().Remove(data); data = data->AsIndir()->Addr();