diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 1fc303b4162b3d..f7d9192cb588ec 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -226,6 +226,8 @@ class CodeGen final : public CodeGenInterface void ensureCurrentFuncIsUnwindable(); void genEmitIf(WasmValueType blockType = WasmValueType::Invalid); void genEmitEndIf(); + void genEmitBeginBlock(WasmValueType blockType = WasmValueType::Invalid); + void genEmitEndBlock(); void genEmitFunctionEnd(bool emitTerminalUnreachable = true); #endif @@ -1085,6 +1087,10 @@ class CodeGen final : public CodeGenInterface #endif // TARGET_ARM64 +#if defined(TARGET_WASM) + void genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrinsic info); +#endif + #endif // FEATURE_HW_INTRINSICS #if !defined(TARGET_64BIT) diff --git a/src/coreclr/jit/codegenwasm.cpp b/src/coreclr/jit/codegenwasm.cpp index 9e62ace9d85c97..dc63a93e5ba918 100644 --- a/src/coreclr/jit/codegenwasm.cpp +++ b/src/coreclr/jit/codegenwasm.cpp @@ -2274,9 +2274,10 @@ void CodeGen::genEmitNullCheck(regNumber reg) void CodeGen::genRangeCheck(GenTree* tree) { assert(tree->OperIs(GT_BOUNDS_CHECK)); - genConsumeOperands(tree->AsOp()); + GenTreeBoundsChk* boundsCheck = tree->AsBoundsChk(); + genConsumeOperands(boundsCheck); GetEmitter()->emitIns(INS_I_ge_u); - genJumpToThrowHlpBlk(SCK_RNGCHK_FAIL); + genJumpToThrowHlpBlk(boundsCheck->gtThrowKind); } //------------------------------------------------------------------------ @@ -3815,6 +3816,38 @@ void CodeGen::genEmitEndIf() GetEmitter()->emitIns(INS_end); } +// ------------------------------------------------------------------------ +// genEmitBeginBlock: Emit a 'block' instruction. +// +// Notes: +// The block is not modeled as part of wasmControlFlowStack. +// This is used for jump tables which don't have corresponding BasicBlock's +// and which will be emitted within a single block with known offsets for each case. +void CodeGen::genEmitBeginBlock(WasmValueType blockType) +{ + wasmExtraControlFlowDepth++; + if (blockType != WasmValueType::Invalid) + { + GetEmitter()->emitIns_BlockTy(INS_block, blockType); + } + else + { + GetEmitter()->emitIns(INS_block); + } +} + +// ----------------------------------------------------------------------- +// genEmitEndBlock: Emit an 'end' instruction closing a 'block'. +// +// Notes: +// Removes the added stack depth from genEmitBeginBlock. +void CodeGen::genEmitEndBlock() +{ + assert(wasmExtraControlFlowDepth > 0); + wasmExtraControlFlowDepth--; + GetEmitter()->emitIns(INS_end); +} + //------------------------------------------------------------------------ // inst_JMP: Emit a jump instruction. // @@ -3867,34 +3900,34 @@ void CodeGen::genWasmEmitterUnitTestsSimd() DROP // Extract lane: v128 -> scalar (i32/i64/f32/f64), then drop -#define TEST_EXTRACT_LANE(bytes, ins, attr, lane) \ +#define TEST_EXTRACT_LANE(bytes, ins, lane) \ PUSH_V128(bytes); \ - emit->emitIns_Lane(ins, attr, lane); \ + emit->emitIns_Lane(ins, lane); \ DROP // Replace lane: [v128, scalar] -> v128, then drop -#define TEST_REPLACE_LANE_I32(bytes, ins, attr, lane) \ +#define TEST_REPLACE_LANE_I32(bytes, ins, lane) \ PUSH_V128(bytes); \ PUSH_I32(42); \ - emit->emitIns_Lane(ins, attr, lane); \ + emit->emitIns_Lane(ins, lane); \ DROP -#define TEST_REPLACE_LANE_I64(bytes, ins, attr, lane) \ +#define TEST_REPLACE_LANE_I64(bytes, ins, lane) \ PUSH_V128(bytes); \ PUSH_I64(42); \ - emit->emitIns_Lane(ins, attr, lane); \ + emit->emitIns_Lane(ins, lane); \ DROP -#define TEST_REPLACE_LANE_F32(bytes, ins, attr, lane) \ +#define TEST_REPLACE_LANE_F32(bytes, ins, lane) \ PUSH_V128(bytes); \ PUSH_F32(0); \ - emit->emitIns_Lane(ins, attr, lane); \ + emit->emitIns_Lane(ins, lane); \ DROP -#define TEST_REPLACE_LANE_F64(bytes, ins, attr, lane) \ +#define TEST_REPLACE_LANE_F64(bytes, ins, lane) \ PUSH_V128(bytes); \ PUSH_F64(0); \ - emit->emitIns_Lane(ins, attr, lane); \ + emit->emitIns_Lane(ins, lane); \ DROP // Load lane: [i32_addr, v128] -> v128, then drop @@ -3936,30 +3969,30 @@ void CodeGen::genWasmEmitterUnitTestsSimd() // --- IF_LANE: extract/replace lane instructions --- // i8x16 lanes (0..15) - TEST_EXTRACT_LANE(v128Ones, INS_i8x16_extract_lane_s, EA_1BYTE, 0); - TEST_EXTRACT_LANE(v128Ones, INS_i8x16_extract_lane_u, EA_1BYTE, 15); - TEST_REPLACE_LANE_I32(v128Ones, INS_i8x16_replace_lane, EA_1BYTE, 7); + TEST_EXTRACT_LANE(v128Ones, INS_i8x16_extract_lane_s, 0); + TEST_EXTRACT_LANE(v128Ones, INS_i8x16_extract_lane_u, 15); + TEST_REPLACE_LANE_I32(v128Ones, INS_i8x16_replace_lane, 7); // i16x8 lanes (0..7) - TEST_EXTRACT_LANE(v128Ones, INS_i16x8_extract_lane_s, EA_2BYTE, 0); - TEST_EXTRACT_LANE(v128Ones, INS_i16x8_extract_lane_u, EA_2BYTE, 7); - TEST_REPLACE_LANE_I32(v128Ones, INS_i16x8_replace_lane, EA_2BYTE, 3); + TEST_EXTRACT_LANE(v128Ones, INS_i16x8_extract_lane_s, 0); + TEST_EXTRACT_LANE(v128Ones, INS_i16x8_extract_lane_u, 7); + TEST_REPLACE_LANE_I32(v128Ones, INS_i16x8_replace_lane, 3); // i32x4 lanes (0..3) - TEST_EXTRACT_LANE(v128Ones, INS_i32x4_extract_lane, EA_4BYTE, 0); - TEST_REPLACE_LANE_I32(v128Ones, INS_i32x4_replace_lane, EA_4BYTE, 3); + TEST_EXTRACT_LANE(v128Ones, INS_i32x4_extract_lane, 0); + TEST_REPLACE_LANE_I32(v128Ones, INS_i32x4_replace_lane, 3); // i64x2 lanes (0..1) - TEST_EXTRACT_LANE(v128Ones, INS_i64x2_extract_lane, EA_8BYTE, 0); - TEST_REPLACE_LANE_I64(v128Ones, INS_i64x2_replace_lane, EA_8BYTE, 1); + TEST_EXTRACT_LANE(v128Ones, INS_i64x2_extract_lane, 0); + TEST_REPLACE_LANE_I64(v128Ones, INS_i64x2_replace_lane, 1); // f32x4 lanes (0..3) - TEST_EXTRACT_LANE(v128Ones, INS_f32x4_extract_lane, EA_4BYTE, 3); - TEST_REPLACE_LANE_F32(v128Ones, INS_f32x4_replace_lane, EA_4BYTE, 0); + TEST_EXTRACT_LANE(v128Ones, INS_f32x4_extract_lane, 3); + TEST_REPLACE_LANE_F32(v128Ones, INS_f32x4_replace_lane, 0); // f64x2 lanes (0..1) - TEST_EXTRACT_LANE(v128Ones, INS_f64x2_extract_lane, EA_8BYTE, 0); - TEST_REPLACE_LANE_F64(v128Ones, INS_f64x2_replace_lane, EA_8BYTE, 1); + TEST_EXTRACT_LANE(v128Ones, INS_f64x2_extract_lane, 0); + TEST_REPLACE_LANE_F64(v128Ones, INS_f64x2_replace_lane, 1); // --- IF_MEMARG_LANE: load/store lane with memarg --- TEST_LOAD_LANE(v128Ones, INS_v128_load8_lane, EA_1BYTE, 0, 5); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 5e7ef8beb9c7ee..e45d2bdbff1945 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -10164,9 +10164,6 @@ class Compiler return sizeBytes; } - // Get the number of elements of baseType of SIMD vector given by its size and baseType - static int getSIMDVectorLength(unsigned simdSize, var_types baseType); - // Get the number of elements of baseType of SIMD vector given by its type handle int getSIMDVectorLength(CORINFO_CLASS_HANDLE typeHnd); @@ -10174,6 +10171,9 @@ class Compiler int getSIMDTypeAlignment(var_types simdType); public: + // Get the number of elements of baseType of SIMD vector given by its size and baseType + static int getSIMDVectorLength(unsigned simdSize, var_types baseType); + // Get the number of bytes in a System.Numeric.Vector for the current compilation. // Note - cannot be used for System.Runtime.Intrinsic uint32_t getVectorTByteLength() diff --git a/src/coreclr/jit/emitwasm.cpp b/src/coreclr/jit/emitwasm.cpp index e9144e0b2ded5d..46f66fd20abd08 100644 --- a/src/coreclr/jit/emitwasm.cpp +++ b/src/coreclr/jit/emitwasm.cpp @@ -106,7 +106,7 @@ void emitter::emitIns_I(instruction ins, emitAttr attr, cnsval_ssize_t imm) // ins - instruction to emit // attr - emit attributes // imm - immediate value (depth in control flow stack) -// targetBlock - block at that depth +// targetBlock - block at that depth (may be null, in the case of jump table which doesn't target a real block) // void emitter::emitIns_J(instruction ins, emitAttr attr, cnsval_ssize_t imm, BasicBlock* targetBlock) { @@ -116,7 +116,7 @@ void emitter::emitIns_J(instruction ins, emitAttr attr, cnsval_ssize_t imm, Basi id->idIns(ins); id->idInsFmt(fmt); - if (m_debugInfoSize > 0) + if (m_debugInfoSize > 0 && targetBlock != nullptr) { id->idDebugOnlyInfo()->idTargetBlock = targetBlock; } @@ -463,14 +463,34 @@ void emitter::emitIns_V128Imm(instruction ins, const uint8_t bytes[16]) // // Arguments: // ins - instruction (e.g., INS_i8x16_extract_lane_s) -// attr - emit attribute indicating the lane element size // laneIdx - lane index byte // -void emitter::emitIns_Lane(instruction ins, emitAttr attr, uint8_t laneIdx) +void emitter::emitIns_Lane(instruction ins, uint8_t laneIdx) { - instrDesc* id = emitNewInstrSC(attr, laneIdx); - insFormat fmt = emitInsFormat(ins); - uint8_t elemSize = CodeGenInterface::instSimdElemSize(ins); + uint8_t elemSize = CodeGenInterface::instSimdElemSize(ins); + + // Add element width as an emit attribute + emitAttr attr = EA_UNKNOWN; + switch (elemSize) + { + case 1: + attr = EA_1BYTE; + break; + case 2: + attr = EA_2BYTE; + break; + case 4: + attr = EA_4BYTE; + break; + case 8: + attr = EA_8BYTE; + break; + default: + unreached(); + } + + instrDesc* id = emitNewInstrSC(attr, laneIdx); + insFormat fmt = emitInsFormat(ins); assert(fmt == IF_LANE); assert(isValidVectorIndex(elemSize, laneIdx)); diff --git a/src/coreclr/jit/emitwasm.h b/src/coreclr/jit/emitwasm.h index b5bbff12e7cc8d..9e825d62774b6d 100644 --- a/src/coreclr/jit/emitwasm.h +++ b/src/coreclr/jit/emitwasm.h @@ -34,7 +34,7 @@ void emitIns_S_R(instruction ins, emitAttr attr, regNumber ireg, int varx, int o // Packed SIMD instruction emit functions void emitIns_V128Imm(instruction ins, const uint8_t bytes[16]); -void emitIns_Lane(instruction ins, emitAttr attr, uint8_t laneIdx); +void emitIns_Lane(instruction ins, uint8_t laneIdx); void emitIns_MemargLane(instruction ins, emitAttr attr, cnsval_ssize_t offset, uint8_t laneIdx); void emitAddressConstant(void* address); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 9ad468aa31c7b2..3257e5a8a9cd56 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -1017,6 +1017,29 @@ bool GenTree::NeedsConsecutiveRegisters() const } #endif +#ifdef TARGET_WASM +//----------------------------------------------------------------------------------- +// GetImmOp: Get the single immediate operand of this hardware intrinsic node. +// +// Return Value: +// The immediate operand. +// +// Notes: +// Wasm SIMD intrinsics have at most one immediate operand. +// +GenTree* GenTreeHWIntrinsic::GetImmOp() const +{ + int imm1Pos = -1; + int imm2Pos = -1; + HWIntrinsicInfo::GetImmOpsPositions(GetHWIntrinsicId(), &imm1Pos, &imm2Pos); + + // We only expect one immediate operand for Wasm SIMD + assert(imm1Pos >= 0 && imm2Pos < 0); + + return Op(imm1Pos); +} +#endif // TARGET_WASM + #if HAS_FIXED_REGISTER_SET //--------------------------------------------------------------- // gtGetContainedRegMask: Get the reg mask of the node including diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index e7e3e0f2bf608c..754d0bd45024e7 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -6773,6 +6773,10 @@ struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic NamedIntrinsic GetHWIntrinsicId() const; +#ifdef TARGET_WASM + GenTree* GetImmOp() const; +#endif // TARGET_WASM + //--------------------------------------------------------------------------------------- // ChangeHWIntrinsicId: Change the intrinsic id for this node. // diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 421b6adffd2d27..a1a4986cb36e12 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -2322,7 +2322,7 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic); hasFullRangeImm = HWIntrinsicInfo::HasFullRangeImm(intrinsic); #elif defined(TARGET_WASM) - immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic, simdBaseType); + immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(intrinsic, simdSize, simdBaseType); #endif if (!CheckHWIntrinsicImmRange(intrinsic, simdBaseType, immOp1, mustExpand, immLowerBound, immUpperBound, @@ -2642,6 +2642,16 @@ GenTree* Compiler::impHWIntrinsic(NamedIntrinsic intrinsic, op2 = addRangeCheckIfNeeded(intrinsic, op2, immLowerBound, immUpperBound); } else +#elif defined(TARGET_WASM) + // On WASM, PackedSimd.ReplaceScalar takes the lane immediate as the middle + // (op2) operand: ReplaceScalar(Vector128 vector, byte imm, T value). Other + // 3-arg PackedSimd immediate intrinsics (LoadScalarAndInsert, StoreSelectedScalar) + // put the immediate at op3 and fall through to the default handling below. + if (intrinsic == NI_PackedSimd_ReplaceScalar) + { + op2 = addRangeCheckIfNeeded(intrinsic, op2, immLowerBound, immUpperBound); + } + else #endif { op3 = addRangeCheckIfNeeded(intrinsic, op3, immLowerBound, immUpperBound); diff --git a/src/coreclr/jit/hwintrinsic.h b/src/coreclr/jit/hwintrinsic.h index 7effb01b75f84b..0b7e966a5e4bf0 100644 --- a/src/coreclr/jit/hwintrinsic.h +++ b/src/coreclr/jit/hwintrinsic.h @@ -575,7 +575,7 @@ struct HWIntrinsicInfo static void lookupImmBounds( NamedIntrinsic intrinsic, int simdSize, var_types baseType, int immNumber, int* lowerBound, int* upperBound); #elif defined(TARGET_WASM) - static int lookupImmUpperBound(NamedIntrinsic intrinsic, var_types baseType); + static int lookupImmUpperBound(NamedIntrinsic intrinsic, unsigned int simdSize, var_types baseType); #else #error Unsupported platform #endif @@ -1291,7 +1291,51 @@ struct HWIntrinsicInfo } } } -#endif // TARGET_ARM64 +#elif defined(TARGET_WASM) + //------------------------------------------------------------------------------------------------ + // GetImmOpPositions: Get the positions of the immediate operands in the signature of an intrinsic + // with an immediate. + // + // Arguments: + // id - The intrinsic ID + // imm1Pos - The position of the first immediate operand + // imm2Pos - The position of the second immediate operand + // + // Notes: + // imm1Pos and imm2Pos are initialized to -1. + static void GetImmOpsPositions(NamedIntrinsic id, int* imm1Pos, int* imm2Pos) + { + *imm1Pos = -1; + *imm2Pos = -1; + + switch (id) + { + case NI_PackedSimd_ExtractScalar: + { + // (v128, lane_imm) + *imm1Pos = 2; + break; + } + case NI_PackedSimd_ReplaceScalar: + { + // (v128, lane_imm, value) + *imm1Pos = 2; + break; + } + case NI_PackedSimd_StoreSelectedScalar: + case NI_PackedSimd_LoadScalarAndInsert: + { + // (scalar_addr, v128, lane_imm) + *imm1Pos = 3; + break; + } + default: + { + unreached(); + } + } + } +#endif // TARGET_ARM64 || TARGET_WASM }; #ifdef TARGET_ARM64 @@ -1406,12 +1450,12 @@ struct HWIntrinsic final , op3(nullptr) , numOperands(0) , baseType(TYP_UNDEF) + , m_node(node) { assert(node != nullptr); id = node->GetHWIntrinsicId(); category = HWIntrinsicInfo::lookupCategory(id); - assert(HWIntrinsicInfo::RequiresCodegen(id)); InitializeOperands(node); @@ -1426,6 +1470,24 @@ struct HWIntrinsic final return isTableDrivenCategory && isTableDrivenFlag; } + inline bool needsJumpTableFallback() const + { + return !m_node->GetImmOp()->IsCnsIntOrI(); + } + + uint8_t GetImmediateLaneOperand() const + { + assert(category == HW_Category_IMM || category == HW_Category_MemoryLoad || + category == HW_Category_MemoryStore); + + GenTree* immOp = m_node->GetImmOp(); + assert(immOp->IsCnsIntOrI()); + ssize_t lane = immOp->AsIntCon()->IconValue(); + assert(FitsIn(lane)); + + return static_cast(lane); + } + NamedIntrinsic id; HWIntrinsicCategory category; GenTree* op1; @@ -1435,7 +1497,8 @@ struct HWIntrinsic final var_types baseType; private: - void InitializeOperands(const GenTreeHWIntrinsic* node) + const GenTreeHWIntrinsic* m_node; + void InitializeOperands(const GenTreeHWIntrinsic* node) { numOperands = node->GetOperandCount(); diff --git a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp index 310cb836db3ab3..d9d47e7914db7f 100644 --- a/src/coreclr/jit/hwintrinsiccodegenwasm.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenwasm.cpp @@ -26,7 +26,6 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX // void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { - // emitIns_v128_Imm // emitIns_Lane // emitIns_Memarg_Lane @@ -37,13 +36,29 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) { instruction const ins = HWIntrinsicInfo::lookupIns(info.id, info.baseType, m_compiler); assert(ins != INS_invalid); - if (info.category == HW_Category_SIMD) + switch (info.category) { - GetEmitter()->emitIns(ins); - } - else - { - NYI_WASM_SIMD("!HW_Category_SIMD"); + case HW_Category_SIMD: + { + GetEmitter()->emitIns(ins); + break; + } + case HW_Category_IMM: + { + if (info.needsJumpTableFallback()) + { + genHWIntrinsicJumpTableFallback(node, info); + } + else + { + GetEmitter()->emitIns_Lane(ins, info.GetImmediateLaneOperand()); + } + break; + } + default: + { + NYI_WASM_SIMD("CodeGen::genHWIntrinsic: Unsupported category for table-driven intrinsic"); + } } } else @@ -54,4 +69,128 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) WasmProduceReg(node); } +// genHWIntrinsicJumpTableFallback: Generates a jump table for a given hardware intrinsic node. +// Arguments: +// node - The hardware intrinsic node +// info - Hardware intrinsic info about the node +// +// Notes: +/* The structure emitted here is a series of nested blocks that looks like the following: + (block $outer + (block $inner + (block $(N-1) + ... + (block $1 + (block $0 + (br_table $0 $1 ... $N-1 $inner) + ) + {(local.get lcl_num(arg)) for each non-immediate operand} + imm=0 + br $outer + ) + ... + ) + {(local.get lcl_num(arg)) for each non-immediate operand} + imm=N-1 + br $outer + ) + unreachable + ) + Essentially, we create a block for each possible value of the immediate, and dispatch according to the immediate + value. In each case of the jump table, we re-materialize the non-immediate operands, whose result values are assigned + to locals by regalloc. Note that it is safe to emit these blocks mid-instruction stream, since we can logically think + of them as one macro "instruction" whose result is the same as the underlying instruction we are wrapping. There + aren't any GC safepoints in any of the generated cases here, but if we have to add any calls/throws/etc. in any of + the generated cases, this approach will need to change. +*/ +void CodeGen::genHWIntrinsicJumpTableFallback(GenTreeHWIntrinsic* node, HWIntrinsic info) +{ + assert(info.category == HW_Category_IMM || info.category == HW_Category_MemoryLoad || + info.category == HW_Category_MemoryStore); + + int simdSize = node->GetSimdSize(); + instruction const ins = HWIntrinsicInfo::lookupIns(info.id, info.baseType, m_compiler); + int immUpperBound = HWIntrinsicInfo::lookupImmUpperBound(info.id, simdSize, info.baseType); + WasmValueType resultType = ActualTypeToWasmValueType(genActualType(node->TypeGet())); + + GenTree* immOp = node->GetImmOp(); + regNumber immReg = GetMultiUseOperandReg(immOp); + + // Drop all incoming operands to actually consume them, they + // will be unusable in the jump table and we will need to re-materialize them + // on the value stack for each case in the table. + for (size_t i = 0; i < node->GetOperandCount(); i++) + { + GetEmitter()->emitIns(INS_drop); + } + + auto getNonImmediateOperands = [=]() { + for (size_t i = 1; i <= node->GetOperandCount(); i++) + { + GenTree* op = node->Op(i); + if (op != immOp) + { + // All of the operands should have been marked MultiplyUsed in lower, + // and so RA should have assigned them locals which prior codegen should + // have local.tee'd them into. + regNumber reg = GetMultiUseOperandReg(op); + GetEmitter()->emitIns_I(INS_local_get, emitActualTypeSize(op), WasmRegToIndex(reg)); + } + } + }; + + genEmitBeginBlock(resultType); // emit $outer block + genEmitBeginBlock(); // emit unreachable $inner block + { + // emit `immUpperBound+1` blocks. The iteration order of the loop doesn't matter here, + // we just need to stage the right number of blocks on the control flow stack. + for (int i = 0; i <= immUpperBound; i++) + { + // emit a block for each case + genEmitBeginBlock(); + } + + // In the innermost block, load the immediate value to branch to the appropriate case block + GetEmitter()->emitIns_I(INS_local_get, emitActualTypeSize(immOp), WasmRegToIndex(immReg)); + + // cases are 0 ... immUpperBound, default, where the last case is the default which branches to the unreachable + // inner block. Case 0 -> imm = 0, Case 1 -> imm = 1, and so on. + int caseCount = immUpperBound + 1; + GetEmitter()->emitIns_I(INS_br_table, EA_4BYTE, caseCount); + for (int caseNum = 0; caseNum <= immUpperBound; caseNum++) + { + GetEmitter()->emitIns_J(INS_label, EA_4BYTE, caseNum, nullptr); + } + // emit default case + GetEmitter()->emitIns_J(INS_label, EA_4BYTE, immUpperBound + 1, nullptr); + + assert(FitsIn(immUpperBound)); + for (int i = 0; i <= immUpperBound; i++) + { + genEmitEndBlock(); // End block for case i. The handling of case i follows + + getNonImmediateOperands(); + switch (info.category) + { + case HW_Category_IMM: + { + GetEmitter()->emitIns_Lane(ins, static_cast(i)); + break; + } + default: + { + NYI_WASM_SIMD( + "CodeGen::genHWIntrinsicJumpTableFallback: Unsupported category for jump table intrinsic"); + } + } + // proper branch depth is immUpperBound + 1 - i; The $inner block accounts for the + 1. + GetEmitter()->emitIns_J(INS_br, EA_4BYTE, static_cast(immUpperBound + 1 - i), + nullptr); // br $outer + } + } + genEmitEndBlock(); // end $inner block + GetEmitter()->emitIns(INS_unreachable); + genEmitEndBlock(); // end $outer block +} + #endif // FEATURE_HW_INTRINSICS diff --git a/src/coreclr/jit/hwintrinsiclistwasm.h b/src/coreclr/jit/hwintrinsiclistwasm.h index ae0eb686c75edd..3f48197b9d1f53 100644 --- a/src/coreclr/jit/hwintrinsiclistwasm.h +++ b/src/coreclr/jit/hwintrinsiclistwasm.h @@ -42,7 +42,7 @@ HARDWARE_INTRINSIC(PackedSimd, ConvertToSingle, HARDWARE_INTRINSIC(PackedSimd, ConvertToUInt32Saturate, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_i32x4_trunc_sat_u_f32x4, INS_i32x4_trunc_sat_u_f64x2_zero, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Divide, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_div, INS_f64x2_div, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Dot, 16, 2, INS_invalid, INS_invalid, INS_i32x4_dot_i16x8_s, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ExtractScalar, 16, 2, INS_i8x16_extract_lane_s, INS_i8x16_extract_lane_u, INS_i16x8_extract_lane_s, INS_i16x8_extract_lane_u, INS_i32x4_extract_lane, INS_i32x4_extract_lane, INS_i64x2_extract_lane, INS_i64x2_extract_lane, INS_f32x4_extract_lane, INS_f64x2_extract_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Floor, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_floor, INS_f64x2_floor, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndInsert, 16, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) HARDWARE_INTRINSIC(PackedSimd, LoadScalarAndSplatVector128, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_Helper, HW_Flag_InvalidNodeId|HW_Flag_SpecialImport) @@ -61,11 +61,11 @@ HARDWARE_INTRINSIC(PackedSimd, Or, HARDWARE_INTRINSIC(PackedSimd, PopCount, 16, 1, INS_invalid, INS_i8x16_popcnt, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMax, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmax, INS_f64x2_pmax, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, PseudoMin, 16, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_pmin, INS_f64x2_pmin, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ReplaceScalar, 16, 3, INS_i8x16_replace_lane, INS_i8x16_replace_lane, INS_i16x8_replace_lane, INS_i16x8_replace_lane, INS_i32x4_replace_lane, INS_i32x4_replace_lane, INS_i64x2_replace_lane, INS_i64x2_replace_lane, INS_f32x4_replace_lane, INS_f64x2_replace_lane, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, RoundToNearest, 16, 1, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_f32x4_nearest, INS_f64x2_nearest, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) -HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) -HARDWARE_INTRINSIC(PackedSimd, ShiftRightLogical, 16, 2, INS_i8x16_shr_u, INS_i8x16_shr_u, INS_i16x8_shr_u, INS_i16x8_shr_u, INS_i32x4_shr_u, INS_i32x4_shr_u, INS_i64x2_shr_u, INS_i64x2_shr_u, INS_invalid, INS_invalid, -1, -1, HW_Category_IMM, HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialImport) +HARDWARE_INTRINSIC(PackedSimd, ShiftLeft, 16, 2, INS_i8x16_shl, INS_i8x16_shl, INS_i16x8_shl, INS_i16x8_shl, INS_i32x4_shl, INS_i32x4_shl, INS_i64x2_shl, INS_i64x2_shl, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ShiftRightArithmetic, 16, 2, INS_i8x16_shr_s, INS_i8x16_shr_s, INS_i16x8_shr_s, INS_i16x8_shr_s, INS_i32x4_shr_s, INS_i32x4_shr_s, INS_i64x2_shr_s, INS_i64x2_shr_s, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(PackedSimd, ShiftRightLogical, 16, 2, INS_i8x16_shr_u, INS_i8x16_shr_u, INS_i16x8_shr_u, INS_i16x8_shr_u, INS_i32x4_shr_u, INS_i32x4_shr_u, INS_i64x2_shr_u, INS_i64x2_shr_u, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, Shuffle, 16, 3, INS_i8x16_shuffle, INS_i8x16_shuffle, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningLower, 16, 1, INS_i16x8_extend_low_s_i8x16, INS_i16x8_extend_low_s_i8x16, INS_i32x4_extend_low_s_i16x8, INS_i32x4_extend_low_s_i16x8, INS_i64x2_extend_low_s_i32x4, INS_i64x2_extend_low_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(PackedSimd, SignExtendWideningUpper, 16, 1, INS_i16x8_extend_high_s_i8x16, INS_i16x8_extend_high_s_i8x16, INS_i32x4_extend_high_s_i16x8, INS_i32x4_extend_high_s_i16x8, INS_i64x2_extend_high_s_i32x4, INS_i64x2_extend_high_s_i32x4, INS_invalid, INS_invalid, INS_invalid, INS_invalid, -1, -1, HW_Category_SIMD, HW_Flag_BaseTypeFromFirstArg) diff --git a/src/coreclr/jit/hwintrinsicwasm.cpp b/src/coreclr/jit/hwintrinsicwasm.cpp index 05d20b8f5f37ee..9e5fe0b4ec45f7 100644 --- a/src/coreclr/jit/hwintrinsicwasm.cpp +++ b/src/coreclr/jit/hwintrinsicwasm.cpp @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#include "jitpch.h" +#include "hwintrinsic.h" + #ifdef FEATURE_HW_INTRINSICS //------------------------------------------------------------------------ @@ -30,9 +33,19 @@ CORINFO_InstructionSet Compiler::lookupInstructionSet(const char* className) return InstructionSet_ILLEGAL; } -int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id, var_types baseType) +int HWIntrinsicInfo::lookupImmUpperBound(NamedIntrinsic id, unsigned int simdSize, var_types baseType) { - NYI_WASM_SIMD("lookupImmUpperBound"); + switch (id) + { + case NI_PackedSimd_ExtractScalar: + case NI_PackedSimd_ReplaceScalar: + case NI_PackedSimd_LoadScalarAndInsert: + case NI_PackedSimd_StoreSelectedScalar: + return Compiler::getSIMDVectorLength(simdSize, baseType) - 1; + default: + unreached(); + } + return 0; } @@ -140,18 +153,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - // The following PackedSimd intrinsics are not yet implemented on WASM. Because they are must-expand, - // when we return nullptr here the importer will insert a PlatformNotSupportedException throw. - case NI_PackedSimd_ExtractScalar: - { - break; - } - - case NI_PackedSimd_ReplaceScalar: - { - break; - } - case NI_PackedSimd_LoadVector128: { assert(sig->numArgs == 1); @@ -177,13 +178,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, break; } - case NI_PackedSimd_ShiftLeft: - case NI_PackedSimd_ShiftRightArithmetic: - case NI_PackedSimd_ShiftRightLogical: - { - break; - } - case NI_PackedSimd_Store: { assert(sig->numArgs == 2); @@ -230,11 +224,29 @@ void Compiler::getHWIntrinsicImmOps(NamedIntrinsic intrinsic, GenTree** immOp1Ptr, GenTree** immOp2Ptr) { - if ((sig->numArgs > 0) && HWIntrinsicInfo::isImmOp(intrinsic, impStackTop().val)) + if (!HWIntrinsicInfo::HasImmediateOperand(intrinsic)) + { + return; + } + + // Position of the immediates (in eval order) + int imm1Pos = -1; + int imm2Pos = -1; + + int numArgs = HWIntrinsicInfo::lookupNumArgs(intrinsic); + HWIntrinsicInfo::GetImmOpsPositions(intrinsic, &imm1Pos, &imm2Pos); + if (imm1Pos >= 0) + { + int imm1StackPos = numArgs - imm1Pos; + *immOp1Ptr = impStackTop(imm1StackPos).val; + assert(HWIntrinsicInfo::isImmOp(intrinsic, *immOp1Ptr)); + } + + if (imm2Pos >= 0) { - // NOTE: The following code assumes that for all intrinsics - // taking an immediate operand, that operand will be last. - *immOp1Ptr = impStackTop().val; + int imm2StackPos = numArgs - imm2Pos; + *immOp2Ptr = impStackTop(imm2StackPos).val; + assert(HWIntrinsicInfo::isImmOp(intrinsic, *immOp2Ptr)); } } #endif diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index b8435548fd8cf0..bd89d392fe12bb 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -519,6 +519,9 @@ class Lowering final : public Phase GenTree* LowerCnsMask(GenTreeMskCon* mask); bool TryLowerAddForPossibleContainment(GenTreeOp* node, GenTree** next); void StoreFFRValue(GenTreeHWIntrinsic* node); +#elif defined(TARGET_WASM) + GenTree* LowerHWIntrinsicCompareUnsignedLong(GenTreeHWIntrinsic* node); + GenTree* LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node); #endif // !TARGET_XARCH && !TARGET_ARM64 GenTree* InsertNewSimdCreateScalarUnsafeNode(var_types type, GenTree* op1, diff --git a/src/coreclr/jit/lowerwasm.cpp b/src/coreclr/jit/lowerwasm.cpp index 124a233e37cb3f..9bdaf3b927f14c 100644 --- a/src/coreclr/jit/lowerwasm.cpp +++ b/src/coreclr/jit/lowerwasm.cpp @@ -21,6 +21,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #endif #include "lower.h" +#include "compiler.h" void Lowering::SetMultiplyUsed(GenTree* node DEBUGARG(const char* reason)) { @@ -853,12 +854,119 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) return LowerHWIntrinsicCmpOp(node, GT_NE); } + case NI_PackedSimd_CompareLessThan: + case NI_PackedSimd_CompareLessThanOrEqual: + case NI_PackedSimd_CompareGreaterThan: + case NI_PackedSimd_CompareGreaterThanOrEqual: + { + if (node->GetSimdBaseType() == TYP_ULONG) + { + return LowerHWIntrinsicCompareUnsignedLong(node); + } + break; + } + + case NI_PackedSimd_ExtractScalar: + case NI_PackedSimd_ReplaceScalar: + { + assert(category == HW_Category_IMM); + return LowerHWIntrinsicWithImm(node); + } + default: { assert(category == HW_Category_SIMD); break; } } + + ContainCheckHWIntrinsic(node); + return node->gtNext; +} + +// -------------------------------------------------------- +// LowerHWIntrinsicWithImm: Lower a hardware intrinsic node with an immediate operand, and determine if +// it needs a jump table fallback. +// +// Arguments: +// node - The hardware intrinsic node. +// +// Notes: +// If the immediate operand is constant, it should be marked as contained. If not, then we set GTF_HW_NEEDS_JUMP_TABLE +// and mark the operands as multiply used so they'll be allocated wasm locals (needed for the jump table which uses +// nested blocks). +GenTree* Lowering::LowerHWIntrinsicWithImm(GenTreeHWIntrinsic* node) +{ + GenTree* immOp = node->GetImmOp(); + if (!immOp->IsCnsIntOrI()) + { + // This node has a non-constant immediate operand, so it will need a jump table + // to cover all the possible immediate values. On Wasm this involves introducing nested blocks, + // which requires us to set the operands as "multiply used" so regalloc assigns them locals. + for (size_t i = 1; i <= node->GetOperandCount(); i++) + { + GenTree* op = node->Op(i); + SetMultiplyUsed(op DEBUGARG("Non-constant imm op needs jump table fallback")); + } + } + + ContainCheckHWIntrinsic(node); + return node->gtNext; +} + +//---------------------------------------------------------------------------------------------- +// LowerHWIntrinsicCompareUnsignedLong: Rewrite a PackedSimd ordered ulong compare into a +// signed compare on sign-bit-flipped operands. +// +// Wasm SIMD does not provide unsigned i64x2 relative comparison opcodes. We apply the +// rewrite of +// cmp_u(a, b) == cmp_s(a VECTOR_XOR signbit_vec, b VECTOR_XOR signbit_vec) +// +// Arguments: +// node - The PackedSimd ordered compare with SimdBaseType TYP_ULONG. +// +// Return Value: +// The next node to lower. +// +GenTree* Lowering::LowerHWIntrinsicCompareUnsignedLong(GenTreeHWIntrinsic* node) +{ + assert(node->GetSimdBaseType() == TYP_ULONG); + assert(node->GetSimdSize() == 16); + + GenTree* op1 = node->Op(1); + GenTree* op2 = node->Op(2); + + // Create two independent 2-element constant vectors, with each element set to the sign bit for i64. + GenTreeVecCon* signMaskA = m_compiler->gtNewVconNode(TYP_SIMD16); + signMaskA->gtSimdVal.u64[0] = 0x8000000000000000ULL; + signMaskA->gtSimdVal.u64[1] = 0x8000000000000000ULL; + + GenTreeVecCon* signMaskB = m_compiler->gtNewVconNode(TYP_SIMD16); + signMaskB->gtSimdVal.u64[0] = 0x8000000000000000ULL; + signMaskB->gtSimdVal.u64[1] = 0x8000000000000000ULL; + + GenTreeHWIntrinsic* xorA = + m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, op1, signMaskA, NI_PackedSimd_Xor, TYP_LONG, 16); + + GenTreeHWIntrinsic* xorB = + m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, op2, signMaskB, NI_PackedSimd_Xor, TYP_LONG, 16); + + // The original LIR execution order is: ... op1 ... op2 ... node ... + // After rewrite we need: ... op1 ... signMaskA xorA op2 ... signMaskB xorB node ... + BlockRange().InsertAfter(op1, signMaskA, xorA); + BlockRange().InsertAfter(op2, signMaskB, xorB); + + LowerNode(signMaskA); + LowerNode(signMaskB); + + LowerNode(xorA); + LowerNode(xorB); + + node->Op(1) = xorA; + node->Op(2) = xorB; + node->SetSimdBaseType(TYP_LONG); + + ContainCheckHWIntrinsic(node); return node->gtNext; } @@ -1103,5 +1211,21 @@ GenTree* Lowering::LowerHWIntrinsicCreate(GenTreeHWIntrinsic* node) // void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) { - // TODO-WASM: implement containment for hardware intrinsics, currently a no-op + HWIntrinsicCategory category = HWIntrinsicInfo::lookupCategory(node->GetHWIntrinsicId()); + switch (category) + { + case HWIntrinsicCategory::HW_Category_IMM: + { + GenTree* immOp = node->GetImmOp(); + if (immOp->IsCnsIntOrI()) + { + MakeSrcContained(node, immOp); + } + break; + } + default: + { + break; + } + } } diff --git a/src/coreclr/jit/regallocwasm.cpp b/src/coreclr/jit/regallocwasm.cpp index accca547daf71f..4b201d6c44284e 100644 --- a/src/coreclr/jit/regallocwasm.cpp +++ b/src/coreclr/jit/regallocwasm.cpp @@ -503,6 +503,10 @@ void WasmRegAlloc::CollectReferencesForNode(GenTree* node) ConsumeTemporaryRegForOperand(node->gtGetOp1() DEBUGARG("ckfinite finiteness check")); break; + case GT_HWINTRINSIC: + CollectReferencesForHardwareIntrinsic(node->AsHWIntrinsic()); + break; + default: assert(!node->OperIsLocalStore()); break; @@ -696,6 +700,35 @@ void WasmRegAlloc::CollectReferencesForLclVar(GenTreeLclVar* lclVar) } } +// ------------------------------------------------------------------------ +// CollectReferencesForHardwareIntrinsic: Collect virtual register references for a hardware intrinsic. +// +// Arguments: +// node - The GT_HWINTRINSIC node +// +// Notes: +// This is a no-op unless a hw intrinsic needs a jump table fallback, in which case we have to consume +// temporary registers for its operands. +void WasmRegAlloc::CollectReferencesForHardwareIntrinsic(GenTreeHWIntrinsic* node) +{ + GenTree* op = node->GetImmOp(); + + // Only intrinsics that have a non-constant immediate need a jump-table fallback, and mark operands + // MultiplyUsed during Lowering (see Lowering::LowerHWIntrinsic in lowerwasm.cpp). + if (!op->IsCnsIntOrI()) + { + return; + } + + // All operands are marked multiply used, so we consume a temporary register for each operand + // in reverse (wasm stack) order. + int operandCount = static_cast(node->GetOperandCount()); + for (int i = operandCount; i >= 1; i--) + { + ConsumeTemporaryRegForOperand(node->Op(i) DEBUGARG("hardware intrinsic fallback")); + } +} + //------------------------------------------------------------------------ // RewriteLocalStackStore: rewrite a store to the stack to STOREIND(LCL_ADDR, ...). // diff --git a/src/coreclr/jit/regallocwasm.h b/src/coreclr/jit/regallocwasm.h index 344a51260955fc..038a188a507c94 100644 --- a/src/coreclr/jit/regallocwasm.h +++ b/src/coreclr/jit/regallocwasm.h @@ -159,6 +159,7 @@ class WasmRegAlloc : public RegAllocInterface void CollectReferencesForBlockStore(GenTreeBlk* node); void CollectReferencesForLclVar(GenTreeLclVar* lclVar); void CollectReferencesForIndexAddr(GenTreeIndexAddr* indexAddrNode); + void CollectReferencesForHardwareIntrinsic(GenTreeHWIntrinsic* node); void RewriteLocalStackStore(GenTreeLclVarCommon* node); void CollectReference(GenTree* node); void RequestTemporaryRegisterForMultiplyUsedNode(GenTree* node); diff --git a/src/coreclr/jit/targetwasm.cpp b/src/coreclr/jit/targetwasm.cpp index 839805f814cc74..0a9db2bf8cd5b6 100644 --- a/src/coreclr/jit/targetwasm.cpp +++ b/src/coreclr/jit/targetwasm.cpp @@ -72,7 +72,7 @@ ABIPassingInformation WasmClassifier::Classify(Compiler* comp, ClassLayout* structLayout, WellKnownArg wellKnownParam) { - if (type == TYP_STRUCT) + if (varTypeIsStruct(type)) { CORINFO_CLASS_HANDLE clsHnd = structLayout->GetClassHandle(); assert(clsHnd != NO_CLASS_HANDLE);