diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index a0b72b06c9a..701a47144a4 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -52,6 +52,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AdxLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -84,6 +85,11 @@ target_compile_features(relinker_core PUBLIC cxx_std_20) set_target_properties(relinker_core PROPERTIES CXX_EXTENSIONS OFF) if(BUILD_TESTING) + add_test_executable(adx_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/AdxLoweringTests.cpp) + target_link_libraries(adx_lowering_tests PRIVATE relinker_core) + add_test(NAME adx_lowering COMMAND adx_lowering_tests) + set_tests_properties(adx_lowering PROPERTIES TIMEOUT 20) + add_test_executable(andn_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/AndnLoweringTests.cpp) target_link_libraries(andn_lowering_tests PRIVATE relinker_core) add_test(NAME andn_lowering COMMAND andn_lowering_tests) @@ -176,6 +182,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AdxLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -361,6 +368,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AdxLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/AdxLowering.hpp b/core/relinker/codegen/include/codegen/x86/AdxLowering.hpp new file mode 100644 index 00000000000..f470ea0e495 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/AdxLowering.hpp @@ -0,0 +1,27 @@ +#ifndef CODEGEN_X86_ADXLOWERING_HPP +#define CODEGEN_X86_ADXLOWERING_HPP + +#include +#include +#include + +namespace Codegen { + +struct AdxOperands { + bool Overflow; + bool Wide; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Rex; + std::size_t OpcodeOffset; + std::size_t ModRmOffset; + std::vector Instruction; + std::optional Memory; +}; + +[[nodiscard]] std::optional DecodeAdx(const std::uint8_t* data, std::size_t length); +void EmitAdx(StubBodyBuilder& body, const AdxOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/AdxLowering.cpp b/core/relinker/codegen/src/x86/AdxLowering.cpp new file mode 100644 index 00000000000..ef32ddaa7ab --- /dev/null +++ b/core/relinker/codegen/src/x86/AdxLowering.cpp @@ -0,0 +1,226 @@ +#include +#include +#include +#include +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kRexBase = 0x40; +constexpr std::uint8_t kRexW = 0x08; +constexpr std::uint8_t kRexR = 0x04; +constexpr std::uint8_t kRexX = 0x02; +constexpr std::uint8_t kRexB = 0x01; +constexpr std::int32_t kFrameSize = 144; +constexpr std::uint8_t kStackRegister = 4; + +void _append32(std::vector& bytes, const std::uint32_t value) { + for (unsigned shift = 0; shift < 32; shift += 8) + bytes.push_back(static_cast(value >> shift)); +} + +std::uint8_t _scratch(const AdxOperands& operands) { + std::array used{}; + used[kStackRegister] = true; + used[operands.Destination] = true; + if (!operands.Memory) { + used[operands.Source] = true; + } else { + const auto& memory = *operands.Memory; + if (memory.Rm == 4) { + const auto base = static_cast((memory.Sib & 7) | ((memory.RexIndexBase & kRexB) << 3)); + const auto index = static_cast(((memory.Sib >> 3) & 7) | ((memory.RexIndexBase & kRexX) << 2)); + if (memory.Mod != 0 || (memory.Sib & 7) != 5 || (memory.RexIndexBase & kRexB) != 0) + used[base] = true; + if (((memory.Sib >> 3) & 7) != 4 || (memory.RexIndexBase & kRexX) != 0) + used[index] = true; + } else if (!memory.RipRelative) { + used[memory.Rm | ((memory.RexIndexBase & kRexB) << 3)] = true; + } + } + for (const auto reg : {11, 10, 9, 8, 0, 1, 2, 3, 5, 6, 7, 12, 13, 14, 15}) + if (!used[reg]) + return static_cast(reg); + throw CodegenException("ADCX/ADOX has no scratch register"); +} + +void _appendRex(std::vector& bytes, const std::uint8_t rex) { + if (rex != kRexBase) + bytes.push_back(rex); +} + +void _emitRegMemory(std::vector& bytes, const std::uint8_t opcode, const std::uint8_t reg, const std::uint8_t base, const std::int8_t displacement) { + _appendRex(bytes, static_cast(kRexBase | kRexW | ((reg & 8) != 0 ? kRexR : 0) | ((base & 8) != 0 ? kRexB : 0))); + bytes.insert(bytes.end(), {opcode, static_cast(0x44 | ((reg & 7) << 3)), 0x24, static_cast(displacement)}); +} + +void _emitRegStack(std::vector& bytes, const std::uint8_t opcode, const std::uint8_t reg, const std::int8_t displacement) { + _emitRegMemory(bytes, opcode, reg, kStackRegister, displacement); +} + +std::vector _translatedInstruction(const AdxOperands& operands, const std::uint8_t scratch) { + using namespace X64OpcodeConstants; + std::vector bytes; + for (std::size_t index = 0; index < operands.OpcodeOffset; ++index) { + const auto prefix = operands.Instruction[index]; + if (prefix != (operands.Overflow ? PrefixRep : PrefixOperandSize) && (prefix < RexMin || prefix > RexMax)) + bytes.push_back(operands.Instruction[index]); + } + auto rex = operands.Rex; + if (!operands.Memory && operands.Source == kStackRegister) + rex = static_cast((rex & (kRexW | kRexR)) | ((scratch & 8) != 0 ? kRexB : 0)); + const auto encodedRex = static_cast(kRexBase | rex); + _appendRex(bytes, encodedRex); + bytes.push_back(0x13); + const auto originalModrm = operands.Instruction[operands.ModRmOffset]; + if (operands.Memory && operands.Memory->StackBase) { + const auto& memory = *operands.Memory; + const auto displacement = static_cast(memory.Displacement) + kFrameSize; + const auto address32 = std::find_if(operands.Instruction.begin(), operands.Instruction.begin() + static_cast(operands.OpcodeOffset), [](const auto prefix) { return prefix == PrefixAddressSize; }) != operands.Instruction.begin() + static_cast(operands.OpcodeOffset); + if (!address32 && (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max())) + throw CodegenException("ADCX/ADOX stack displacement does not fit after spilling"); + bytes.push_back(static_cast((originalModrm & (ModRmRegMask << ModRmRegShift)) | (ModRmModDisp32 << ModRmModShift) | ModRmRmSibPresent)); + bytes.push_back(memory.Sib); + _append32(bytes, static_cast(displacement)); + } else if (!operands.Memory) { + const auto source = operands.Source == kStackRegister ? scratch : operands.Source; + bytes.push_back(static_cast((originalModrm & (ModRmModMask << ModRmModShift | ModRmRegMask << ModRmRegShift)) | (source & ModRmRmMask))); + } else { + bytes.insert(bytes.end(), operands.Instruction.begin() + static_cast(operands.ModRmOffset), operands.Instruction.end()); + } + return bytes; +} + +void _emitImmediate(std::vector& bytes, const std::uint8_t opcode, const std::uint8_t extension, const std::uint8_t reg, const std::uint32_t immediate) { + _appendRex(bytes, static_cast(kRexBase | kRexW | ((reg & 8) != 0 ? kRexB : 0))); + bytes.insert(bytes.end(), {opcode, static_cast(0xC0 | (extension << 3) | (reg & 7))}); + _append32(bytes, immediate); +} + +void _emitStackAnd(std::vector& bytes, const std::uint32_t immediate) { + bytes.insert(bytes.end(), {0x48, 0x81, 0x64, 0x24, 0x08}); + _append32(bytes, immediate); +} + +void _emitStackPushPopFlags(std::vector& bytes) { + bytes.insert(bytes.end(), {0xFF, 0x74, 0x24, 0x08, 0x9D}); +} + +} + +std::optional DecodeAdx(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + std::size_t opcode = 0; + std::size_t operandPrefixes = 0; + std::size_t repPrefixes = 0; + std::size_t repnePrefixes = 0; + std::size_t lockPrefixes = 0; + std::uint8_t rex = 0; + while (opcode < length) { + const auto value = data[opcode]; + if (value == PrefixOperandSize) { + ++operandPrefixes; + rex = 0; + } else if (value == PrefixRep) { + ++repPrefixes; + rex = 0; + } else if (value == PrefixRepne) { + ++repnePrefixes; + rex = 0; + } else if (value == PrefixLock) { + ++lockPrefixes; + rex = 0; + } else if (value == PrefixAddressSize || value == PrefixSegCs || value == PrefixSegSs || value == PrefixSegDs || value == PrefixSegEs || value == PrefixSegFs || value == PrefixSegGs) { + rex = 0; + } else if (value >= RexMin && value <= RexMax) { + rex = value; + } else { + break; + } + ++opcode; + } + if (opcode + 2 >= length || data[opcode] != TwoByteOpcodeEscape || data[opcode + 1] != ThreeByteEscape38 || data[opcode + 2] != 0xF6) + return std::nullopt; + const bool adcx = operandPrefixes == 1 && repPrefixes == 0 && repnePrefixes == 0; + const bool adox = repPrefixes == 1 && operandPrefixes == 0 && repnePrefixes == 0; + if (!adcx && !adox) + return std::nullopt; + if (lockPrefixes != 0) + throw CodegenException("ADCX/ADOX does not accept the LOCK prefix"); + const auto modrmOffset = opcode + 3; + if (modrmOffset >= length) + throw CodegenException("ADCX/ADOX truncated before its ModRM byte"); + const auto modrm = data[modrmOffset]; + const auto destination = static_cast(((modrm >> ModRmRegShift) & ModRmRegMask) | ((rex & kRexR) != 0 ? 8 : 0)); + const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); + AdxOperands operands{adox, (rex & kRexW) != 0, destination, + static_cast((modrm & ModRmRmMask) | ((rex & kRexB) != 0 ? 8 : 0)), rex, opcode, modrmOffset, + {data, data + length}, std::nullopt}; + if (mod == ModRmModRegister) { + if (modrmOffset + 1 != length) + throw CodegenException("ADCX/ADOX register operand has trailing bytes"); + } else { + std::vector prefixes(data, data + opcode); + operands.Memory = DecodeMemoryOperand(data, length, modrmOffset, static_cast(rex & (kRexX | kRexB)), std::move(prefixes)); + const auto address32 = std::find(data, data + opcode, PrefixAddressSize) != data + opcode; + if (address32 && operands.Memory->RipRelative) + operands.Memory->RipRelative = false; + if (modrmOffset + operands.Memory->EncodedSize != length) + throw CodegenException("ADCX/ADOX memory operand has trailing bytes"); + } + return operands; +} + +void EmitAdx(StubBodyBuilder& body, const AdxOperands& operands) { + if (operands.Destination == kStackRegister) + throw CodegenException("ADCX/ADOX with RSP destination cannot be lowered through a trampoline"); + const auto scratch = _scratch(operands); + const auto flagMask = operands.Overflow ? std::uint32_t{0x800} : std::uint32_t{1}; + const auto clearedMask = ~flagMask; + std::vector prologue{0x48, 0x8D, 0xA4, 0x24, 0x70, 0xFF, 0xFF, 0xFF}; + _emitRegStack(prologue, 0x89, scratch, 0); + prologue.push_back(0x9C); + _emitRegMemory(prologue, 0x8B, scratch, kStackRegister, 0); + prologue.push_back(0x9D); + _emitRegStack(prologue, 0x89, scratch, 8); + body.Raw(prologue); + if (operands.Source == kStackRegister && !operands.Memory) { + std::vector originalRsp; + _appendRex(originalRsp, static_cast(kRexBase | kRexW | ((scratch & 8) != 0 ? kRexR : 0))); + originalRsp.push_back(0x8D); + originalRsp.push_back(static_cast(0x84 | ((scratch & 7) << 3))); + originalRsp.push_back(0x24); + _append32(originalRsp, kFrameSize); + body.Raw(originalRsp); + } + auto instruction = _translatedInstruction(operands, scratch); + if (operands.Overflow) + body.Raw(std::array{0x48, 0x0F, 0xBA, 0x64, 0x24, 0x08}); + if (operands.Overflow) + body.Raw(std::array{0x0B}); + if (operands.Memory && operands.Memory->RipRelative) { + const auto displacementOffset = instruction.size() - 4; + body.RipRelative(instruction, displacementOffset, static_cast(operands.Instruction.size()) + operands.Memory->Displacement); + } else { + body.Raw(instruction); + } + std::vector epilogue{0x9C}; + _emitRegMemory(epilogue, 0x8B, scratch, kStackRegister, 0); + epilogue.push_back(0x9D); + _emitStackAnd(epilogue, clearedMask); + _emitImmediate(epilogue, 0x81, 4, scratch, 1); + if (operands.Overflow) { + _appendRex(epilogue, static_cast(kRexBase | kRexW | ((scratch & 8) != 0 ? kRexB : 0))); + epilogue.insert(epilogue.end(), {0xC1, static_cast(0xE0 | (scratch & 7)), 11}); + } + _emitRegMemory(epilogue, 0x09, scratch, kStackRegister, 8); + _emitStackPushPopFlags(epilogue); + _emitRegMemory(epilogue, 0x8B, scratch, kStackRegister, 0); + epilogue.insert(epilogue.end(), {0x48, 0x8D, 0xA4, 0x24, 0x90, 0x00, 0x00, 0x00}); + body.Raw(epilogue); +} + +} diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 7b6175bdff3..5973abda44e 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -218,6 +219,8 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchClzero(const DecodedInstructio std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const DecodedInstruction& instr, std::span trailing) const { if (_target != Amd64OnlyTarget::Rosetta) return std::nullopt; + if (const auto adx = DecodeAdx(instr.Data, instr.Length)) + return _trampoline(adx->Overflow ? "ADOX" : "ADCX", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAdx(body, *adx); })); if (const auto andn = DecodeAndn(instr.Data, instr.Length)) return _trampoline("ANDN", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAndn(body, *andn); })); if (instr.IsRdseed()) { @@ -268,6 +271,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitAndn(body, *andn); return "ANDN"; } + if (const auto adx = _target == Amd64OnlyTarget::Rosetta ? DecodeAdx(instr.Data, instr.Length) : std::nullopt) { + EmitAdx(body, *adx); + return adx->Overflow ? "ADOX" : "ADCX"; + } if (const auto wide = _target == Amd64OnlyTarget::Rosetta ? DecodeVexWideStore(instr.Data, instr.Length) : std::nullopt) { _wideStoreLowering.EmitOutOfLine(body, *wide); return kWideStore.Name; diff --git a/core/relinker/codegen/tests/AdxLoweringTests.cpp b/core/relinker/codegen/tests/AdxLoweringTests.cpp new file mode 100644 index 00000000000..b1018581602 --- /dev/null +++ b/core/relinker/codegen/tests/AdxLoweringTests.cpp @@ -0,0 +1,230 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) +#include +#include +#endif + +namespace { + +using Bytes = std::vector; + +void require(const bool condition, const char* message) { + if (!condition) + throw std::runtime_error(message); +} + +void decoding() { + const Bytes adcx32{0x66, 0x0F, 0x38, 0xF6, 0xC1}; + const Bytes adcx64{0x66, 0x4C, 0x0F, 0x38, 0xF6, 0xC1}; + const Bytes adox64{0xF3, 0x4C, 0x0F, 0x38, 0xF6, 0xC2}; + const Bytes adcxStack{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x44, 0x24, 0x08}; + const Bytes adcxRip{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x05, 0x10, 0x00, 0x00, 0x00}; + const Bytes adcxAddress32{0x67, 0x66, 0x0F, 0x38, 0xF6, 0x05, 0x10, 0x00, 0x00, 0x00}; + const auto first = Codegen::DecodeAdx(adcx32.data(), adcx32.size()); + const auto second = Codegen::DecodeAdx(adcx64.data(), adcx64.size()); + const auto third = Codegen::DecodeAdx(adox64.data(), adox64.size()); + const auto fourth = Codegen::DecodeAdx(adcxStack.data(), adcxStack.size()); + const auto fifth = Codegen::DecodeAdx(adcxRip.data(), adcxRip.size()); + const auto sixth = Codegen::DecodeAdx(adcxAddress32.data(), adcxAddress32.size()); + require(first && !first->Overflow && !first->Wide && first->Destination == 0 && first->Source == 1, "ADCX 32-bit registers decoded incorrectly"); + require(second && !second->Overflow && second->Wide && second->Destination == 8 && second->Source == 1, "ADCX REX extensions decoded incorrectly"); + require(third && third->Overflow && third->Wide && third->Destination == 8 && third->Source == 2, "ADOX REX extensions decoded incorrectly"); + require(fourth && fourth->Memory && fourth->Memory->StackBase && fourth->Memory->Displacement == 8, "ADCX stack operand decoded incorrectly"); + require(fifth && fifth->Memory && fifth->Memory->RipRelative, "ADCX RIP-relative source was not identified"); + require(sixth && sixth->Memory && !sixth->Memory->RipRelative, "ADCX address-size source was mistaken for RIP-relative"); + for (const auto& bytes : {Bytes{0x0F, 0x38, 0xF6, 0xC1}, Bytes{0xF2, 0x0F, 0x38, 0xF6, 0xC1}, Bytes{0x66, 0xF3, 0x0F, 0x38, 0xF6, 0xC1}}) + require(!Codegen::DecodeAdx(bytes.data(), bytes.size()), "An instruction without the mandatory ADX prefix matched"); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Intel); + require(rosetta->Match(adcx32.data(), adcx32.size()).has_value(), "Rosetta mode did not lower ADCX"); + require(rosetta->Match(adox64.data(), adox64.size()).has_value(), "Rosetta mode did not lower ADOX"); + require(rosetta->Match(adcxRip.data(), adcxRip.size())->Relocations.size() == 1, "ADCX RIP-relative source has no relocation"); + require(!intel->Match(adcx32.data(), adcx32.size()), "Intel mode rewrote ADCX"); + bool rejected = false; + const Bytes rspDestination{0x66, 0x0F, 0x38, 0xF6, 0xE1}; + try { + (void)rosetta->Match(rspDestination.data(), rspDestination.size()); + } catch (const Codegen::CodegenException&) { + rejected = true; + } + require(rejected, "ADCX with an RSP destination was silently accepted"); +} + +void conversion() { + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); + const std::array adxOpcode{0x0F, 0x38, 0xF6}; + for (const auto& instruction : {Bytes{0x66, 0x0F, 0x38, 0xF6, 0xC1}, Bytes{0xF3, 0x0F, 0x38, 0xF6, 0xC1}}) { + Bytes file(0x300, 0xCC); + std::copy(instruction.begin(), instruction.end(), file.begin() + 0x200); + const auto trailing = instruction.size() == 4 ? std::uint8_t{0x90} : std::uint8_t{0xC3}; + file[0x200 + instruction.size()] = trailing; + file[0x200 + instruction.size() + 1] = 0xC3; + const Domain::ProgramHeader segment{1, 5, 0x200, 0x1000, 0, instruction.size() + (trailing == 0x90 ? 2 : 1), instruction.size() + (trailing == 0x90 ? 2 : 1), 16}; + const auto result = converter->Convert(file, {segment}); + require(result.Trampolines.size() == 1 && result.Reports.size() == 1, "The converter did not lower an ADX instruction"); + const auto& trampoline = result.Trampolines.front(); + require(result.Reports.front().InstructionName == (instruction.front() == 0x66 ? "ADCX" : "ADOX"), "The converter reported the wrong ADX instruction"); + require(std::search(trampoline.Body.begin(), trampoline.Body.begin() + static_cast(trampoline.ReturnBranchOffset), adxOpcode.begin(), adxOpcode.end()) == trampoline.Body.begin() + static_cast(trampoline.ReturnBranchOffset), "The converter left an unsupported ADX opcode in the trampoline"); + } +} + +#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) + +struct Run { + std::uint64_t Value; + std::uint64_t Flags; + std::uint64_t Stack; +}; + +Run executeCode(Bytes codeBytes, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags, const std::uint64_t memorySource = 0, const std::optional returnBranchOffset = std::nullopt) { + const auto ret = codeBytes.size(); + if (returnBranchOffset) { + const auto displacement = static_cast(ret - (*returnBranchOffset + 5)); + std::memcpy(codeBytes.data() + *returnBranchOffset + 1, &displacement, sizeof(displacement)); + } + codeBytes.push_back(0xC3); + auto* code = static_cast(mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + require(code != MAP_FAILED, "Cannot allocate executable memory for an ADX stub"); + std::memcpy(code, codeBytes.data(), codeBytes.size()); + Run result{}; + asm volatile( + "mov %[destination], %%rax\n\t" + "mov %[source], %%rcx\n\t" + "mov %[source], %%r9\n\t" + "sub $128, %%rsp\n\t" + "mov %[memorySource], (%%rsp)\n\t" + "lea -8(%%rsp), %[stack]\n\t" + "mov %[initialFlags], %%r11\n\t" + "push %%r11\n\t" + "popfq\n\t" + "call *%[code]\n\t" + "pushfq\n\t" + "pop %[flags]\n\t" + "add $128, %%rsp\n\t" + "mov %%rax, %[value]\n\t" + : [value] "=&r"(result.Value), [flags] "=&r"(result.Flags), [stack] "=&r"(result.Stack) + : [destination] "r"(destination), [source] "r"(source), [initialFlags] "r"(initialFlags), [memorySource] "r"(memorySource), [code] "r"(code) + : "rax", "rcx", "r9", "r11", "memory", "cc"); + munmap(code, 4096); + return result; +} + +Run execute(const Bytes& site, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags, const std::uint64_t memorySource = 0) { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto match = matcher->Match(site.data(), site.size()); + require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "An ADX trampoline was not produced"); + return executeCode(match->StubBody, destination, source, initialFlags, memorySource, match->ReturnBranchOffset); +} + +Run executeNative(const bool wide, const bool adox, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags) { + Bytes instruction{static_cast(adox ? 0xF3 : 0x66)}; + if (wide) + instruction.push_back(0x48); + instruction.insert(instruction.end(), {0x0F, 0x38, 0xF6, 0xC1, 0xC3}); + return executeCode(std::move(instruction), destination, source, initialFlags); +} + +std::uint64_t expectedResult(const bool wide, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t carry) { + if (wide) + return destination + source + carry; + return static_cast(destination) + static_cast(source) + static_cast(carry); +} + +void execution() { + for (const bool wide : {false, true}) { + for (const bool adox : {false, true}) { + Bytes site{static_cast(adox ? 0xF3 : 0x66)}; + if (wide) + site.push_back(0x48); + site.insert(site.end(), {0x0F, 0x38, 0xF6, 0xC1}); + const auto mask = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; + const auto high = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; + const auto upperBits = wide ? std::uint64_t{2} : std::uint64_t{0xDEADBEEF00000001}; + const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high, upperBits}; + for (const auto destination : values) { + for (const auto source : values) { + for (const auto inputFlag : {std::uint64_t{0}, std::uint64_t{1}}) { + const auto flagMask = adox ? std::uint64_t{0x800} : std::uint64_t{1}; + const auto initial = (std::uint64_t{0x8D7} & ~flagMask) | (inputFlag ? flagMask : 0); + const auto run = execute(site, destination, source, initial); + const auto expected = expectedResult(wide, destination, source, inputFlag) & mask; + require(run.Value == expected, "ADX register lowering produced the wrong sum"); + const bool carry = wide + ? (expected < destination || (inputFlag != 0 && expected == destination)) + : ((static_cast(static_cast(destination)) + static_cast(source) + inputFlag) >> 32) != 0; + const auto expectedFlags = (initial & ~flagMask) | (carry ? flagMask : 0); + require((run.Flags & 0x8D5) == (expectedFlags & 0x8D5), "ADX lowering produced the wrong carry flag or changed an unrelated arithmetic flag"); + } + } + } + } + } + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + const bool hasAdx = __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) != 0 && (ebx & (1u << 19)) != 0; + if (hasAdx) { + for (const bool wide : {false, true}) { + for (const bool adox : {false, true}) { + const auto high = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; + const auto upperBits = wide ? std::uint64_t{2} : std::uint64_t{0xDEADBEEF00000001}; + const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high, upperBits}; + for (const auto destination : values) { + for (const auto source : values) { + for (const auto inputFlag : {std::uint64_t{0}, std::uint64_t{1}}) { + const auto flagMask = adox ? std::uint64_t{0x800} : std::uint64_t{1}; + const auto initial = (std::uint64_t{0x8D7} & ~flagMask) | (inputFlag ? flagMask : 0); + Bytes site{static_cast(adox ? 0xF3 : 0x66)}; + if (wide) + site.push_back(0x48); + site.insert(site.end(), {0x0F, 0x38, 0xF6, 0xC1}); + const auto lowered = execute(site, destination, source, initial); + const auto native = executeNative(wide, adox, destination, source, initial); + require(lowered.Value == native.Value && (lowered.Flags & 0x8D5) == (native.Flags & 0x8D5), "ADX lowering differs from native ADX"); + } + } + } + } + } + } + const Bytes stackSource{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x44, 0x24, 0x08}; + const auto stackRun = execute(stackSource, 5, 0, 0x202, 7); + require(stackRun.Value == 12 && (stackRun.Flags & 1) == 0, "ADX lowering did not compensate an RSP-relative memory source"); + const Bytes rspSource{0x66, 0x48, 0x0F, 0x38, 0xF6, 0xC4}; + const auto rspRun = execute(rspSource, 5, 0, 0xA03); + require(rspRun.Value == rspRun.Stack + 6, "ADX lowering did not preserve the original RSP source value and carry-in"); + require(((rspRun.Flags ^ 0xA03) & 0x8D4) == 0, "ADX lowering did not preserve unrelated flags with RSP as the source"); + const Bytes highSource{0x66, 0x49, 0x0F, 0x38, 0xF6, 0xC1}; + const auto highRun = execute(highSource, 3, 5, 0x202); + require(highRun.Value == 8, "ADX lowering did not preserve the high source-register extension"); +} + +#else + +void execution() {} + +#endif + +} + +int main() { + try { + decoding(); + conversion(); + execution(); + std::cout << "ADX lowering tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index acab2b4e54e..487e11cff3a 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -22,3 +22,4 @@ - `--trace-imports` ([entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp)) exists for Windows output only. Only imports bound through a PLT slot, or through a GOT slot whose symbol is a function, are traced; a function reached through any other relocation, or looked up at run time with `sceKernelDlsym`, is not, and neither are weak imports or the imports of modules loaded at run time - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws; other BMI instructions are not lowered. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. +- The `--to-rosetta` ADCX/ADOX lowering follows their entries in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf), preserving the other arithmetic flags. An instruction with RSP as its destination register is rejected because changing the stack pointer inside the spill trampoline cannot be safely restored.