From 63c454aa639890e46f1db3257542dd3f97b3cbbc Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Sat, 17 Oct 2026 14:53:03 +0000 Subject: [PATCH 1/4] fix(relinker): lower ADCX and ADOX for Rosetta --- core/relinker/CMakeLists.txt | 8 + .../include/codegen/x86/AdxLowering.hpp | 27 +++ core/relinker/codegen/src/x86/AdxLowering.cpp | 222 ++++++++++++++++++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 7 + .../codegen/tests/AdxLoweringTests.cpp | 185 +++++++++++++++ docs/dev/debt/relinker.md | 1 + 6 files changed, 450 insertions(+) create mode 100644 core/relinker/codegen/include/codegen/x86/AdxLowering.hpp create mode 100644 core/relinker/codegen/src/x86/AdxLowering.cpp create mode 100644 core/relinker/codegen/tests/AdxLoweringTests.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index a0b72b06c9a..701a47144a4 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -52,6 +52,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AdxLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -84,6 +85,11 @@ target_compile_features(relinker_core PUBLIC cxx_std_20) set_target_properties(relinker_core PROPERTIES CXX_EXTENSIONS OFF) if(BUILD_TESTING) + add_test_executable(adx_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/AdxLoweringTests.cpp) + target_link_libraries(adx_lowering_tests PRIVATE relinker_core) + add_test(NAME adx_lowering COMMAND adx_lowering_tests) + set_tests_properties(adx_lowering PROPERTIES TIMEOUT 20) + add_test_executable(andn_lowering_tests ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/AndnLoweringTests.cpp) target_link_libraries(andn_lowering_tests PRIVATE relinker_core) add_test(NAME andn_lowering COMMAND andn_lowering_tests) @@ -176,6 +182,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AdxLowering.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -361,6 +368,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AndnLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/AdxLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/AdxLowering.hpp b/core/relinker/codegen/include/codegen/x86/AdxLowering.hpp new file mode 100644 index 00000000000..f470ea0e495 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/AdxLowering.hpp @@ -0,0 +1,27 @@ +#ifndef CODEGEN_X86_ADXLOWERING_HPP +#define CODEGEN_X86_ADXLOWERING_HPP + +#include +#include +#include + +namespace Codegen { + +struct AdxOperands { + bool Overflow; + bool Wide; + std::uint8_t Destination; + std::uint8_t Source; + std::uint8_t Rex; + std::size_t OpcodeOffset; + std::size_t ModRmOffset; + std::vector Instruction; + std::optional Memory; +}; + +[[nodiscard]] std::optional DecodeAdx(const std::uint8_t* data, std::size_t length); +void EmitAdx(StubBodyBuilder& body, const AdxOperands& operands); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/AdxLowering.cpp b/core/relinker/codegen/src/x86/AdxLowering.cpp new file mode 100644 index 00000000000..32cfe85eafa --- /dev/null +++ b/core/relinker/codegen/src/x86/AdxLowering.cpp @@ -0,0 +1,222 @@ +#include +#include +#include +#include +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kRexBase = 0x40; +constexpr std::uint8_t kRexW = 0x08; +constexpr std::uint8_t kRexR = 0x04; +constexpr std::uint8_t kRexX = 0x02; +constexpr std::uint8_t kRexB = 0x01; +constexpr std::int32_t kFrameSize = 144; +constexpr std::uint8_t kStackRegister = 4; + +void _append32(std::vector& bytes, const std::uint32_t value) { + for (unsigned shift = 0; shift < 32; shift += 8) + bytes.push_back(static_cast(value >> shift)); +} + +std::uint8_t _scratch(const AdxOperands& operands) { + std::array used{}; + used[kStackRegister] = true; + used[operands.Destination] = true; + if (!operands.Memory) { + used[operands.Source] = true; + } else { + const auto& memory = *operands.Memory; + if (memory.Rm == 4) { + const auto base = static_cast((memory.Sib & 7) | ((memory.RexIndexBase & kRexB) << 3)); + const auto index = static_cast(((memory.Sib >> 3) & 7) | ((memory.RexIndexBase & kRexX) << 2)); + if (memory.Mod != 0 || (memory.Sib & 7) != 5 || (memory.RexIndexBase & kRexB) != 0) + used[base] = true; + if (((memory.Sib >> 3) & 7) != 4 || (memory.RexIndexBase & kRexX) != 0) + used[index] = true; + } else if (!memory.RipRelative) { + used[memory.Rm | ((memory.RexIndexBase & kRexB) << 3)] = true; + } + } + for (const auto reg : {11, 10, 9, 8, 0, 1, 2, 3, 5, 6, 7, 12, 13, 14, 15}) + if (!used[reg]) + return static_cast(reg); + throw CodegenException("ADCX/ADOX has no scratch register"); +} + +void _appendRex(std::vector& bytes, const std::uint8_t rex) { + if (rex != kRexBase) + bytes.push_back(rex); +} + +void _emitRegMemory(std::vector& bytes, const std::uint8_t opcode, const std::uint8_t reg, const std::uint8_t base, const std::int8_t displacement) { + _appendRex(bytes, static_cast(kRexBase | kRexW | ((reg & 8) != 0 ? kRexR : 0) | ((base & 8) != 0 ? kRexB : 0))); + bytes.insert(bytes.end(), {opcode, static_cast(0x44 | ((reg & 7) << 3)), 0x24, static_cast(displacement)}); +} + +void _emitRegStack(std::vector& bytes, const std::uint8_t opcode, const std::uint8_t reg, const std::int8_t displacement) { + _emitRegMemory(bytes, opcode, reg, kStackRegister, displacement); +} + +std::vector _translatedInstruction(const AdxOperands& operands, const std::uint8_t scratch) { + using namespace X64OpcodeConstants; + std::vector bytes; + for (std::size_t index = 0; index < operands.OpcodeOffset; ++index) { + const auto prefix = operands.Instruction[index]; + if (prefix != (operands.Overflow ? PrefixRep : PrefixOperandSize) && (prefix < RexMin || prefix > RexMax)) + bytes.push_back(operands.Instruction[index]); + } + auto rex = operands.Rex; + if (!operands.Memory && operands.Source == kStackRegister) + rex = static_cast((rex & (kRexW | kRexR)) | ((scratch & 8) != 0 ? kRexB : 0)); + const auto encodedRex = static_cast(kRexBase | rex); + _appendRex(bytes, encodedRex); + bytes.push_back(0x13); + const auto originalModrm = operands.Instruction[operands.ModRmOffset]; + if (operands.Memory && operands.Memory->StackBase) { + const auto& memory = *operands.Memory; + const auto displacement = static_cast(memory.Displacement) + kFrameSize; + const auto address32 = std::find_if(operands.Instruction.begin(), operands.Instruction.begin() + static_cast(operands.OpcodeOffset), [](const auto prefix) { return prefix == PrefixAddressSize; }) != operands.Instruction.begin() + static_cast(operands.OpcodeOffset); + if (!address32 && (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max())) + throw CodegenException("ADCX/ADOX stack displacement does not fit after spilling"); + bytes.push_back(static_cast((originalModrm & (ModRmRegMask << ModRmRegShift)) | (ModRmModDisp32 << ModRmModShift) | ModRmRmSibPresent)); + bytes.push_back(memory.Sib); + _append32(bytes, static_cast(displacement)); + } else if (!operands.Memory) { + const auto source = operands.Source == kStackRegister ? scratch : operands.Source; + bytes.push_back(static_cast((originalModrm & (ModRmModMask << ModRmModShift | ModRmRegMask << ModRmRegShift)) | (source & ModRmRmMask))); + } else { + bytes.insert(bytes.end(), operands.Instruction.begin() + static_cast(operands.ModRmOffset), operands.Instruction.end()); + } + return bytes; +} + +void _emitImmediate(std::vector& bytes, const std::uint8_t opcode, const std::uint8_t extension, const std::uint8_t reg, const std::uint32_t immediate) { + _appendRex(bytes, static_cast(kRexBase | kRexW | ((reg & 8) != 0 ? kRexB : 0))); + bytes.insert(bytes.end(), {opcode, static_cast(0xC0 | (extension << 3) | (reg & 7))}); + _append32(bytes, immediate); +} + +void _emitStackAnd(std::vector& bytes, const std::uint32_t immediate) { + bytes.insert(bytes.end(), {0x48, 0x81, 0x64, 0x24, 0x08}); + _append32(bytes, immediate); +} + +void _emitStackPushPopFlags(std::vector& bytes) { + bytes.insert(bytes.end(), {0xFF, 0x74, 0x24, 0x08, 0x9D}); +} + +} + +std::optional DecodeAdx(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + std::size_t opcode = 0; + std::size_t operandPrefixes = 0; + std::size_t repPrefixes = 0; + std::size_t repnePrefixes = 0; + std::size_t lockPrefixes = 0; + std::uint8_t rex = 0; + while (opcode < length) { + const auto value = data[opcode]; + if (value == PrefixOperandSize) { + ++operandPrefixes; + rex = 0; + } else if (value == PrefixRep) { + ++repPrefixes; + rex = 0; + } else if (value == PrefixRepne) { + ++repnePrefixes; + rex = 0; + } else if (value == PrefixLock) { + ++lockPrefixes; + rex = 0; + } else if (value == PrefixAddressSize || value == PrefixSegCs || value == PrefixSegSs || value == PrefixSegDs || value == PrefixSegEs || value == PrefixSegFs || value == PrefixSegGs) { + rex = 0; + } else if (value >= RexMin && value <= RexMax) { + rex = value; + } else { + break; + } + ++opcode; + } + if (opcode + 2 >= length || data[opcode] != TwoByteOpcodeEscape || data[opcode + 1] != ThreeByteEscape38 || data[opcode + 2] != 0xF6) + return std::nullopt; + const bool adcx = operandPrefixes == 1 && repPrefixes == 0 && repnePrefixes == 0; + const bool adox = repPrefixes == 1 && operandPrefixes == 0 && repnePrefixes == 0; + if (!adcx && !adox) + return std::nullopt; + if (lockPrefixes != 0) + throw CodegenException("ADCX/ADOX does not accept the LOCK prefix"); + const auto modrmOffset = opcode + 3; + if (modrmOffset >= length) + throw CodegenException("ADCX/ADOX truncated before its ModRM byte"); + const auto modrm = data[modrmOffset]; + const auto destination = static_cast(((modrm >> ModRmRegShift) & ModRmRegMask) | ((rex & kRexR) != 0 ? 8 : 0)); + const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); + AdxOperands operands{adox, (rex & kRexW) != 0, destination, + static_cast((modrm & ModRmRmMask) | ((rex & kRexB) != 0 ? 8 : 0)), rex, opcode, modrmOffset, + {data, data + length}, std::nullopt}; + if (mod == ModRmModRegister) { + if (modrmOffset + 1 != length) + throw CodegenException("ADCX/ADOX register operand has trailing bytes"); + } else { + std::vector prefixes(data, data + opcode); + operands.Memory = DecodeMemoryOperand(data, length, modrmOffset, static_cast(rex & (kRexX | kRexB)), std::move(prefixes)); + const auto address32 = std::find(data, data + opcode, PrefixAddressSize) != data + opcode; + if (address32 && operands.Memory->RipRelative) + operands.Memory->RipRelative = false; + if (modrmOffset + operands.Memory->EncodedSize != length) + throw CodegenException("ADCX/ADOX memory operand has trailing bytes"); + } + return operands; +} + +void EmitAdx(StubBodyBuilder& body, const AdxOperands& operands) { + if (operands.Destination == kStackRegister) + throw CodegenException("ADCX/ADOX with RSP destination cannot be lowered through a trampoline"); + const auto scratch = _scratch(operands); + const auto flagMask = operands.Overflow ? std::uint32_t{0x800} : std::uint32_t{1}; + const auto clearedMask = ~flagMask; + std::vector prologue{0x48, 0x8D, 0xA4, 0x24, 0x70, 0xFF, 0xFF, 0xFF}; + _emitRegStack(prologue, 0x89, scratch, 0); + prologue.push_back(0x9C); + _emitRegMemory(prologue, 0x8B, scratch, kStackRegister, 0); + prologue.push_back(0x9D); + _emitRegStack(prologue, 0x89, scratch, 8); + body.Raw(prologue); + if (operands.Source == kStackRegister && !operands.Memory) { + std::vector originalRsp; + _appendRex(originalRsp, static_cast(kRexBase | kRexW | ((scratch & 8) != 0 ? kRexR : 0))); + originalRsp.push_back(0x8D); + originalRsp.push_back(static_cast(0x84 | ((scratch & 7) << 3))); + originalRsp.push_back(0x24); + _append32(originalRsp, kFrameSize); + body.Raw(originalRsp); + } + auto instruction = _translatedInstruction(operands, scratch); + if (operands.Overflow) + body.Raw(std::array{0x48, 0x0F, 0xBA, 0x64, 0x24, 0x08}); + if (operands.Overflow) + body.Raw(std::array{0x0B}); + if (operands.Memory && operands.Memory->RipRelative) { + const auto displacementOffset = instruction.size() - 4; + body.RipRelative(instruction, displacementOffset, static_cast(operands.Instruction.size()) + operands.Memory->Displacement); + } else { + body.Raw(instruction); + } + std::vector epilogue{0x9C}; + _emitRegMemory(epilogue, 0x8B, scratch, kStackRegister, 0); + epilogue.push_back(0x9D); + _emitStackAnd(epilogue, clearedMask); + _emitImmediate(epilogue, 0x81, 4, scratch, flagMask); + _emitRegMemory(epilogue, 0x09, scratch, kStackRegister, 8); + _emitStackPushPopFlags(epilogue); + _emitRegMemory(epilogue, 0x8B, scratch, kStackRegister, 0); + epilogue.insert(epilogue.end(), {0x48, 0x8D, 0xA4, 0x24, 0x90, 0x00, 0x00, 0x00}); + body.Raw(epilogue); +} + +} diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index 7b6175bdff3..5973abda44e 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -218,6 +219,8 @@ Amd64OnlyMatch Amd64OnlyInstructionMatcher::_matchClzero(const DecodedInstructio std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const DecodedInstruction& instr, std::span trailing) const { if (_target != Amd64OnlyTarget::Rosetta) return std::nullopt; + if (const auto adx = DecodeAdx(instr.Data, instr.Length)) + return _trampoline(adx->Overflow ? "ADOX" : "ADCX", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAdx(body, *adx); })); if (const auto andn = DecodeAndn(instr.Data, instr.Length)) return _trampoline("ANDN", instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { EmitAndn(body, *andn); })); if (instr.IsRdseed()) { @@ -268,6 +271,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c EmitAndn(body, *andn); return "ANDN"; } + if (const auto adx = _target == Amd64OnlyTarget::Rosetta ? DecodeAdx(instr.Data, instr.Length) : std::nullopt) { + EmitAdx(body, *adx); + return adx->Overflow ? "ADOX" : "ADCX"; + } if (const auto wide = _target == Amd64OnlyTarget::Rosetta ? DecodeVexWideStore(instr.Data, instr.Length) : std::nullopt) { _wideStoreLowering.EmitOutOfLine(body, *wide); return kWideStore.Name; diff --git a/core/relinker/codegen/tests/AdxLoweringTests.cpp b/core/relinker/codegen/tests/AdxLoweringTests.cpp new file mode 100644 index 00000000000..434e53024f1 --- /dev/null +++ b/core/relinker/codegen/tests/AdxLoweringTests.cpp @@ -0,0 +1,185 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) +#include +#endif + +namespace { + +using Bytes = std::vector; + +void require(const bool condition, const char* message) { + if (!condition) + throw std::runtime_error(message); +} + +void decoding() { + const Bytes adcx32{0x66, 0x0F, 0x38, 0xF6, 0xC1}; + const Bytes adcx64{0x66, 0x4C, 0x0F, 0x38, 0xF6, 0xC1}; + const Bytes adox64{0xF3, 0x4C, 0x0F, 0x38, 0xF6, 0xC2}; + const Bytes adcxStack{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x44, 0x24, 0x08}; + const Bytes adcxRip{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x05, 0x10, 0x00, 0x00, 0x00}; + const Bytes adcxAddress32{0x67, 0x66, 0x0F, 0x38, 0xF6, 0x05, 0x10, 0x00, 0x00, 0x00}; + const auto first = Codegen::DecodeAdx(adcx32.data(), adcx32.size()); + const auto second = Codegen::DecodeAdx(adcx64.data(), adcx64.size()); + const auto third = Codegen::DecodeAdx(adox64.data(), adox64.size()); + const auto fourth = Codegen::DecodeAdx(adcxStack.data(), adcxStack.size()); + const auto fifth = Codegen::DecodeAdx(adcxRip.data(), adcxRip.size()); + const auto sixth = Codegen::DecodeAdx(adcxAddress32.data(), adcxAddress32.size()); + require(first && !first->Overflow && !first->Wide && first->Destination == 0 && first->Source == 1, "ADCX 32-bit registers decoded incorrectly"); + require(second && !second->Overflow && second->Wide && second->Destination == 8 && second->Source == 1, "ADCX REX extensions decoded incorrectly"); + require(third && third->Overflow && third->Wide && third->Destination == 8 && third->Source == 2, "ADOX REX extensions decoded incorrectly"); + require(fourth && fourth->Memory && fourth->Memory->StackBase && fourth->Memory->Displacement == 8, "ADCX stack operand decoded incorrectly"); + require(fifth && fifth->Memory && fifth->Memory->RipRelative, "ADCX RIP-relative source was not identified"); + require(sixth && sixth->Memory && !sixth->Memory->RipRelative, "ADCX address-size source was mistaken for RIP-relative"); + for (const auto& bytes : {Bytes{0x0F, 0x38, 0xF6, 0xC1}, Bytes{0xF2, 0x0F, 0x38, 0xF6, 0xC1}, Bytes{0x66, 0xF3, 0x0F, 0x38, 0xF6, 0xC1}}) + require(!Codegen::DecodeAdx(bytes.data(), bytes.size()), "An instruction without the mandatory ADX prefix matched"); + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto intel = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Intel); + require(rosetta->Match(adcx32.data(), adcx32.size()).has_value(), "Rosetta mode did not lower ADCX"); + require(rosetta->Match(adox64.data(), adox64.size()).has_value(), "Rosetta mode did not lower ADOX"); + require(rosetta->Match(adcxRip.data(), adcxRip.size())->Relocations.size() == 1, "ADCX RIP-relative source has no relocation"); + require(!intel->Match(adcx32.data(), adcx32.size()), "Intel mode rewrote ADCX"); + bool rejected = false; + const Bytes rspDestination{0x66, 0x0F, 0x38, 0xF6, 0xE1}; + try { + (void)rosetta->Match(rspDestination.data(), rspDestination.size()); + } catch (const Codegen::CodegenException&) { + rejected = true; + } + require(rejected, "ADCX with an RSP destination was silently accepted"); +} + +void conversion() { + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); + const std::array adxOpcode{0x0F, 0x38, 0xF6}; + for (const auto& instruction : {Bytes{0x66, 0x0F, 0x38, 0xF6, 0xC1}, Bytes{0xF3, 0x0F, 0x38, 0xF6, 0xC1}}) { + Bytes file(0x300, 0xCC); + std::copy(instruction.begin(), instruction.end(), file.begin() + 0x200); + const auto trailing = instruction.size() == 4 ? std::uint8_t{0x90} : std::uint8_t{0xC3}; + file[0x200 + instruction.size()] = trailing; + file[0x200 + instruction.size() + 1] = 0xC3; + const Domain::ProgramHeader segment{1, 5, 0x200, 0x1000, 0, instruction.size() + (trailing == 0x90 ? 2 : 1), instruction.size() + (trailing == 0x90 ? 2 : 1), 16}; + const auto result = converter->Convert(file, {segment}); + require(result.Trampolines.size() == 1 && result.Reports.size() == 1, "The converter did not lower an ADX instruction"); + const auto& trampoline = result.Trampolines.front(); + require(result.Reports.front().InstructionName == (instruction.front() == 0x66 ? "ADCX" : "ADOX"), "The converter reported the wrong ADX instruction"); + require(std::search(trampoline.Body.begin(), trampoline.Body.begin() + static_cast(trampoline.ReturnBranchOffset), adxOpcode.begin(), adxOpcode.end()) == trampoline.Body.begin() + static_cast(trampoline.ReturnBranchOffset), "The converter left an unsupported ADX opcode in the trampoline"); + } +} + +#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) + +struct Run { + std::uint64_t Value; + std::uint64_t Flags; + std::uint64_t Stack; +}; + +Run execute(const Bytes& site, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags, const std::uint64_t memorySource = 0) { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto match = matcher->Match(site.data(), site.size()); + require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "An ADX trampoline was not produced"); + auto codeBytes = match->StubBody; + const auto ret = codeBytes.size(); + codeBytes.push_back(0xC3); + const auto displacement = static_cast(ret - (match->ReturnBranchOffset + 5)); + std::memcpy(codeBytes.data() + match->ReturnBranchOffset + 1, &displacement, sizeof(displacement)); + auto* code = static_cast(mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + require(code != MAP_FAILED, "Cannot allocate executable memory for an ADX stub"); + std::memcpy(code, codeBytes.data(), codeBytes.size()); + Run result{}; + asm volatile( + "mov %[destination], %%rax\n\t" + "mov %[source], %%rcx\n\t" + "mov %[source], %%r9\n\t" + "sub $128, %%rsp\n\t" + "mov %[memorySource], (%%rsp)\n\t" + "lea -8(%%rsp), %[stack]\n\t" + "mov %[initialFlags], %%r11\n\t" + "push %%r11\n\t" + "popfq\n\t" + "call *%[code]\n\t" + "pushfq\n\t" + "pop %[flags]\n\t" + "add $128, %%rsp\n\t" + "mov %%rax, %[value]\n\t" + : [value] "=&r"(result.Value), [flags] "=&r"(result.Flags), [stack] "=&r"(result.Stack) + : [destination] "r"(destination), [source] "r"(source), [initialFlags] "r"(initialFlags), [memorySource] "r"(memorySource), [code] "r"(code) + : "rax", "rcx", "r9", "r11", "memory", "cc"); + munmap(code, 4096); + return result; +} + +std::uint64_t expectedResult(const bool wide, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t carry) { + if (wide) + return destination + source + carry; + return static_cast(destination) + static_cast(source) + static_cast(carry); +} + +void execution() { + for (const bool wide : {false, true}) { + for (const bool adox : {false, true}) { + const auto initial = std::uint64_t{0xA03}; + Bytes site{static_cast(adox ? 0xF3 : 0x66)}; + if (wide) + site.push_back(0x48); + site.insert(site.end(), {0x0F, 0x38, 0xF6, 0xC1}); + const auto destination = wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull; + const auto source = wide ? 0ull : 0ull; + const auto inputFlag = adox ? ((initial >> 11) & 1) : (initial & 1); + const auto run = execute(site, destination, source, initial); + const auto expected = expectedResult(wide, destination, source, inputFlag); + require(run.Value == expected, "ADX register lowering produced the wrong sum"); + const auto sign = wide ? (std::uint64_t{1} << 63) : (std::uint64_t{1} << 31); + const auto mask = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; + const auto result = expected & mask; + const bool overflow = ((~(destination ^ source) & (destination ^ result) & sign) != 0); + const bool carry = wide + ? (result < destination || (inputFlag != 0 && result == destination)) + : ((static_cast(static_cast(destination)) + static_cast(source) + inputFlag) >> 32) != 0; + const auto flagMask = adox ? std::uint64_t{0x800} : std::uint64_t{1}; + const auto expectedFlags = (initial & ~flagMask) | ((adox ? overflow : carry) ? flagMask : 0); + require((run.Flags & 0x8D5) == (expectedFlags & 0x8D5), "ADX lowering changed an unrelated arithmetic flag"); + } + } + const Bytes stackSource{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x44, 0x24, 0x08}; + const auto stackRun = execute(stackSource, 5, 0, 0x202, 7); + require(stackRun.Value == 12 && (stackRun.Flags & 1) == 0, "ADX lowering did not compensate an RSP-relative memory source"); + const Bytes rspSource{0x66, 0x48, 0x0F, 0x38, 0xF6, 0xC4}; + const auto rspRun = execute(rspSource, 5, 0, 0xA03); + require(rspRun.Value == rspRun.Stack + 6, "ADX lowering did not preserve the original RSP source value and carry-in"); + require(((rspRun.Flags ^ 0xA03) & 0x8D4) == 0, "ADX lowering did not preserve unrelated flags with RSP as the source"); + const Bytes highSource{0x66, 0x49, 0x0F, 0x38, 0xF6, 0xC1}; + const auto highRun = execute(highSource, 3, 5, 0x202); + require(highRun.Value == 8, "ADX lowering did not preserve the high source-register extension"); +} + +#else + +void execution() {} + +#endif + +} + +int main() { + try { + decoding(); + conversion(); + execution(); + std::cout << "ADX lowering tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/docs/dev/debt/relinker.md b/docs/dev/debt/relinker.md index acab2b4e54e..487e11cff3a 100644 --- a/docs/dev/debt/relinker.md +++ b/docs/dev/debt/relinker.md @@ -22,3 +22,4 @@ - `--trace-imports` ([entry stub](../../../core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp)) exists for Windows output only. Only imports bound through a PLT slot, or through a GOT slot whose symbol is a function, are traced; a function reached through any other relocation, or looked up at run time with `sceKernelDlsym`, is not, and neither are weak imports or the imports of modules loaded at run time - [ANDN lowering](../../../core/relinker/codegen/src/x86/AndnLowering.cpp) uses scalar NOT and AND for 32- and 64-bit register and memory operands under `--to-rosetta`, following the ANDN entry in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf). SF and ZF follow the result, CF and OF clear, and AF and PF are undefined. The scratch register is saved below the 128-byte red zone. A 64-bit stack-relative displacement that exceeds signed disp32 after reserving the spill frame throws; other BMI instructions are not lowered. The M1 test covers register overlap, RSP operands, memory addressing, RIP relocation, flags and red-zone preservation; precise fault delivery inside a spill sequence has not been verified. +- The `--to-rosetta` ADCX/ADOX lowering follows their entries in [Intel SDM volume 2A](https://cdrdv2-public.intel.com/812383/253666-sdm-vol-2a.pdf), preserving the other arithmetic flags. An instruction with RSP as its destination register is rejected because changing the stack pointer inside the spill trampoline cannot be safely restored. From f5e857c489c2238e901577a0316ff041ca60a390 Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Sat, 17 Oct 2026 15:15:37 +0000 Subject: [PATCH 2/4] fix(relinker): preserve ADOX carry-out flag --- core/relinker/codegen/src/x86/AdxLowering.cpp | 6 +- .../codegen/tests/AdxLoweringTests.cpp | 89 ++++++++++++++----- 2 files changed, 71 insertions(+), 24 deletions(-) diff --git a/core/relinker/codegen/src/x86/AdxLowering.cpp b/core/relinker/codegen/src/x86/AdxLowering.cpp index 32cfe85eafa..23c6777c570 100644 --- a/core/relinker/codegen/src/x86/AdxLowering.cpp +++ b/core/relinker/codegen/src/x86/AdxLowering.cpp @@ -211,7 +211,11 @@ void EmitAdx(StubBodyBuilder& body, const AdxOperands& operands) { _emitRegMemory(epilogue, 0x8B, scratch, kStackRegister, 0); epilogue.push_back(0x9D); _emitStackAnd(epilogue, clearedMask); - _emitImmediate(epilogue, 0x81, 4, scratch, flagMask); + _emitImmediate(epilogue, 0x81, 4, scratch, 1); + if (operands.Overflow) { + _appendRex(epilogue, static_cast(kRexBase | kRexW | ((scratch & 8) != 0 ? kRexB : 0))); + epilogue.insert(epilogue.end(), {0xC1, static_cast(0xC0 | (scratch & 7)), 11}); + } _emitRegMemory(epilogue, 0x09, scratch, kStackRegister, 8); _emitStackPushPopFlags(epilogue); _emitRegMemory(epilogue, 0x8B, scratch, kStackRegister, 0); diff --git a/core/relinker/codegen/tests/AdxLoweringTests.cpp b/core/relinker/codegen/tests/AdxLoweringTests.cpp index 434e53024f1..9d7be38c05f 100644 --- a/core/relinker/codegen/tests/AdxLoweringTests.cpp +++ b/core/relinker/codegen/tests/AdxLoweringTests.cpp @@ -7,9 +7,12 @@ #include #include #include +#include #include +#include #include #if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) +#include #include #endif @@ -85,15 +88,13 @@ struct Run { std::uint64_t Stack; }; -Run execute(const Bytes& site, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags, const std::uint64_t memorySource = 0) { - const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); - const auto match = matcher->Match(site.data(), site.size()); - require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "An ADX trampoline was not produced"); - auto codeBytes = match->StubBody; +Run executeCode(Bytes codeBytes, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags, const std::uint64_t memorySource = 0, const std::optional returnBranchOffset = std::nullopt) { const auto ret = codeBytes.size(); + if (returnBranchOffset) { + const auto displacement = static_cast(ret - (*returnBranchOffset + 5)); + std::memcpy(codeBytes.data() + *returnBranchOffset + 1, &displacement, sizeof(displacement)); + } codeBytes.push_back(0xC3); - const auto displacement = static_cast(ret - (match->ReturnBranchOffset + 5)); - std::memcpy(codeBytes.data() + match->ReturnBranchOffset + 1, &displacement, sizeof(displacement)); auto* code = static_cast(mmap(nullptr, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); require(code != MAP_FAILED, "Cannot allocate executable memory for an ADX stub"); std::memcpy(code, codeBytes.data(), codeBytes.size()); @@ -120,6 +121,21 @@ Run execute(const Bytes& site, const std::uint64_t destination, const std::uint6 return result; } +Run execute(const Bytes& site, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags, const std::uint64_t memorySource = 0) { + const auto matcher = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto match = matcher->Match(site.data(), site.size()); + require(match && match->Lowering == Codegen::Amd64OnlyLowering::Trampoline, "An ADX trampoline was not produced"); + return executeCode(match->StubBody, destination, source, initialFlags, memorySource, match->ReturnBranchOffset); +} + +Run executeNative(const bool wide, const bool adox, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t initialFlags) { + Bytes instruction{static_cast(adox ? 0xF3 : 0x66)}; + if (wide) + instruction.push_back(0x48); + instruction.insert(instruction.end(), {0x0F, 0x38, 0xF6, 0xC1, 0xC3}); + return executeCode(std::move(instruction), destination, source, initialFlags); +} + std::uint64_t expectedResult(const bool wide, const std::uint64_t destination, const std::uint64_t source, const std::uint64_t carry) { if (wide) return destination + source + carry; @@ -129,27 +145,54 @@ std::uint64_t expectedResult(const bool wide, const std::uint64_t destination, c void execution() { for (const bool wide : {false, true}) { for (const bool adox : {false, true}) { - const auto initial = std::uint64_t{0xA03}; Bytes site{static_cast(adox ? 0xF3 : 0x66)}; if (wide) site.push_back(0x48); site.insert(site.end(), {0x0F, 0x38, 0xF6, 0xC1}); - const auto destination = wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull; - const auto source = wide ? 0ull : 0ull; - const auto inputFlag = adox ? ((initial >> 11) & 1) : (initial & 1); - const auto run = execute(site, destination, source, initial); - const auto expected = expectedResult(wide, destination, source, inputFlag); - require(run.Value == expected, "ADX register lowering produced the wrong sum"); - const auto sign = wide ? (std::uint64_t{1} << 63) : (std::uint64_t{1} << 31); const auto mask = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; - const auto result = expected & mask; - const bool overflow = ((~(destination ^ source) & (destination ^ result) & sign) != 0); - const bool carry = wide - ? (result < destination || (inputFlag != 0 && result == destination)) - : ((static_cast(static_cast(destination)) + static_cast(source) + inputFlag) >> 32) != 0; - const auto flagMask = adox ? std::uint64_t{0x800} : std::uint64_t{1}; - const auto expectedFlags = (initial & ~flagMask) | ((adox ? overflow : carry) ? flagMask : 0); - require((run.Flags & 0x8D5) == (expectedFlags & 0x8D5), "ADX lowering changed an unrelated arithmetic flag"); + const auto high = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; + const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high}; + for (const auto destination : values) { + for (const auto source : values) { + for (const auto inputFlag : {std::uint64_t{0}, std::uint64_t{1}}) { + const auto flagMask = adox ? std::uint64_t{0x800} : std::uint64_t{1}; + const auto initial = (std::uint64_t{0x8D7} & ~flagMask) | (inputFlag ? flagMask : 0); + const auto run = execute(site, destination, source, initial); + const auto expected = expectedResult(wide, destination, source, inputFlag) & mask; + require(run.Value == expected, "ADX register lowering produced the wrong sum"); + const bool carry = wide + ? (expected < destination || (inputFlag != 0 && expected == destination)) + : ((static_cast(static_cast(destination)) + static_cast(source) + inputFlag) >> 32) != 0; + const auto expectedFlags = (initial & ~flagMask) | (carry ? flagMask : 0); + require((run.Flags & 0x8D5) == (expectedFlags & 0x8D5), "ADX lowering produced the wrong carry flag or changed an unrelated arithmetic flag"); + } + } + } + } + } + unsigned int eax = 0, ebx = 0, ecx = 0, edx = 0; + const bool hasAdx = __get_cpuid_count(7, 0, &eax, &ebx, &ecx, &edx) != 0 && (ebx & (1u << 19)) != 0; + if (hasAdx) { + for (const bool wide : {false, true}) { + for (const bool adox : {false, true}) { + const auto high = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; + const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high}; + for (const auto destination : values) { + for (const auto source : values) { + for (const auto inputFlag : {std::uint64_t{0}, std::uint64_t{1}}) { + const auto flagMask = adox ? std::uint64_t{0x800} : std::uint64_t{1}; + const auto initial = (std::uint64_t{0x8D7} & ~flagMask) | (inputFlag ? flagMask : 0); + Bytes site{static_cast(adox ? 0xF3 : 0x66)}; + if (wide) + site.push_back(0x48); + site.insert(site.end(), {0x0F, 0x38, 0xF6, 0xC1}); + const auto lowered = execute(site, destination, source, initial); + const auto native = executeNative(wide, adox, destination, source, initial); + require(lowered.Value == native.Value && (lowered.Flags & 0x8D5) == (native.Flags & 0x8D5), "ADX lowering differs from native ADX"); + } + } + } + } } } const Bytes stackSource{0x66, 0x48, 0x0F, 0x38, 0xF6, 0x44, 0x24, 0x08}; From 8e83a61a2721b03783eb1c7043c2057a33582cd9 Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Sat, 17 Oct 2026 15:18:41 +0000 Subject: [PATCH 3/4] fix(relinker): encode ADOX carry shift --- core/relinker/codegen/src/x86/AdxLowering.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/relinker/codegen/src/x86/AdxLowering.cpp b/core/relinker/codegen/src/x86/AdxLowering.cpp index 23c6777c570..ef32ddaa7ab 100644 --- a/core/relinker/codegen/src/x86/AdxLowering.cpp +++ b/core/relinker/codegen/src/x86/AdxLowering.cpp @@ -214,7 +214,7 @@ void EmitAdx(StubBodyBuilder& body, const AdxOperands& operands) { _emitImmediate(epilogue, 0x81, 4, scratch, 1); if (operands.Overflow) { _appendRex(epilogue, static_cast(kRexBase | kRexW | ((scratch & 8) != 0 ? kRexB : 0))); - epilogue.insert(epilogue.end(), {0xC1, static_cast(0xC0 | (scratch & 7)), 11}); + epilogue.insert(epilogue.end(), {0xC1, static_cast(0xE0 | (scratch & 7)), 11}); } _emitRegMemory(epilogue, 0x09, scratch, kStackRegister, 8); _emitStackPushPopFlags(epilogue); From 010ce9b02f0403a2fdf44f0d0e86ece3651d4ccc Mon Sep 17 00:00:00 2001 From: MarcZX100 Date: Sun, 18 Oct 2026 12:24:34 +0000 Subject: [PATCH 4/4] test(relinker): cover upper bits in 32-bit ADX results --- core/relinker/codegen/tests/AdxLoweringTests.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/relinker/codegen/tests/AdxLoweringTests.cpp b/core/relinker/codegen/tests/AdxLoweringTests.cpp index 9d7be38c05f..b1018581602 100644 --- a/core/relinker/codegen/tests/AdxLoweringTests.cpp +++ b/core/relinker/codegen/tests/AdxLoweringTests.cpp @@ -151,7 +151,8 @@ void execution() { site.insert(site.end(), {0x0F, 0x38, 0xF6, 0xC1}); const auto mask = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; const auto high = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; - const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high}; + const auto upperBits = wide ? std::uint64_t{2} : std::uint64_t{0xDEADBEEF00000001}; + const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high, upperBits}; for (const auto destination : values) { for (const auto source : values) { for (const auto inputFlag : {std::uint64_t{0}, std::uint64_t{1}}) { @@ -176,7 +177,8 @@ void execution() { for (const bool wide : {false, true}) { for (const bool adox : {false, true}) { const auto high = wide ? std::numeric_limits::max() : std::uint64_t{0xFFFFFFFF}; - const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high}; + const auto upperBits = wide ? std::uint64_t{2} : std::uint64_t{0xDEADBEEF00000001}; + const std::array values{0, 1, wide ? 0x7FFFFFFFFFFFFFFFull : 0x7FFFFFFFull, high, upperBits}; for (const auto destination : values) { for (const auto source : values) { for (const auto inputFlag : {std::uint64_t{0}, std::uint64_t{1}}) {