From 8b400afc5d5635e5af803c10b5725d7f4405ede7 Mon Sep 17 00:00:00 2001 From: Leandro Soares Date: Wed, 14 Oct 2026 00:07:13 -0300 Subject: [PATCH 1/2] fix(relinker): move hazard scalar compares out of line for Rosetta --- core/relinker/CMakeLists.txt | 3 + .../x86/Amd64OnlySubstitutionTable.hpp | 1 + .../codegen/x86/ScalarCompareOperands.hpp | 19 ++++++ .../src/x86/Amd64OnlyInstructionMatcher.cpp | 15 +++++ .../codegen/src/x86/ScalarCompareOperands.cpp | 63 +++++++++++++++++++ .../codegen/tests/Amd64OnlyConverterTests.cpp | 57 +++++++++++++++++ .../relinker/src/pipeline/Conversion.cpp | 2 +- 7 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp create mode 100644 core/relinker/codegen/src/x86/ScalarCompareOperands.cpp diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index 2a01d4bee20..5eddd14cc28 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -50,6 +50,7 @@ add_library(relinker_core STATIC ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/parsing/ElfReader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/SyscallScanner.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/CallSiteResolver.cpp @@ -167,6 +168,7 @@ if(WIN32) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ) target_include_directories(amd64_only_windows_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include ${CMAKE_CURRENT_SOURCE_DIR}/domain/include ${CMAKE_CURRENT_SOURCE_DIR}/io/include ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include ${CMAKE_CURRENT_SOURCE_DIR}/../libs/prx/libc/src/specifics/windows) if(BUILD_TESTING) @@ -345,6 +347,7 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ReciprocalLowering.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/WideStoreLowering.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/ScalarCompareOperands.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/linux/LinuxElfPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp diff --git a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp index 7f09e114ad1..41948aec851 100644 --- a/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp +++ b/core/relinker/codegen/include/codegen/x86/Amd64OnlySubstitutionTable.hpp @@ -42,6 +42,7 @@ inline constexpr Entry kRdseed = {"RDSEED", nullptr, 0}; inline constexpr Entry kRdpid = {"RDPID", nullptr, 0}; inline constexpr Entry kClwb = {"CLWB", nullptr, 0}; inline constexpr Entry kWideStore = {"256-bit store", nullptr, 0}; +inline constexpr Entry kScalarCompare = {"scalar compare", nullptr, 0}; inline constexpr Entry kExtrq = {"EXTRQ", nullptr, 0}; inline constexpr Entry kInsertq = {"INSERTQ", nullptr, 0}; inline constexpr Entry kExtrqRegisterForm = {"EXTRQ register form", nullptr, 0}; diff --git a/core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp b/core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp new file mode 100644 index 00000000000..458f799ce46 --- /dev/null +++ b/core/relinker/codegen/include/codegen/x86/ScalarCompareOperands.hpp @@ -0,0 +1,19 @@ +#ifndef CODEGEN_X86_SCALARCOMPAREOPERANDS_HPP +#define CODEGEN_X86_SCALARCOMPAREOPERANDS_HPP + +#include +#include +#include + +namespace Codegen { + +struct ScalarCompareOperands { + std::optional RipDisplacementOffset; + std::int64_t RipTarget = 0; +}; + +[[nodiscard]] std::optional DecodeScalarCompare(const std::uint8_t* data, std::size_t length); + +} + +#endif diff --git a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp index edf59c514c6..112fed6de79 100644 --- a/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp +++ b/core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -106,6 +107,14 @@ Amd64OnlyMatch _inPlace(const Entry& entry, const std::size_t length, std::vecto return Amd64OnlyMatch{entry.Name, length, Amd64OnlyLowering::InPlace, std::move(replacement), {}, 0}; } +void _emitScalarCompare(StubBodyBuilder& body, const DecodedInstruction& instr, const ScalarCompareOperands& operands) { + const std::span bytes(instr.Data, instr.Length); + if (operands.RipDisplacementOffset) + body.RipRelative(bytes, *operands.RipDisplacementOffset, operands.RipTarget); + else + body.Raw(bytes); +} + bool _isClzeroOpcode(const DecodedInstruction& instr) { const auto pos = instr.OpcodeOffset(); return pos + 2 < instr.Length && instr.Data[pos] == X64OpcodeConstants::TwoByteOpcodeEscape && instr.Data[pos + 1] == X64OpcodeConstants::TwoByteGrp7 && instr.Data[pos + 2] == 0xFC; @@ -219,6 +228,8 @@ std::optional Amd64OnlyInstructionMatcher::_matchRosetta(const D return _trampoline(kRdpid.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { body.Raw(_rdpidReplacement(instr)); }), true); if (const auto wide = DecodeVexWideStore(instr.Data, instr.Length)) return _trampoline(kWideStore.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _wideStoreLowering.EmitOutOfLine(body, *wide); }), false, true); + if (const auto compare = DecodeScalarCompare(instr.Data, instr.Length)) + return _trampoline(kScalarCompare.Name, instr.Length, _outOfLine(instr.Length, trailing, [&](StubBodyBuilder& body) { _emitScalarCompare(body, instr, *compare); }), true); return std::nullopt; } @@ -254,6 +265,10 @@ const char* Amd64OnlyInstructionMatcher::_emitOutOfLine(StubBodyBuilder& body, c _wideStoreLowering.EmitOutOfLine(body, *wide); return kWideStore.Name; } + if (const auto compare = _target == Amd64OnlyTarget::Rosetta ? DecodeScalarCompare(instr.Data, instr.Length) : std::nullopt) { + _emitScalarCompare(body, instr, *compare); + return kScalarCompare.Name; + } return nullptr; } diff --git a/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp b/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp new file mode 100644 index 00000000000..c0c31a92e6a --- /dev/null +++ b/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp @@ -0,0 +1,63 @@ +#include +#include + +namespace Codegen { + +namespace { + +constexpr std::uint8_t kVexMap0F = 0x01; +constexpr std::uint8_t kPpF3 = 0x02; +constexpr std::uint8_t kCompareOpcode = 0xC2; +constexpr std::uint8_t kPredicateLeOs = 0x02; + +bool _segmentPrefix(const std::uint8_t prefix) { + using namespace X64OpcodeConstants; + return prefix == PrefixSegCs || prefix == PrefixSegSs || prefix == PrefixSegDs || prefix == PrefixSegEs || prefix == PrefixSegFs || prefix == PrefixSegGs; +} + +} + +std::optional DecodeScalarCompare(const std::uint8_t* data, const std::size_t length) { + using namespace X64OpcodeConstants; + std::size_t pos = 0; + while (pos < length && _segmentPrefix(data[pos])) + ++pos; + std::size_t modRmOffset = 0; + if (pos < length && data[pos] == OneByteVex2) { + if (pos + 4 > length || (data[pos + 1] & 0x03) < kPpF3 || data[pos + 2] != kCompareOpcode) + return std::nullopt; + modRmOffset = pos + 3; + } else if (pos < length && data[pos] == OneByteVex3) { + if (pos + 5 > length || (data[pos + 1] & Vex3MapMask) != kVexMap0F || (data[pos + 2] & 0x03) < kPpF3 || data[pos + 3] != kCompareOpcode) + return std::nullopt; + modRmOffset = pos + 4; + } else { + return std::nullopt; + } + const auto modrm = data[modRmOffset]; + const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); + const auto rm = static_cast(modrm & ModRmRmMask); + auto cursor = modRmOffset + 1; + std::optional sib; + if (mod != ModRmModRegister && rm == ModRmRmSibPresent) { + if (cursor >= length) + return std::nullopt; + sib = data[cursor++]; + } + const bool ripRelative = mod == ModRmModIndirect && rm == ModRmRmRipRelative; + const bool absolute = mod == ModRmModIndirect && sib.has_value() && (*sib & SibBaseMask) == SibBaseDisp32; + const std::size_t displacementSize = mod == ModRmModDisp8 ? Disp8Size : (mod == ModRmModDisp32 || ripRelative || absolute) ? Disp32Size : 0; + if (cursor + displacementSize + ImmSize8 != length || data[length - 1] <= kPredicateLeOs) + return std::nullopt; + ScalarCompareOperands operands; + if (ripRelative) { + std::uint32_t displacement = 0; + for (std::size_t index = 0; index < Disp32Size; ++index) + displacement |= static_cast(data[cursor + index]) << (index * 8); + operands.RipDisplacementOffset = cursor; + operands.RipTarget = static_cast(length) + static_cast(displacement); + } + return operands; +} + +} diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index 8c4e4e00907..dcb82e697da 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -952,6 +952,61 @@ void converterRosetta() { require(intel.Reports.empty() && intel.Bytes == file, "The Intel target converted RDSEED, RDPID or CLWB"); } +void rosettaScalarCompares() { + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto match = [&](const Bytes& bytes) { return rosetta->Match(bytes.data(), bytes.size()); }; + const Bytes compare = {0xC5, 0xEA, 0xC2, 0xCA, 0x03}; + const auto lowered = match(compare); + require(lowered && lowered->Lowering == Codegen::Amd64OnlyLowering::Trampoline && lowered->Optional && !lowered->RelocatableWindow && lowered->InstructionName == "scalar compare", "A Rosetta-hazard scalar compare was not lowered through an optional stub"); + require(lowered->ReturnBranchOffset == 5 && lowered->StubBody.size() == 10 && Bytes(lowered->StubBody.begin(), lowered->StubBody.begin() + 5) == compare && lowered->StubBody[5] == 0xE9 && lowered->Relocations.empty(), "The scalar compare stub does not run the compare unchanged before its return jump"); + const auto unaligned = match(Bytes{0xC5, 0xEA, 0xC2, 0x47, 0x68, 0x03}); + require(unaligned && Bytes(unaligned->StubBody.begin(), unaligned->StubBody.begin() + 6) == Bytes{0xC5, 0xEA, 0xC2, 0x47, 0x68, 0x03} && unaligned->Relocations.empty(), "A memory-form scalar compare does not move unchanged"); + const Bytes rip = {0xC5, 0xEA, 0xC2, 0x0D, 0x00, 0x01, 0x00, 0x00, 0x03}; + const auto ripLowered = match(rip); + require(ripLowered && ripLowered->Relocations.size() == 1 && ripLowered->Relocations[0].DisplacementOffset == 4 && ripLowered->Relocations[0].InstructionEnd == 9 && ripLowered->Relocations[0].SiteTarget == 9 + 0x100, "A RIP-relative scalar compare was not relocated from its place in the site"); + require(Bytes(ripLowered->StubBody.begin() + 4, ripLowered->StubBody.begin() + 8) == Bytes(4, 0), "The RIP-relative scalar compare keeps its displacement in the stub"); + const auto vex3 = match(Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x03}); + require(vex3 && Bytes(vex3->StubBody.begin(), vex3->StubBody.begin() + 6) == Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x03}, "A VEX3 VCMPSD with a hazard predicate does not move unchanged"); + for (const Bytes& kept : {Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x00}, Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x01}, Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x02}, Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x01}}) + require(!match(kept).has_value(), "An EQ_OQ, LT_OS or LE_OS scalar compare must stay inline"); + for (const Bytes& other : {Bytes{0xC5, 0xE9, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xE8, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xEA, 0x2E, 0xC8}, Bytes{0xC5, 0xEA, 0xC2, 0xCA}, Bytes{0xC4, 0xE2, 0xEB, 0xC2, 0xCB, 0x03}, Bytes{0x62, 0xD1, 0x76, 0x08, 0xC2, 0xCB, 0x03}, Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x03}}) + require(!match(other).has_value(), "An instruction other than a Rosetta-hazard scalar compare was moved"); + require(!Codegen::MakeAmd64OnlyInstructionMatcher()->Match(compare.data(), compare.size()).has_value(), "A scalar compare was moved for an Intel host"); + const std::vector> pair = {std::span(compare), std::span(rip)}; + const auto sequence = rosetta->MatchSequence(pair, {}); + require(sequence && sequence->InstructionName == "scalar compare" && sequence->ReturnBranchOffset == 14 && sequence->Relocations.size() == 1 && sequence->Relocations[0].SiteTarget == 5 + 9 + 0x100, "A scalar compare after another is not emitted out of line in one stub"); +} + +void converterRosettaScalarCompare() { + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); + Bytes file(0x300, 0xCC); + const Bytes text = { + 0xC5, 0xFA, 0x10, 0x15, 0x10, 0x00, 0x00, 0x00, + 0xC5, 0xF8, 0x28, 0xC2, + 0xC5, 0xEA, 0xC2, 0xCA, 0x03, + 0xC5, 0xF8, 0x28, 0x0D, 0x20, 0x00, 0x00, 0x00, + 0xC5, 0xF8, 0x2E, 0xC8, + 0xC3}; + std::copy(text.begin(), text.end(), file.begin() + 0x200); + const auto result = converter->Convert(file, {segmentHeader(text.size())}); + require(result.ReplacedCount == 0 && result.KeptCount == 0 && result.Trampolines.size() == 1 && result.Reports.size() == 1 && result.Reports[0].InstructionName == "scalar compare" && result.Reports[0].Offset == 0x20C && result.Reports[0].Lowering == Codegen::Amd64OnlyLowering::Trampoline, "The Rosetta-hazard compare in the fmax shape was not the only moved instruction"); + const auto& site = result.Trampolines[0]; + require(site.Offset == 0x20C && site.Length == 5 && site.ReturnBranchOffset == 5 && Bytes(site.Body.begin(), site.Body.begin() + 5) == Bytes(text.begin() + 12, text.begin() + 17) && Bytes(site.Body.begin() + 5, site.Body.begin() + 10) == Bytes{0xE9, 0, 0, 0, 0} && site.Relocations.empty(), "The compare stub does not run the compare unchanged and return"); + require(result.Bytes == file, "Moving the compare changed the file image"); + auto branchInside = file; + const Bytes jumped = {0xEB, 0x02, 0xC5, 0xEA, 0xC2, 0xCA, 0x03, 0xC3}; + std::copy(jumped.begin(), jumped.end(), branchInside.begin() + 0x200); + const auto kept = converter->Convert(branchInside, {segmentHeader(jumped.size())}); + require(kept.Trampolines.empty() && kept.KeptCount == 1 && kept.Reports.size() == 1 && kept.Reports[0].Lowering == Codegen::Amd64OnlyLowering::Kept && kept.Bytes == branchInside, "A branch into the compare did not keep it native"); + auto ripSource = file; + const Bytes ripText = {0xC5, 0xEA, 0xC2, 0x0D, 0x00, 0x01, 0x00, 0x00, 0x03, 0xC3}; + std::copy(ripText.begin(), ripText.end(), ripSource.begin() + 0x200); + const auto ripMoved = converter->Convert(ripSource, {segmentHeader(ripText.size())}); + require(ripMoved.Trampolines.size() == 1 && ripMoved.Trampolines[0].Relocations.size() == 1 && ripMoved.Trampolines[0].Relocations[0].SiteTarget == 9 + 0x100, "A RIP-relative compare source was not relocated"); + const auto intel = Codegen::MakeAmd64OnlyConverter()->Convert(file, {segmentHeader(text.size())}); + require(intel.Reports.empty() && intel.Bytes == file, "The Intel target moved a scalar compare"); +} + void reciprocalOperands() { const auto vex2 = Codegen::DecodeReciprocal(Bytes{0xC5, 0xF8, 0x52, 0xD5}.data(), 4); require(vex2 && vex2->Operation == Codegen::ReciprocalOperation::ReciprocalSquareRoot && vex2->Destination == 2 && vex2->Source == 5, "VEX2 VRSQRTPS was not decoded"); @@ -1672,6 +1727,8 @@ int main() { converterClzero(); rosettaSubstitutions(); converterRosetta(); + rosettaScalarCompares(); + converterRosettaScalarCompare(); wideStoreSplit(); converterWideStoreWindows(); converterJumpTableTargets(); diff --git a/core/relinker/relinker/src/pipeline/Conversion.cpp b/core/relinker/relinker/src/pipeline/Conversion.cpp index 798d086efb6..28923e1e2e8 100644 --- a/core/relinker/relinker/src/pipeline/Conversion.cpp +++ b/core/relinker/relinker/src/pipeline/Conversion.cpp @@ -55,7 +55,7 @@ ConversionOutput Conversion::Convert(ConversionInput input, const ConversionOpti sourceBytes = std::move(converted.Bytes); trampolines = std::move(converted.Trampolines); std::map stubsByName; - const auto counted = [](const std::string& name) { return name == "VRSQRTPS" || name == "VRCPPS" || name == "256-bit store"; }; + const auto counted = [](const std::string& name) { return name == "VRSQRTPS" || name == "VRCPPS" || name == "256-bit store" || name == "scalar compare"; }; for (const auto& report : converted.Reports) { if (report.Lowering == Codegen::Amd64OnlyLowering::Kept) { std::ostringstream message; message << "Intel substitution: " << report.InstructionName << " at 0x" << std::hex << report.Offset << std::dec << " (" << report.OriginalLength << " bytes) kept: no room for a jump\n"; inputProgress.Message(message.str()); } From ba4b2db2386739ba69cb6539074bc75de59bf066 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexej=20Szpev=C3=A1k?= Date: Wed, 14 Oct 2026 11:48:11 +0200 Subject: [PATCH 2/2] fix(relinker): move legacy CMPSS and CMPSD out of line for Rosetta The Rosetta split covered only VEX scalar compares. #5642 measured the same lost load after legacy CMPSS and CMPSD, and the decoder already hands them to the matcher with the right length and RIP offset. Recognise F3 or F2, segment overrides and an optional REX before 0F C2 /r ib, with the same predicate rule as the VEX forms. A 66, 67 or LOCK prefix, or both F2 and F3, keeps the instruction inline as before. --- .../codegen/src/x86/ScalarCompareOperands.cpp | 18 +++++++- .../codegen/tests/Amd64OnlyConverterTests.cpp | 46 ++++++++++++++++++- 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp b/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp index c0c31a92e6a..0f1e217d5c2 100644 --- a/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp +++ b/core/relinker/codegen/src/x86/ScalarCompareOperands.cpp @@ -32,7 +32,23 @@ std::optional DecodeScalarCompare(const std::uint8_t* dat return std::nullopt; modRmOffset = pos + 4; } else { - return std::nullopt; + bool singlePrecision = false; + bool doublePrecision = false; + for (; pos < length; ++pos) { + if (data[pos] == PrefixRep) + singlePrecision = true; + else if (data[pos] == PrefixRepne) + doublePrecision = true; + else if (!_segmentPrefix(data[pos])) + break; + } + if (singlePrecision == doublePrecision) + return std::nullopt; + if (pos < length && data[pos] >= RexMin && data[pos] <= RexMax) + ++pos; + if (pos + 3 > length || data[pos] != TwoByteOpcodeEscape || data[pos + 1] != kCompareOpcode) + return std::nullopt; + modRmOffset = pos + 2; } const auto modrm = data[modRmOffset]; const auto mod = static_cast((modrm >> ModRmModShift) & ModRmModMask); diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index dcb82e697da..c97cea6a0b6 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -969,7 +969,7 @@ void rosettaScalarCompares() { require(vex3 && Bytes(vex3->StubBody.begin(), vex3->StubBody.begin() + 6) == Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x03}, "A VEX3 VCMPSD with a hazard predicate does not move unchanged"); for (const Bytes& kept : {Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x00}, Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x01}, Bytes{0xC5, 0xEA, 0xC2, 0xCA, 0x02}, Bytes{0xC4, 0xE1, 0xEB, 0xC2, 0xCB, 0x01}}) require(!match(kept).has_value(), "An EQ_OQ, LT_OS or LE_OS scalar compare must stay inline"); - for (const Bytes& other : {Bytes{0xC5, 0xE9, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xE8, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xEA, 0x2E, 0xC8}, Bytes{0xC5, 0xEA, 0xC2, 0xCA}, Bytes{0xC4, 0xE2, 0xEB, 0xC2, 0xCB, 0x03}, Bytes{0x62, 0xD1, 0x76, 0x08, 0xC2, 0xCB, 0x03}, Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x03}}) + for (const Bytes& other : {Bytes{0xC5, 0xE9, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xE8, 0xC2, 0xCA, 0x03}, Bytes{0xC5, 0xEA, 0x2E, 0xC8}, Bytes{0xC5, 0xEA, 0xC2, 0xCA}, Bytes{0xC4, 0xE2, 0xEB, 0xC2, 0xCB, 0x03}, Bytes{0x62, 0xD1, 0x76, 0x08, 0xC2, 0xCB, 0x03}}) require(!match(other).has_value(), "An instruction other than a Rosetta-hazard scalar compare was moved"); require(!Codegen::MakeAmd64OnlyInstructionMatcher()->Match(compare.data(), compare.size()).has_value(), "A scalar compare was moved for an Intel host"); const std::vector> pair = {std::span(compare), std::span(rip)}; @@ -977,6 +977,48 @@ void rosettaScalarCompares() { require(sequence && sequence->InstructionName == "scalar compare" && sequence->ReturnBranchOffset == 14 && sequence->Relocations.size() == 1 && sequence->Relocations[0].SiteTarget == 5 + 9 + 0x100, "A scalar compare after another is not emitted out of line in one stub"); } +void rosettaLegacyScalarCompares() { + const auto rosetta = Codegen::MakeAmd64OnlyInstructionMatcher(Codegen::Amd64OnlyTarget::Rosetta); + const auto match = [&](const Bytes& bytes) { return rosetta->Match(bytes.data(), bytes.size()); }; + for (const Bytes& moved : {Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x03}, Bytes{0xF2, 0x0F, 0xC2, 0xCA, 0x04}, Bytes{0xF2, 0x41, 0x0F, 0xC2, 0xC9, 0x05}, + Bytes{0x64, 0xF3, 0x0F, 0xC2, 0x4F, 0x68, 0x06}, Bytes{0xF3, 0x2E, 0x0F, 0xC2, 0x44, 0x24, 0x08, 0x07}, Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x08}}) { + const auto lowered = match(moved); + require(lowered && lowered->Lowering == Codegen::Amd64OnlyLowering::Trampoline && lowered->Optional && lowered->InstructionName == "scalar compare", "A legacy Rosetta-hazard scalar compare was not lowered through an optional stub"); + require(lowered->ReturnBranchOffset == moved.size() && lowered->StubBody.size() == moved.size() + 5 && Bytes(lowered->StubBody.begin(), lowered->StubBody.begin() + static_cast(moved.size())) == moved && lowered->StubBody[moved.size()] == 0xE9 && lowered->Relocations.empty(), "A legacy scalar compare does not run unchanged before its return jump"); + } + const auto rip = match(Bytes{0xF3, 0x0F, 0xC2, 0x0D, 0x00, 0x01, 0x00, 0x00, 0x06}); + require(rip && rip->Relocations.size() == 1 && rip->Relocations[0].DisplacementOffset == 4 && rip->Relocations[0].InstructionEnd == 9 && rip->Relocations[0].SiteTarget == 9 + 0x100, "A RIP-relative legacy CMPSS was not relocated from its place in the site"); + const auto ripRex = match(Bytes{0xF2, 0x48, 0x0F, 0xC2, 0x05, 0x00, 0x01, 0x00, 0x00, 0x07}); + require(ripRex && ripRex->Relocations.size() == 1 && ripRex->Relocations[0].DisplacementOffset == 5 && ripRex->Relocations[0].InstructionEnd == 10 && ripRex->Relocations[0].SiteTarget == 10 + 0x100, "A RIP-relative legacy CMPSD with REX was not relocated"); + for (const Bytes& kept : {Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x00}, Bytes{0xF3, 0x0F, 0xC2, 0xCA, 0x01}, Bytes{0xF2, 0x0F, 0xC2, 0xCA, 0x02}}) + require(!match(kept).has_value(), "A legacy EQ_OQ, LT_OS or LE_OS scalar compare must stay inline"); + for (const Bytes& other : {Bytes{0x0F, 0xC2, 0xCA, 0x03}, Bytes{0x66, 0x0F, 0xC2, 0xCA, 0x03}, Bytes{0x66, 0xF3, 0x0F, 0xC2, 0xCA, 0x03}, Bytes{0xF2, 0xF3, 0x0F, 0xC2, 0xCA, 0x03}, + Bytes{0x67, 0xF3, 0x0F, 0xC2, 0x0D, 0x00, 0x01, 0x00, 0x00, 0x03}, Bytes{0xF0, 0xF3, 0x0F, 0xC2, 0xCA, 0x03}, Bytes{0x48, 0xF3, 0x0F, 0xC2, 0xCA, 0x03}, + Bytes{0xF3, 0x0F, 0xC2, 0xCA}, Bytes{0xF3, 0x0F, 0xC3, 0xCA, 0x03}, Bytes{0xF3, 0x0F, 0x10, 0xCA}}) + require(!match(other).has_value(), "An instruction other than a legacy Rosetta-hazard scalar compare was moved"); + const Bytes compare = {0xF3, 0x0F, 0xC2, 0xCA, 0x03}; + require(!Codegen::MakeAmd64OnlyInstructionMatcher()->Match(compare.data(), compare.size()).has_value(), "A legacy scalar compare was moved for an Intel host"); +} + +void converterRosettaLegacyScalarCompare() { + const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); + Bytes file(0x300, 0xCC); + const Bytes text = { + 0xF3, 0x0F, 0x10, 0x15, 0x10, 0x00, 0x00, 0x00, + 0x0F, 0x28, 0xC2, + 0xF3, 0x0F, 0xC2, 0xCA, 0x03, + 0xC5, 0xF8, 0x28, 0x0D, 0x20, 0x00, 0x00, 0x00, + 0xC5, 0xF8, 0x2E, 0xC8, + 0xC3}; + std::copy(text.begin(), text.end(), file.begin() + 0x200); + const auto result = converter->Convert(file, {segmentHeader(text.size())}); + require(result.Trampolines.size() == 1 && result.Reports.size() == 1 && result.Reports[0].InstructionName == "scalar compare" && result.Reports[0].Offset == 0x20B && result.Reports[0].Lowering == Codegen::Amd64OnlyLowering::Trampoline, "The legacy compare in the fmax shape was not the only moved instruction"); + const auto& site = result.Trampolines[0]; + require(site.Offset == 0x20B && site.Length == 5 && Bytes(site.Body.begin(), site.Body.begin() + 5) == Bytes(text.begin() + 11, text.begin() + 16) && site.Relocations.empty(), "The legacy compare stub does not run the compare unchanged"); + const auto intel = Codegen::MakeAmd64OnlyConverter()->Convert(file, {segmentHeader(text.size())}); + require(intel.Reports.empty() && intel.Bytes == file, "The Intel target moved a legacy scalar compare"); +} + void converterRosettaScalarCompare() { const auto converter = Codegen::MakeAmd64OnlyConverter(Codegen::Amd64OnlyTarget::Rosetta); Bytes file(0x300, 0xCC); @@ -1729,6 +1771,8 @@ int main() { converterRosetta(); rosettaScalarCompares(); converterRosettaScalarCompare(); + rosettaLegacyScalarCompares(); + converterRosettaLegacyScalarCompare(); wideStoreSplit(); converterWideStoreWindows(); converterJumpTableTargets();