diff --git a/core/libs/prx/libc/include/GuestLocale.hpp b/core/libs/prx/libc/include/GuestLocale.hpp index 45e29df3f1c..745d6dd82be 100644 --- a/core/libs/prx/libc/include/GuestLocale.hpp +++ b/core/libs/prx/libc/include/GuestLocale.hpp @@ -58,6 +58,23 @@ struct CollateVtable { std::int64_t (APS5_VABI *hash)(const CollateFacet* self, const char* first, const char* last); }; +struct WideString { + std::uint64_t reserved; + union { + std::uint16_t buffer[8]; + std::uint16_t* pointer; + }; + std::uint64_t size; + std::uint64_t capacity; +}; + +struct WideCollateVtable { + FacetVtable facet; + int (APS5_VABI *compare)(const CollateFacet* self, const std::uint16_t* first1, const std::uint16_t* last1, const std::uint16_t* first2, const std::uint16_t* last2); + WideString* (APS5_VABI *transform)(WideString* result, const CollateFacet* self, const std::uint16_t* first, const std::uint16_t* last); + std::int64_t (APS5_VABI *hash)(const CollateFacet* self, const std::uint16_t* first, const std::uint16_t* last); +}; + struct IosBase { const void* vtable; std::int64_t standardStream; @@ -89,6 +106,11 @@ static_assert(offsetof(CollateFacet, collation) == 0x10); static_assert(sizeof(CollateFacet) == 0x20); static_assert(offsetof(CollateVtable, compare) == 0x20); static_assert(offsetof(CollateVtable, hash) == 0x30); +static_assert(offsetof(WideString, buffer) == 0x8); +static_assert(offsetof(WideString, size) == 0x18); +static_assert(sizeof(WideString) == 0x28); +static_assert(offsetof(WideCollateVtable, compare) == 0x20); +static_assert(offsetof(WideCollateVtable, hash) == 0x30); static_assert(alignof(LocinfoStorage) == 8); } diff --git a/core/libs/prx/libc/src/LocaleSupport.cpp b/core/libs/prx/libc/src/LocaleSupport.cpp index 941df32b3b4..421f8bd4f4e 100644 --- a/core/libs/prx/libc/src/LocaleSupport.cpp +++ b/core/libs/prx/libc/src/LocaleSupport.cpp @@ -79,6 +79,7 @@ GuestLocale::Facet* g_classicFacets[1]{}; GuestLocale::Implementation g_classicLocale{{&g_localeVtable, 1, 0}, g_classicFacets, 1, 0, false, "C"}; constexpr std::size_t GuestStringInlineCapacity = 15; +constexpr std::size_t GuestWideStringInlineCapacity = 7; constexpr std::size_t GuestStringLargeAllocation = 0x1000; void ValidateCharacterRange(const char* first, const char* last, const char* function) { @@ -143,6 +144,62 @@ std::int64_t APS5_VABI CollateHash(const GuestLocale::CollateFacet*, const char* const GuestLocale::CollateVtable g_collateVtable{{DestroyCollate, DeleteCollate, RetainFacet, ReleaseFacet}, CollateCompare, CollateTransform, CollateHash}; +void ValidateWideRange(const std::uint16_t* first, const std::uint16_t* last, const char* function) { + if ((first == nullptr) != (last == nullptr) || last < first) throw std::invalid_argument(std::string(function) + ": invalid character range"); +} + +int APS5_VABI WideCollateCompare(const GuestLocale::CollateFacet*, const std::uint16_t* first1, const std::uint16_t* last1, const std::uint16_t* first2, const std::uint16_t* last2) { + ValidateWideRange(first1, last1, "collate::do_compare"); + ValidateWideRange(first2, last2, "collate::do_compare"); + for (; first1 != last1 && first2 != last2; ++first1, ++first2) { + if (*first1 != *first2) return *first1 < *first2 ? -1 : 1; + } + if (first1 == last1 && first2 == last2) return 0; + return first1 == last1 ? -1 : 1; +} + +GuestLocale::WideString* APS5_VABI WideCollateTransform(GuestLocale::WideString* result, const GuestLocale::CollateFacet*, const std::uint16_t* first, const std::uint16_t* last) { + if (result == nullptr) throw std::invalid_argument("collate::do_transform: null result"); + ValidateWideRange(first, last, "collate::do_transform"); + const auto length = static_cast(last - first); + if (std::find(first, last, std::uint16_t{0}) != last) throw std::runtime_error("collate::do_transform: embedded null characters are not supported"); + if ((length + 1) * sizeof(std::uint16_t) >= GuestStringLargeAllocation) throw std::runtime_error("collate::do_transform: results of 2047 characters or more are not supported"); + std::uint16_t* data = result->buffer; + std::size_t capacity = GuestWideStringInlineCapacity; + if (length > GuestWideStringInlineCapacity) { + capacity = length; + data = static_cast(_Znwm_nid_postfix((capacity + 1) * sizeof(std::uint16_t))); + } + if (length != 0) std::memcpy(data, first, length * sizeof(std::uint16_t)); + data[length] = 0; + if (data != result->buffer) result->pointer = data; + result->size = length; + result->capacity = capacity; + return result; +} + +std::int64_t APS5_VABI WideCollateHash(const GuestLocale::CollateFacet*, const std::uint16_t* first, const std::uint16_t* last) { + ValidateWideRange(first, last, "collate::do_hash"); + std::uint64_t hash = 0xcbf29ce484222325; + for (; first != last; ++first) { + hash = (hash ^ (*first & 0xff)) * 0x100000001b3; + hash = (hash ^ (*first >> 8)) * 0x100000001b3; + } + return static_cast(hash); +} + +const GuestLocale::WideCollateVtable g_wideCollateVtable{{DestroyCollate, DeleteCollate, RetainFacet, ReleaseFacet}, WideCollateCompare, WideCollateTransform, WideCollateHash}; + +std::size_t GetCollateCategory(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale, const GuestLocale::FacetVtable* vtable) { + if (facet != nullptr && *facet == nullptr) { + if (locale == nullptr || *locale == nullptr || (*locale)->name == nullptr || std::strcmp((*locale)->name, "C") != 0) throw std::invalid_argument("collate::_Getcat: only the C locale is supported"); + auto* collate = static_cast(_Znwm_nid_postfix(sizeof(GuestLocale::CollateFacet))); + *collate = {{vtable, 0, 0}, nullptr, nullptr}; + *facet = &collate->base; + } + return 1; +} + constexpr std::array MakeClassificationTable() { std::array table{}; for (unsigned int value = 0; value < 256; ++value) { @@ -257,18 +314,12 @@ GuestLocale::Implementation* APS5_VABI _ZNSt6locale16_GetgloballocaleEv_nid_post return &g_classicLocale; } -void APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet**, const GuestLocale::Implementation*) { - NotImplemented_nid_no_patch(__func__); +std::size_t APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale) { + return GetCollateCategory(facet, locale, &g_wideCollateVtable.facet); } std::size_t APS5_VABI _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale) { - if (facet != nullptr && *facet == nullptr) { - if (locale == nullptr || *locale == nullptr || (*locale)->name == nullptr || std::strcmp((*locale)->name, "C") != 0) throw std::invalid_argument("collate::_Getcat: only the C locale is supported"); - auto* collate = static_cast(_Znwm_nid_postfix(sizeof(GuestLocale::CollateFacet))); - *collate = {{&g_collateVtable.facet, 0, 0}, nullptr, nullptr}; - *facet = &collate->base; - } - return 1; + return GetCollateCategory(facet, locale, &g_collateVtable.facet); } void APS5_VABI _ZNSt8_LocinfoC1EPKc_nid_postfix(GuestLocale::LocinfoStorage* self, const char* localeName) { diff --git a/core/libs/tests/GuestCollate.cpp b/core/libs/tests/GuestCollate.cpp index ed4b1976d40..ba1ef5b3fc8 100644 --- a/core/libs/tests/GuestCollate.cpp +++ b/core/libs/tests/GuestCollate.cpp @@ -7,10 +7,12 @@ #include #include #include +#include extern "C" { extern GuestLocale::Implementation* _ZSt21_sceLibcClassicLocale_nid_postfix; std::size_t APS5_VABI _ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale); +std::size_t APS5_VABI _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(GuestLocale::Facet** facet, const GuestLocale::Implementation* const* locale); } namespace { @@ -63,6 +65,121 @@ std::uint64_t Hash(const GuestLocale::CollateFacet* facet, const std::string& te return static_cast(Vtable(&facet->base).hash(facet, text.data(), text.data() + text.size())); } +using WideText = std::vector; + +WideText Terminated(const std::uint16_t* text) { + WideText result; + for (; *text != 0; ++text) result.push_back(*text); + return result; +} + +WideText Wide(const char* text) { + WideText result; + for (; *text != 0; ++text) result.push_back(static_cast(*text)); + return result; +} + +const GuestLocale::WideCollateVtable& WideVtable(const GuestLocale::Facet* facet) { + return *reinterpret_cast(facet->vtable); +} + +int WideCompare(const GuestLocale::CollateFacet* facet, const WideText& left, const WideText& right) { + return WideVtable(&facet->base).compare(facet, left.data(), left.data() + left.size(), right.data(), right.data() + right.size()); +} + +std::uint64_t WideHash(const GuestLocale::CollateFacet* facet, const WideText& text) { + return static_cast(WideVtable(&facet->base).hash(facet, text.data(), text.data() + text.size())); +} + +void CheckWideCollate() { + const GuestLocale::Implementation* const* classic = &_ZSt21_sceLibcClassicLocale_nid_postfix; + const auto before = allocations; + require(_ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(nullptr, classic) == 1); + GuestLocale::Facet existing{}; + GuestLocale::Facet* facet = &existing; + require(_ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, classic) == 1); + require(facet == &existing && allocations == before); + + GuestLocale::Implementation french = *_ZSt21_sceLibcClassicLocale_nid_postfix; + french.name = "fr_FR"; + const GuestLocale::Implementation* frenchPointer = &french; + facet = nullptr; + reject([&] { _ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, &frenchPointer); }); + require(facet == nullptr && allocations == before); + + require(_ZNSt7collateIwE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&facet, classic) == 1); + require(facet != nullptr && allocations == before + 1 && lastSize == sizeof(GuestLocale::CollateFacet) && lastAllocation == facet); + const auto* collate = reinterpret_cast(facet); + require(facet->references == 0 && collate->collation == nullptr && collate->wideCollation == nullptr); + GuestLocale::Facet* narrow = nullptr; + require(_ZNSt7collateIcE7_GetcatEPPKNSt6locale5facetEPKS1__nid_postfix(&narrow, classic) == 1); + require(narrow->vtable != facet->vtable); + std::free(narrow); + + require(WideCompare(collate, Wide("abc"), Wide("abd")) == -1); + require(WideCompare(collate, Wide("abd"), Wide("abc")) == 1); + require(WideCompare(collate, Wide("abc"), Wide("abc")) == 0); + require(WideCompare(collate, Wide("ab"), Wide("abc")) == -1); + require(WideCompare(collate, Wide("abc"), Wide("ab")) == 1); + require(WideCompare(collate, Wide(""), Wide("")) == 0); + require(WideCompare(collate, WideText{0x8000}, Wide("a")) == 1); + require(WideCompare(collate, WideText{0x0100}, WideText{0x00ff}) == 1); + require(WideCompare(collate, WideText{'a', 0}, Wide("a")) == 1); + require(WideCompare(collate, WideText{'a', 0, 'b'}, WideText{'a', 0, 'c'}) == -1); + require(WideVtable(facet).compare(collate, nullptr, nullptr, nullptr, nullptr) == 0); + const std::uint16_t text[] = {'a', 'b', 'c'}; + reject([&] { WideVtable(facet).compare(collate, text + 2, text, text, text + 1); }); + reject([&] { WideVtable(facet).compare(collate, nullptr, text, text, text + 1); }); + + require(WideHash(collate, WideText{}) == 0xcbf29ce484222325ull); + require(WideHash(collate, Wide("a")) == 0x089be207b544f1e4ull); + require(WideHash(collate, Wide("foobar")) == 0xc882d5010e965f1aull); + require(WideHash(collate, WideText{0x0100}) == 0x08328707b4eb6e3aull); + require(WideHash(collate, WideText{0x1234}) == 0x07ee9e07b4b1c883ull); + reject([&] { WideVtable(facet).hash(collate, text + 1, text); }); + + GuestLocale::WideString result; + std::memset(&result, 0xcd, sizeof(result)); + const WideText shortText = Wide("hello"); + const auto beforeTransform = allocations; + require(WideVtable(facet).transform(&result, collate, shortText.data(), shortText.data() + shortText.size()) == &result); + require(result.reserved == 0xcdcdcdcdcdcdcdcdull && result.size == 5 && result.capacity == 7 && Terminated(result.buffer) == shortText); + require(allocations == beforeTransform); + + std::memset(&result, 0xcd, sizeof(result)); + require(WideVtable(facet).transform(&result, collate, nullptr, nullptr) == &result); + require(result.size == 0 && result.capacity == 7 && result.buffer[0] == 0); + + const WideText seven(7, 'q'); + require(WideVtable(facet).transform(&result, collate, seven.data(), seven.data() + seven.size()) == &result); + require(result.size == 7 && result.capacity == 7 && Terminated(result.buffer) == seven && allocations == beforeTransform); + + const WideText eight(8, 'r'); + require(WideVtable(facet).transform(&result, collate, eight.data(), eight.data() + eight.size()) == &result); + require(allocations == beforeTransform + 1 && lastSize == 18 && result.pointer == lastAllocation); + require(result.size == 8 && result.capacity == 8 && Terminated(result.pointer) == eight); + std::free(result.pointer); + + const WideText largest(2046, 0x4e2d); + require(WideVtable(facet).transform(&result, collate, largest.data(), largest.data() + largest.size()) == &result); + require(allocations == beforeTransform + 2 && lastSize == 4094 && result.capacity == 2046 && Terminated(result.pointer) == largest); + std::free(result.pointer); + + const WideText tooLong(2047, 'z'); + reject([&] { WideVtable(facet).transform(&result, collate, tooLong.data(), tooLong.data() + tooLong.size()); }); + const WideText embedded{'a', 0, 'b'}; + reject([&] { WideVtable(facet).transform(&result, collate, embedded.data(), embedded.data() + embedded.size()); }); + reject([&] { WideVtable(facet).transform(nullptr, collate, text, text + 1); }); + require(allocations == beforeTransform + 2); + + WideVtable(facet).facet.retain(facet); + require(facet->references == 1); + require(WideVtable(facet).facet.release(facet) == facet && facet->references == 0); + const auto freesBefore = frees; + WideVtable(facet).facet.deleteObject(facet); + require(frees == freesBefore + 1 && lastFree == collate); +} + } int main() { @@ -152,5 +269,6 @@ int main() { require(frees == 0); Vtable(facet).facet.deleteObject(facet); require(frees == 1 && lastFree == collate); + CheckWideCollate(); return 0; } diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 81b67436620..4323c41ea46 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -339,6 +339,7 @@ Throughout the project, every function at every stage either **does exactly what - [wcsrtombs_s](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached - [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) passes `%ls` and `%S` to glibc on Linux, which reads them as strings of 32-bit `wchar_t`; the guest's `wchar_t` is 16 bits (UTF-16), so these conversions fail with -1. The Windows path converts them to UTF-8, and `guest_formatting` checks wide conversions on Windows only - [collate::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation. +- [collate::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet the way `collate::_Getcat` does, with the same C-locale-only and null-table limits. Only the narrow facet was read from the console's `libc.prx`; the wide one is taken from the same Dinkumware templates (MS STL `collate` and `basic_string`) and is not checked against the console. Characters are the guest's 16-bit unsigned `wchar_t`, compared by value as the C locale's one-to-one `wcscoll` does; `do_hash` is the FNV-1a of their little-endian bytes, as MS STL's `collate::do_hash` hashes `count * sizeof(wchar_t)` bytes, and the string's inline buffer holds 7 characters (16 bytes). `do_transform` throws for embedded null characters and for results of 2047 characters or more, whose 4096-byte string storage uses the guest's over-aligned large allocation - [poll](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) waits only on guest sockets, through `WSAPoll` on Windows and `poll` on Linux; the `pollfd` layout and flag values are FreeBSD's. A descriptor below the socket range (files, pipes, stdio) throws, as do event bits other than `POLLIN`, `POLLPRI`, `POLLOUT`, `POLLRDNORM`, `POLLRDBAND` and `POLLWRBAND` (the `POLLERR`, `POLLHUP` and `POLLNVAL` bits are ignored in `events`), `POLLPRI` on Windows, which `WSAPoll` rejects, and an infinite timeout with no socket to wait on. A closed socket descriptor reports `POLLNVAL`; the console's limit on the descriptor count is unknown and not checked - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for.