fix(avplayer): correct audio-video playback timing and sync

core/libs/SceTypes.hpp

@@ -680,10 +680,13 @@ struct AvPlayerVideoEx {
     std::uint8_t luma_bit_depth;
     std::uint8_t chroma_bit_depth;
     bool video_full_range_flag;
-    std::uint8_t reserved1[37];
+    std::uint8_t reserved1[5];
+    double frame_rate;
+    std::uint8_t reserved2[24];
 };
 static_assert(sizeof(AvPlayerVideoEx) == 80);
 static_assert(offsetof(AvPlayerVideoEx, crop_left_offset) == 20 && offsetof(AvPlayerVideoEx, pitch) == 36);
+static_assert(offsetof(AvPlayerVideoEx, frame_rate) == 48);
 
 struct AvPlayerTimedTextEx {
     std::uint8_t language_code[4];

core/libs/prx/libSceAudioOut/CMakeLists.txt

@@ -20,6 +20,15 @@ configure_windows_unwind(libSceAudioOut)
 
 if(BUILD_TESTING)
     if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+        add_test_executable(audio_out_port_locks_tests tests/AudioOutPortLocks.cpp)
+        target_include_directories(audio_out_port_locks_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_SDL2_INCLUDE_DIR})
+        target_link_libraries(audio_out_port_locks_tests PRIVATE libSceAudioOut libc)
+        set_target_properties(audio_out_port_locks_tests PROPERTIES ENABLE_EXPORTS ON)
+        add_test(NAME audio_out_independent_output COMMAND audio_out_port_locks_tests output)
+        add_test(NAME audio_out_independent_close COMMAND audio_out_port_locks_tests close)
+        add_test(NAME audio_out_same_port_close COMMAND audio_out_port_locks_tests same-close)
+        set_tests_properties(audio_out_independent_output audio_out_independent_close audio_out_same_port_close PROPERTIES TIMEOUT 10)
+
         add_test_executable(audio_out_resampling_tests tests/AudioOutResampling.cpp)
         target_include_directories(audio_out_resampling_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_SDL2_INCLUDE_DIR})
         target_link_libraries(audio_out_resampling_tests PRIVATE libSceAudioOut libc)

core/libs/prx/libSceAudioOut/src/AudioOut.cpp

@@ -1,5 +1,3 @@
-#include <cstdlib>
-#include <cstdio>
 #include <algorithm>
 #include <array>
 #include <atomic>
@@ -110,7 +108,7 @@ struct Port {
 };
 
 static std::mutex g_mutex;
-static std::mutex g_outputMutex;
+static std::array<std::mutex, PORTS_MAX> g_outputMutexes;
 static Port g_ports[PORTS_MAX];
 static bool g_sdlInitialized = false;
 
@@ -351,7 +349,8 @@ int APS5_VABI sceAudioOutOpen(int userId, int type, int index, std::uint32_t len
 }
 
 int APS5_VABI sceAudioOutClose(int handle) {
-    std::lock_guard<std::mutex> outputLock(g_outputMutex);
+    if (handle <= 0 || handle > PORTS_MAX) return -2144993277;
+    std::lock_guard<std::mutex> outputLock(g_outputMutexes[handle - 1]);
     std::lock_guard<std::mutex> lock(g_mutex);
     Port* port = getPort(handle);
     if (port == nullptr) {
@@ -364,7 +363,8 @@ int APS5_VABI sceAudioOutClose(int handle) {
 
 int APS5_VABI sceAudioOutOutput(int handle, const void* ptr) try {
     const AudioOutCall call;
-    std::lock_guard<std::mutex> outputLock(g_outputMutex);
+    if (handle <= 0 || handle > PORTS_MAX) return -2144993277;
+    std::lock_guard<std::mutex> outputLock(g_outputMutexes[handle - 1]);
     std::unique_lock<std::mutex> lock(g_mutex);
     Port* port = getPort(handle);
     if (port == nullptr) {
@@ -383,12 +383,20 @@ int APS5_VABI sceAudioOutOutput(int handle, const void* ptr) try {
 }
 
 int APS5_VABI sceAudioOutOutputs(AudioOutOutputParam* param, std::uint32_t num) try {
+    const AudioOutCall call;
     if (param == nullptr || num == 0) {
         return -2144993276;
     }
 
-    const AudioOutCall call;
-    std::lock_guard<std::mutex> outputLock(g_outputMutex);
+    std::array<bool, PORTS_MAX> selected{};
+    for (std::uint32_t i = 0; i < num; ++i) {
+        if (param[i].handle <= 0 || param[i].handle > PORTS_MAX) return -2144993277;
+        selected[param[i].handle - 1] = true;
+    }
+    std::array<std::unique_lock<std::mutex>, PORTS_MAX> outputLocks;
+    for (int index = 0; index < PORTS_MAX; ++index) {
+        if (selected[index]) outputLocks[index] = std::unique_lock<std::mutex>(g_outputMutexes[index]);
+    }
     std::unique_lock<std::mutex> lock(g_mutex);
 
     for (std::uint32_t i = 0; i < num; i++) {
@@ -551,7 +559,10 @@ int APS5_VABI sceAudioOutSetMixLevelPadSpk(int handle, int mixLevel) {
 }
 
 int APS5_VABI sceAudioOutExClose(int handle) {
-    std::lock_guard<std::mutex> outputLock(g_outputMutex);
+    if (handle <= 0 || handle > PORTS_MAX) {
+        throw std::runtime_error("sceAudioOutExClose: handle " + std::to_string(handle) + " is not an open Ex port");
+    }
+    std::lock_guard<std::mutex> outputLock(g_outputMutexes[handle - 1]);
     std::lock_guard<std::mutex> lock(g_mutex);
     Port* port = getPort(handle);
     if (port == nullptr || port->type != PORT_TYPE_EX) {

core/libs/prx/libSceAudioOut/tests/AudioOutLastOutputTime.cpp

@@ -156,6 +156,48 @@ void TestErrors() {
     Require(destination == untouched, "a rejected call must not write the destination");
 }
 
+void TestIndependentPortOutputDuringWait(bool batch) {
+    const std::vector<std::int16_t> longBlock(frequency, 256);
+    const std::vector<std::int16_t> shortBlock(frames, 256);
+    const int waiting = sceAudioOutOpen(user, portTypeVibration, 0, frequency, frequency, formatS16Mono);
+    const int independent = Open(portTypeVibration);
+    Require(waiting > 0, "waiting port must open");
+    Require(sceAudioOutOutput(waiting, longBlock.data()) == static_cast<int>(frequency), "initial output failed");
+    std::atomic<bool> started = false;
+    std::thread output([&] {
+        started = true;
+        AudioOutOutputParam param{waiting, longBlock.data()};
+        const int result = batch ? sceAudioOutOutputs(&param, 1) : sceAudioOutOutput(waiting, longBlock.data());
+        Require(result == static_cast<int>(frequency), "paced output failed");
+    });
+    while (!started) std::this_thread::yield();
+    std::this_thread::sleep_for(std::chrono::milliseconds(50));
+    const auto before = std::chrono::steady_clock::now();
+    Require(sceAudioOutOutput(independent, shortBlock.data()) == static_cast<int>(frames), "independent output failed");
+    const auto elapsed = std::chrono::steady_clock::now() - before;
+    output.join();
+    Require(sceAudioOutClose(waiting) == 0 && sceAudioOutClose(independent) == 0, "ports must close");
+    std::fprintf(stderr, "Independent port output waited %lld ms\n", static_cast<long long>(std::chrono::duration_cast<std::chrono::milliseconds>(elapsed).count()));
+    Require(elapsed < std::chrono::milliseconds(100), "pacing one audio port must not block output on another port");
+}
+
+void TestOverlappingOutputBatches() {
+    const std::vector<std::int16_t> block(frames, 256);
+    const int first = Open(portTypeVibration);
+    const int second = Open(portTypeVibration);
+    AudioOutOutputParam duplicate[2] = {{first, block.data()}, {first, block.data()}};
+    Require(sceAudioOutOutputs(duplicate, 2) == static_cast<int>(frames), "duplicate handles in a batch must not deadlock");
+    const auto output = [&](bool reverse) {
+        AudioOutOutputParam params[2] = {{reverse ? second : first, block.data()}, {reverse ? first : second, block.data()}};
+        for (int i = 0; i < 12; ++i) Require(sceAudioOutOutputs(params, 2) == static_cast<int>(frames), "overlapping output batch failed");
+    };
+    std::thread forward(output, false);
+    std::thread reverse(output, true);
+    forward.join();
+    reverse.join();
+    Require(sceAudioOutClose(first) == 0 && sceAudioOutClose(second) == 0, "batch ports must close");
+}
+
 void TestQueriesDuringOutput(bool batch, bool drain) {
     const std::vector<std::int16_t> block(frequency, 256);
     const int handle = sceAudioOutOpen(user, portTypeVibration, 0, frequency, frequency, formatS16Mono);
@@ -196,6 +238,9 @@ int main() {
     TestReopenedPort();
     TestErrors();
     TestPortWithoutDeviceKeepsRealTime();
+    TestIndependentPortOutputDuringWait(false);
+    TestIndependentPortOutputDuringWait(true);
+    TestOverlappingOutputBatches();
     TestQueriesDuringOutput(false, false);
     TestQueriesDuringOutput(false, true);
     TestQueriesDuringOutput(true, false);

core/libs/prx/libSceAudioOut/tests/AudioOutPortLocks.cpp

@@ -0,0 +1,135 @@
+#include "SDL.h"
+#include "prx/libc/include/general/VabiMacros.hpp"
+
+#include <array>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <cstdint>
+#include <cstdio>
+#include <cstdlib>
+#include <mutex>
+#include <string_view>
+#include <thread>
+
+extern "C" {
+int APS5_VABI sceAudioOutOpen(int, int, int, std::uint32_t, std::uint32_t, std::uint32_t);
+int APS5_VABI sceAudioOutOutput(int, const void*);
+int APS5_VABI sceAudioOutClose(int);
+}
+
+namespace {
+
+constexpr std::uint32_t Grain = 256;
+std::atomic<SDL_AudioDeviceID> nextDevice{2};
+std::atomic<bool> blocked{false};
+std::atomic<bool> released{false};
+std::atomic<unsigned> clockQueries{0};
+std::atomic<unsigned> clears{0};
+std::array<std::atomic<unsigned>, 4> queued{};
+std::array<std::atomic<bool>, 4> closed{};
+std::mutex stateMutex;
+std::condition_variable changed;
+
+void Require(bool condition, const char* message) {
+    if (condition) return;
+    std::fprintf(stderr, "AudioOut port locks: %s\n", message);
+    std::abort();
+}
+
+bool WaitFor(const std::atomic<bool>& condition, std::chrono::milliseconds timeout) {
+    std::unique_lock lock(stateMutex);
+    return changed.wait_for(lock, timeout, [&] { return condition.load(std::memory_order_acquire); });
+}
+
+}
+
+extern "C" std::uint64_t APS5_VABI sceKernelGetProcessTime(void) {
+    clockQueries.fetch_add(1, std::memory_order_relaxed);
+    return 1;
+}
+
+extern "C" int SDLCALL SDL_InitSubSystem(Uint32) { return 0; }
+extern "C" Uint32 SDLCALL SDL_WasInit(Uint32) { return SDL_INIT_AUDIO; }
+extern "C" void SDLCALL SDL_PauseAudioDevice(SDL_AudioDeviceID, int) {}
+
+extern "C" SDL_AudioDeviceID SDLCALL SDL_OpenAudioDevice(const char*, int, const SDL_AudioSpec* desired, SDL_AudioSpec* obtained, int) {
+    *obtained = *desired;
+    return nextDevice.fetch_add(1, std::memory_order_relaxed);
+}
+
+extern "C" Uint32 SDLCALL SDL_GetQueuedAudioSize(SDL_AudioDeviceID device) {
+    Require(device == 2 || device == 3, "unknown device polled");
+    if (device == 2 && !released.load(std::memory_order_acquire)) {
+        blocked.store(true, std::memory_order_release);
+        changed.notify_all();
+        return 1u << 20;
+    }
+    return 0;
+}
+
+extern "C" void SDLCALL SDL_ClearQueuedAudio(SDL_AudioDeviceID device) {
+    if (device == 2 && !released.load(std::memory_order_acquire)) {
+        clears.fetch_add(1, std::memory_order_relaxed);
+        released.store(true, std::memory_order_release);
+    }
+}
+
+extern "C" int SDLCALL SDL_QueueAudio(SDL_AudioDeviceID device, const void* data, Uint32 bytes) {
+    Require(device == 2 || device == 3, "unknown device queued");
+    Require(!closed[device].load(std::memory_order_acquire), "output used a closed device");
+    Require(bytes == Grain * sizeof(std::int16_t), "output size changed");
+    const auto* samples = static_cast<const std::int16_t*>(data);
+    const auto expected = device == 2 ? 1000 : 2000;
+    for (std::uint32_t i = 0; i < Grain; ++i) Require(samples[i] == expected, "output samples changed");
+    queued[device].fetch_add(1, std::memory_order_relaxed);
+    return 0;
+}
+
+extern "C" void SDLCALL SDL_CloseAudioDevice(SDL_AudioDeviceID device) {
+    Require(device == 2 || device == 3, "unknown device closed");
+    Require(!closed[device].exchange(true, std::memory_order_acq_rel), "device closed twice");
+}
+
+int main(int argc, char** argv) {
+    Require(argc == 2, "expected a test mode");
+    const std::string_view mode(argv[1]);
+    const bool unrelatedOutput = mode == "output";
+    const bool unrelatedClose = mode == "close";
+    const bool samePortClose = mode == "same-close";
+    Require(unrelatedOutput || unrelatedClose || samePortClose, "unknown test mode");
+    const int first = sceAudioOutOpen(0, 0, 0, Grain, 48000, 0);
+    const int second = sceAudioOutOpen(0, 0, 0, Grain, 48000, 0);
+    Require(first > 0 && second > 0 && first != second, "port creation failed");
+    std::array<std::int16_t, Grain> firstSamples;
+    std::array<std::int16_t, Grain> secondSamples;
+    firstSamples.fill(1000);
+    secondSamples.fill(2000);
+    int firstResult = -1;
+    int secondResult = -1;
+    std::atomic<bool> secondStarted{false};
+    std::atomic<bool> secondDone{false};
+    std::thread producer([&] { firstResult = sceAudioOutOutput(first, firstSamples.data()); });
+    Require(WaitFor(blocked, std::chrono::seconds(2)), "first port did not enter backpressure");
+    std::thread other([&] {
+        secondStarted.store(true, std::memory_order_release);
+        changed.notify_all();
+        secondResult = unrelatedOutput ? sceAudioOutOutput(second, secondSamples.data()) : sceAudioOutClose(samePortClose ? first : second);
+        secondDone.store(true, std::memory_order_release);
+        changed.notify_all();
+    });
+    Require(WaitFor(secondStarted, std::chrono::seconds(2)), "second operation did not start");
+    const bool completedWhileBlocked = WaitFor(secondDone, std::chrono::milliseconds(samePortClose ? 150 : 2000));
+    released.store(true, std::memory_order_release);
+    producer.join();
+    other.join();
+    Require(clockQueries.load() != 0 && clears.load() == 0, "drain timeout invalidated the controlled wait");
+    Require(firstResult == static_cast<int>(Grain), "blocked output failed");
+    Require(secondResult == (unrelatedOutput ? static_cast<int>(Grain) : 0), "second operation failed");
+    Require(completedWhileBlocked != samePortClose, "port output/close lock scope is wrong");
+    Require(queued[2].load() == 1 && queued[3].load() == (unrelatedOutput ? 1u : 0u), "unexpected queued output count");
+    if (!samePortClose) Require(sceAudioOutClose(first) == 0, "first port close failed");
+    if (!unrelatedClose) Require(sceAudioOutClose(second) == 0, "second port close failed");
+    Require(closed[2].load() && closed[3].load(), "a device remained open");
+    return 0;
+}

core/libs/prx/libSceAvPlayer/Export.cpp

@@ -112,7 +112,7 @@ int APS5_VABI sceAvPlayerInitEx(const AvPlayerInitDataEx* init_ex, AvPlayerInter
         init.auto_start = init_ex->auto_start;
     }
     init.default_language = init_ex->default_language;
-    *handle = new Player(init, VideoBufferCount(init.num_output_video_framebuffers));
+    *handle = new Player(init, VideoBufferCount(init.num_output_video_framebuffers), init_ex->this_size >= Ps5InitDataExSize);
     return SCE_OK;
 }
 

core/libs/prx/libSceAvPlayer/include/AvPlayer.hpp

@@ -32,6 +32,7 @@ constexpr std::int32_t EventWarningId = 0x20;
 
 constexpr std::uint32_t StreamTypeVideo = 1;
 constexpr std::uint32_t StreamTypeAudio = 0;
+constexpr std::uint32_t Ps5StreamTypeAudio = 2;
 constexpr std::uint32_t StreamTypeTimedText = 2;
 
 constexpr std::uint32_t SourceTypeUnknown = 0;
@@ -86,7 +87,7 @@ std::uint32_t DetectSourceType(std::string_view path);
 
 class Player final : public AvPlayerInternal, private ISourceEvents {
 public:
-    Player(const AvPlayerInitData& init, std::uint32_t bufferCount);
+    Player(const AvPlayerInitData& init, std::uint32_t bufferCount, bool ps5Layout = false);
     ~Player() override;
     Player(const Player&) = delete;
     Player& operator=(const Player&) = delete;
@@ -138,6 +139,7 @@ class Player final : public AvPlayerInternal, private ISourceEvents {
     AvPlayerMemAllocator memory{};
     AvPlayerEventReplacement callback{};
     bool autoStartEnabled = false;
+    bool ps5Layout = false;
     char language[4]{};
     std::uint32_t videoBuffers = 2;
 

core/libs/prx/libSceAvPlayer/src/Player.cpp

@@ -61,8 +61,8 @@ std::uint32_t DetectSourceType(std::string_view path) {
     return SourceTypeUnknown;
 }
 
-Player::Player(const AvPlayerInitData& init, std::uint32_t bufferCount)
-    : memory(WithGuestHeapFallback(init.memory_replacement)), callback(init.event_replacement), videoBuffers(bufferCount) {
+Player::Player(const AvPlayerInitData& init, std::uint32_t bufferCount, bool ps5Layout)
+    : memory(WithGuestHeapFallback(init.memory_replacement)), callback(init.event_replacement), ps5Layout(ps5Layout), videoBuffers(bufferCount) {
     if (init.file_replacement.open && init.file_replacement.close && init.file_replacement.read_offset && init.file_replacement.size) {
         file = init.file_replacement;
     }
@@ -169,7 +169,7 @@ void Player::autoStart() {
             return;
         }
         if (info.type == StreamTypeVideo && (video == -1 || SameLanguage(language, info.details.video.language_code))) video = index;
-        if (info.type == StreamTypeAudio && (audio == -1 || SameLanguage(language, info.details.audio.language_code))) audio = index;
+        if (info.type == (ps5Layout ? Ps5StreamTypeAudio : StreamTypeAudio) && (audio == -1 || SameLanguage(language, info.details.audio.language_code))) audio = index;
     }
     if (video != -1) EnableStream(static_cast<std::uint32_t>(video));
     if (audio != -1) EnableStream(static_cast<std::uint32_t>(audio));
@@ -233,13 +233,15 @@ int Player::StreamCount() {
 int Player::GetStreamInfo(std::uint32_t index, AvPlayerStreamInfo& info) {
     std::lock_guard lock(mutex);
     if (!source || !source->GetStreamInfo(index, info)) return SCE_AVPLAYER_ERROR_OPERATION_FAILED;
+    if (ps5Layout && info.type == StreamTypeAudio) info.type = Ps5StreamTypeAudio;
     return SCE_OK;
 }
 
 int Player::GetStreamInfoEx(std::uint32_t index, AvPlayerStreamInfoEx& info) {
     std::lock_guard lock(mutex);
     AvPlayerStreamInfoEx described{};
     if (!source || !source->GetStreamInfoEx(index, described)) return SCE_AVPLAYER_ERROR_OPERATION_FAILED;
+    if (ps5Layout && described.type == StreamTypeAudio) described.type = Ps5StreamTypeAudio;
     described.this_size = info.this_size;
     info = described;
     return SCE_OK;

core/libs/prx/libSceAvPlayer/src/Source.cpp

@@ -580,6 +580,8 @@ class FfmpegSource final : public ISource {
             picture.luma_bit_depth = 8;
             picture.chroma_bit_depth = 8;
             picture.video_full_range_flag = parameters->color_range == AVCOL_RANGE_JPEG;
+            const auto rate = av_guess_frame_rate(format, const_cast<AVStream*>(&stream), nullptr);
+            picture.frame_rate = rate.num > 0 && rate.den > 0 ? av_q2d(rate) : 0.0;
         } else {
             auto& sound = info.details.audio;
             sound.channel_count = basic.details.audio.channel_count;
@@ -988,6 +990,8 @@ class FfmpegSource final : public ISource {
         info.details.video.luma_bit_depth = 8;
         info.details.video.chroma_bit_depth = 8;
         info.details.video.video_full_range_flag = frame.color_range == AVCOL_RANGE_JPEG;
+        const auto rate = av_guess_frame_rate(format, format->streams[index], nullptr);
+        info.details.video.frame_rate = rate.num > 0 && rate.den > 0 ? av_q2d(rate) : 0.0;
         publish(decoder, output);
         return true;
     }