diff --git a/src/android/app/build.gradle.kts b/src/android/app/build.gradle.kts index a4076978a1..fd7c40e415 100644 --- a/src/android/app/build.gradle.kts +++ b/src/android/app/build.gradle.kts @@ -65,7 +65,7 @@ android { defaultConfig { applicationId = "dev.eden.eden_emulator" - minSdk = 24 + minSdk = 33 targetSdk = 36 versionName = getGitVersion() versionCode = autoVersion diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt index 7f6e793199..16e74553cc 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt @@ -27,6 +27,7 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"), RENDERER_ASYNC_PRESENTATION("async_presentation"), RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"), + RENDERER_UNIFIED_MEMORY("use_unified_memory"), RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"), ENABLE_BUFFER_HISTORY("enable_buffer_history"), USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"), @@ -36,6 +37,8 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_DEBUG("debug"), RENDERER_PATCH_OLD_QCOM_DRIVERS("patch_old_qcom_drivers"), RENDERER_VERTEX_INPUT_DYNAMIC_STATE("vertex_input_dynamic_state"), + RENDERER_DYNAMIC_RENDERING("dynamic_rendering"), + RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT("workgroup_memory_explicit_layout"), RENDERER_SAMPLE_SHADING("sample_shading"), GPU_UNSWIZZLE_ENABLED("gpu_unswizzle_enabled"), PICTURE_IN_PICTURE("picture_in_picture"), diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt index 7c6063988f..12051e356e 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt @@ -155,6 +155,20 @@ abstract class SettingsItem( descriptionId = R.string.vertex_input_dynamic_state_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_DYNAMIC_RENDERING, + titleId = R.string.dynamic_rendering, + descriptionId = R.string.dynamic_rendering_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT, + titleId = R.string.workgroup_memory_explicit_layout, + descriptionId = R.string.workgroup_memory_explicit_layout_description + ) + ) put( SliderSetting( IntSetting.RENDERER_SAMPLE_SHADING, @@ -594,7 +608,7 @@ abstract class SettingsItem( IntSetting.ANDROID_PIPELINE_WORKERS, titleId = R.string.pipeline_worker_cores, descriptionId = R.string.pipeline_worker_cores_description, - min = 4, + min = 2, max = 8, units = "cores" ) @@ -685,6 +699,13 @@ abstract class SettingsItem( descriptionId = R.string.renderer_asynchronous_shaders_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_UNIFIED_MEMORY, + titleId = R.string.renderer_unified_memory, + descriptionId = R.string.renderer_unified_memory_description + ) + ) put( SingleChoiceSetting( IntSetting.FAST_GPU_TIME, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt index 81702d7110..4f11fdb96d 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt @@ -304,6 +304,7 @@ class SettingsFragmentPresenter( add(BooleanSetting.EMULATE_BGR565.key) add(BooleanSetting.RESCALE_HACK.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key) + add(BooleanSetting.RENDERER_UNIFIED_MEMORY.key) add(IntSetting.ANDROID_PIPELINE_WORKERS.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key) add(BooleanSetting.RENDERER_ASYNC_PRESENTATION.key) @@ -313,6 +314,8 @@ class SettingsFragmentPresenter( add(IntSetting.RENDERER_DYNA_STATE.key) add(BooleanSetting.RENDERER_VERTEX_INPUT_DYNAMIC_STATE.key) + add(BooleanSetting.RENDERER_DYNAMIC_RENDERING.key) + add(BooleanSetting.RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT.key) add(IntSetting.RENDERER_SAMPLE_SHADING.key) add(HeaderSetting(R.string.display)) diff --git a/src/android/app/src/main/jni/android_settings.h b/src/android/app/src/main/jni/android_settings.h index 304ff62f07..37da651519 100644 --- a/src/android/app/src/main/jni/android_settings.h +++ b/src/android/app/src/main/jni/android_settings.h @@ -147,13 +147,6 @@ namespace AndroidSettings { &show_performance_overlay}; - Settings::Setting pipeline_worker_count{linkage, 4, "pipeline_worker_count", - Settings::Category::Android, - Settings::Specialization::Default, - true, - true}; - - Settings::Setting show_input_overlay{linkage, true, "show_input_overlay", Settings::Category::Overlay}; Settings::Setting overlay_snap_to_grid{linkage, false, "overlay_snap_to_grid", diff --git a/src/android/app/src/main/res/values/strings.xml b/src/android/app/src/main/res/values/strings.xml index cd5d0c41e7..b57c610e76 100644 --- a/src/android/app/src/main/res/values/strings.xml +++ b/src/android/app/src/main/res/values/strings.xml @@ -524,6 +524,8 @@ Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path Use asynchronous shaders Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches. + Unified memory access (UMA) + Allows GPU write buffer readbacks directly into guest memory, skipping the CPU staging copy. GPU Unswizzle Settings Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality. Enable GPU Unswizzle @@ -544,6 +546,10 @@ Disabled Vertex Input Dynamic State Enabling this feature allows for more flexible vertex input handling, potentially reducing pipeline compilation time in vertex/buffer. + Dynamic Rendering + Render without render pass and framebuffer objects. Results vary by driver: some gain performance, others lose it. + Workgroup Memory Explicit Layout + Let shaders declare explicit layouts for workgroup memory. Disabled by default: some Qualcomm drivers are unstable with it. Sample Shading Allows the fragment shader to execute per sample in a multi-sampled fragment instead once per fragment. Improves graphics quality at the cost of some performance. diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 445d34c444..66ff190257 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,14 +51,60 @@ #endif // ^^^ POSIX ^^^ +#include #include #include +#include #include "common/alignment.h" #include "common/assert.h" #include "common/free_region_manager.h" #include "common/host_memory.h" #include "common/logging.h" +#include "common/memory_detect.h" +#include "common/settings.h" + +#ifdef __ANDROID__ +#include +#include + +namespace { + +struct NativeHandle { + int version; + int numFds; + int numInts; + int data[1]; +}; + +using PFN_AHardwareBuffer_getNativeHandle = const NativeHandle* (*)(const AHardwareBuffer*); +using PFN_AHardwareBuffer_isSupported = int (*)(const AHardwareBuffer_Desc*); + +void* NativeWindowLibrary() { + static void* const lib = dlopen("libnativewindow.so", RTLD_NOW); + return lib; +} + +PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() { + void* const lib = NativeWindowLibrary(); + if (lib == nullptr) { + return nullptr; + } + return reinterpret_cast( + dlsym(lib, "AHardwareBuffer_getNativeHandle")); +} + +PFN_AHardwareBuffer_isSupported ResolveIsSupported() { + void* const lib = NativeWindowLibrary(); + if (lib == nullptr) { + return nullptr; + } + return reinterpret_cast( + dlsym(lib, "AHardwareBuffer_isSupported")); +} + +} // namespace +#endif #if defined(__ANDROID__) && __ANDROID_API__ < 30 #include @@ -75,6 +121,12 @@ namespace Common { [[maybe_unused]] constexpr size_t PageAlignment = 0x1000; [[maybe_unused]] constexpr size_t HugePageSize = 0x200000; +static std::atomic committed_backing_size{}; + +u64 GetCommittedBackingSize() noexcept { + return committed_backing_size.load(std::memory_order_relaxed); +} + #ifdef _WIN32 // Manually imported for MinGW compatibility @@ -509,6 +561,11 @@ public: bool Init() { long page_size = sysconf(_SC_PAGESIZE); ASSERT_MSG(page_size == 0x1000, "page size {:#x} is incompatible with 4K paging", page_size); +#ifdef __ANDROID__ + if (InitAhbBacking()) { + return InitVirtual(); + } +#endif // Backing memory initialization #if defined(__sun__) || defined(__HAIKU__) || defined(__NetBSD__) || defined(__DragonFly__) fd = shm_open_anon(O_RDWR | O_CREAT | O_EXCL | O_NOFOLLOW, 0600); @@ -554,7 +611,10 @@ public: return false; } - // Virtual memory initialization + return InitVirtual(); + } + + bool InitVirtual() { virtual_base = virtual_map_base = static_cast(ChooseVirtualBase(virtual_size)); if (virtual_base == MAP_FAILED) { LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); @@ -567,6 +627,180 @@ public: return true; } +#ifdef __ANDROID__ + static AHardwareBuffer_Desc MakeBlobDesc(size_t len) { + return AHardwareBuffer_Desc{ + .width = static_cast(len), + .height = 1, + .layers = 1, + .format = AHARDWAREBUFFER_FORMAT_BLOB, + .usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | + AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER, + .stride = 0, + .rfu0 = 0, + .rfu1 = 0, + }; + } + + static bool ProbeAhbBacking(PFN_AHardwareBuffer_getNativeHandle get_native_handle) { + const AHardwareBuffer_Desc desc = MakeBlobDesc(PageAlignment * 2); + AHardwareBuffer* buffer{}; + if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { + LOG_WARNING(HW_Memory, "Hardware buffer probe allocation failed"); + return false; + } + const NativeHandle* const handle = get_native_handle(buffer); + if (handle == nullptr || handle->numFds < 1) { + LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor"); + AHardwareBuffer_release(buffer); + return false; + } + const int probe_fd = handle->data[0]; + bool ok = true; + const auto try_map = [&](int prot, off_t offset, const char* what) { + if (!ok) { + return; + } + void* const ptr = mmap(nullptr, PageAlignment, prot, MAP_SHARED, probe_fd, offset); + if (ptr == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Hardware buffer backing rejects {}: {}", what, + strerror(errno)); + ok = false; + return; + } + munmap(ptr, PageAlignment); + }; + try_map(PROT_READ | PROT_WRITE, 0, "shared mappings"); + try_map(PROT_READ | PROT_WRITE, static_cast(PageAlignment), "mappings at an offset"); +#ifdef ARCHITECTURE_arm64 + try_map(PROT_READ | PROT_EXEC, 0, "executable mappings"); +#endif + AHardwareBuffer_release(buffer); + return ok; + } + + bool InitAhbBacking() { + if (!Settings::values.use_unified_memory.GetValue()) { + return false; + } + static const PFN_AHardwareBuffer_getNativeHandle get_native_handle = + ResolveGetNativeHandle(); + if (get_native_handle == nullptr) { + LOG_WARNING(HW_Memory, "AHardwareBuffer_getNativeHandle is not available"); + return false; + } + static const PFN_AHardwareBuffer_isSupported is_supported = ResolveIsSupported(); + if (is_supported == nullptr) { + LOG_WARNING(HW_Memory, "AHardwareBuffer_isSupported is not available"); + return false; + } + const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory; + if (total_physical != 0 && backing_size > total_physical / 2) { + LOG_WARNING(HW_Memory, + "Hardware buffer backing would commit {} MiB on a {} MiB system, keeping " + "lazily committed memory", + backing_size >> 20, total_physical >> 20); + return false; + } + if (!ProbeAhbBacking(get_native_handle)) { + return false; + } + const auto try_window_size = [&](size_t window_size) -> bool { + const size_t num_windows = (backing_size + window_size - 1) / window_size; + std::vector buffers; + std::vector buffer_fds; + const auto cleanup = [&] { + for (AHardwareBuffer* buffer : buffers) { + AHardwareBuffer_release(buffer); + } + buffers.clear(); + buffer_fds.clear(); + }; + for (size_t i = 0; i < num_windows; ++i) { + const size_t len = (std::min)(window_size, backing_size - i * window_size); + const AHardwareBuffer_Desc desc = MakeBlobDesc(len); + AHardwareBuffer* buffer{}; + if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { + LOG_WARNING(HW_Memory, "Hardware buffer allocation failed for window {}", i); + cleanup(); + return false; + } + buffers.push_back(buffer); + const NativeHandle* const handle = get_native_handle(buffer); + if (handle == nullptr || handle->numFds < 1) { + LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor"); + cleanup(); + return false; + } + const int buffer_fd = handle->data[0]; + const off_t buffer_len = lseek(buffer_fd, 0, SEEK_END); + if (buffer_len < static_cast(len)) { + LOG_WARNING(HW_Memory, "Hardware buffer descriptor smaller than requested"); + cleanup(); + return false; + } + buffer_fds.push_back(buffer_fd); + } + u8* const base = + static_cast(mmap(nullptr, backing_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0)); + if (base == MAP_FAILED) { + cleanup(); + return false; + } + for (size_t i = 0; i < num_windows; ++i) { + const size_t len = (std::min)(window_size, backing_size - i * window_size); + if (mmap(base + i * window_size, len, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_FIXED, buffer_fds[i], 0) == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Hardware buffer mmap failed: {}", strerror(errno)); + munmap(base, backing_size); + cleanup(); + return false; + } + } + backing_base = base; + ahb_windows = std::move(buffers); + ahb_fds = std::move(buffer_fds); + ahb_window_size = window_size; + ahb_backing = true; + committed_backing_size.store(backing_size, std::memory_order_relaxed); + LOG_INFO(HW_Memory, + "Guest memory backed by {} hardware buffer windows of {} MiB, {} MiB committed", + ahb_windows.size(), window_size >> 20, backing_size >> 20); + return true; + }; + static constexpr size_t candidate_window_sizes[] = { + 1024ULL << 20, + 512ULL << 20, + 256ULL << 20, + 128ULL << 20, + }; + for (const size_t candidate : candidate_window_sizes) { + const AHardwareBuffer_Desc window_desc = MakeBlobDesc(candidate); + if (is_supported(&window_desc) == 0) { + LOG_DEBUG(HW_Memory, "Allocator rejects {} MiB hardware buffer windows", + candidate >> 20); + continue; + } + if (try_window_size(candidate)) { + return true; + } + LOG_WARNING(HW_Memory, "Could not back guest memory with {} MiB windows", + candidate >> 20); + } + return false; + } + + std::span AhbWindows() const noexcept { + return ahb_windows; + } + + size_t AhbWindowSize() const noexcept { + return ahb_backing ? ahb_window_size : 0; + } +#endif + ~Impl() { Release(); } @@ -587,6 +821,26 @@ public: #ifdef ARCHITECTURE_arm64 if (True(perms & MemoryPermission::Execute)) prot_flags |= PROT_EXEC; +#endif +#ifdef __ANDROID__ + if (ahb_backing) { + size_t voff = virtual_offset; + size_t hoff = host_offset; + size_t remaining = length; + while (remaining > 0) { + const size_t window = hoff / ahb_window_size; + const size_t local = hoff % ahb_window_size; + const size_t chunk = (std::min)(remaining, ahb_window_size - local); + void* const ret = + mmap(virtual_base + voff, chunk, prot_flags, MAP_SHARED | MAP_FIXED, + ahb_fds[window], static_cast(local)); + ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno)); + voff += chunk; + hoff += chunk; + remaining -= chunk; + } + return; + } #endif int flags = (fd >= 0 ? MAP_SHARED : MAP_PRIVATE) | MAP_FIXED; void* ret = mmap(virtual_base + virtual_offset, length, prot_flags, flags, fd, host_offset); @@ -656,6 +910,18 @@ private: int ret = close(fd); ASSERT_MSG(ret == 0, "close failed: {}", strerror(errno)); } + +#ifdef __ANDROID__ + for (AHardwareBuffer* buffer : ahb_windows) { + AHardwareBuffer_release(buffer); + } + ahb_windows.clear(); + ahb_fds.clear(); + if (ahb_backing) { + committed_backing_size.store(0, std::memory_order_relaxed); + ahb_backing = false; + } +#endif } void AdjustMap(size_t* virtual_offset, size_t* length) { @@ -681,6 +947,13 @@ private: int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create FreeRegionManager free_manager{}; + +#ifdef __ANDROID__ + bool ahb_backing{}; + std::vector ahb_windows; + std::vector ahb_fds; + size_t ahb_window_size{}; +#endif }; #endif // ^^^ POSIX ^^^ @@ -767,6 +1040,22 @@ void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 f std::memset(backing_base + physical_offset, fill_value, length); } +std::span HostMemory::BackingHardwareBuffers() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbWindows() : std::span{}; +#else + return {}; +#endif +} + +size_t HostMemory::BackingHardwareBufferWindowSize() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbWindowSize() : 0; +#else + return 0; +#endif +} + void HostMemory::EnableDirectMappedAddress() { #if !(defined(__OPENORBIS__) || defined(__managarm__)) if (impl) { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 24f4670732..5d385247d1 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -8,12 +8,17 @@ #include #include +#include #include "common/common_funcs.h" #include "common/common_types.h" #include "common/virtual_buffer.h" +struct AHardwareBuffer; + namespace Common { +[[nodiscard]] u64 GetCommittedBackingSize() noexcept; + enum class MemoryPermission : u32 { Read = 1 << 0, Write = 1 << 1, @@ -62,6 +67,14 @@ public: return backing_base; } + [[nodiscard]] size_t BackingSize() const noexcept { + return backing_size; + } + + [[nodiscard]] std::span BackingHardwareBuffers() const noexcept; + + [[nodiscard]] size_t BackingHardwareBufferWindowSize() const noexcept; + [[nodiscard]] u8* VirtualBasePointer() noexcept { return virtual_base; } diff --git a/src/common/settings.h b/src/common/settings.h index 00ba7dddc9..0b74601a6e 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -587,6 +587,9 @@ struct Values { SwitchableSetting use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders", Category::RendererHacks}; + SwitchableSetting use_unified_memory{linkage, false, "use_unified_memory", + Category::RendererHacks}; + SwitchableSetting gpu_unswizzle_texture_size{linkage, GpuUnswizzleSize::Large, "gpu_unswizzle_texture_size", @@ -635,6 +638,16 @@ struct Values { #endif "vertex_input_dynamic_state", Category::RendererExtensions}; + SwitchableSetting dynamic_rendering{linkage, true, "dynamic_rendering", + Category::RendererExtensions}; + + SwitchableSetting workgroup_memory_explicit_layout{ + linkage, false, "workgroup_memory_explicit_layout", Category::RendererExtensions}; + + SwitchableSetting pipeline_worker_count{ + linkage, 2, 2, 8, "pipeline_worker_count", Category::RendererAdvanced, + Specialization::Scalar}; + Setting renderer_debug{linkage, false, "debug", Category::RendererDebug}; Setting renderer_shader_feedback{linkage, false, "shader_feedback", Category::RendererDebug}; diff --git a/src/common/thread.cpp b/src/common/thread.cpp index c947c88373..2b920d3c42 100644 --- a/src/common/thread.cpp +++ b/src/common/thread.cpp @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: 2013 Dolphin Emulator Project // SPDX-FileCopyrightText: 2014 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -39,6 +40,110 @@ #include #endif +#ifdef __ANDROID__ +#include +#include +#include +#include +#include + +namespace { +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_URGENT_AUDIO = -19; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_AUDIO = -16; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_URGENT_DISPLAY = -8; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_DISPLAY = -4; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_FOREGROUND = -2; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_MORE_FAVORABLE = -1; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_DEFAULT = 0; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_LESS_FAVORABLE = 1; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_BACKGROUND = 10; +[[maybe_unused]] constexpr int ANDROID_THREAD_PRIORITY_LOWEST = 19; + +constexpr size_t ANDROID_MINIMUM_PERFORMANCE_CORES = 4; + +cpu_set_t ComputePerformanceCoreMask() { + cpu_set_t mask; + CPU_ZERO(&mask); + + cpu_set_t allowed; + CPU_ZERO(&allowed); + if (sched_getaffinity(gettid(), sizeof(allowed), &allowed) != 0) { + return mask; + } + + std::vector> cores; + const int total = static_cast(std::thread::hardware_concurrency()); + for (int cpu = 0; cpu < total; ++cpu) { + if (!CPU_ISSET(cpu, &allowed)) { + continue; + } + long max_frequency = 0; + std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + + "/cpufreq/cpuinfo_max_freq"); + if (!file || !(file >> max_frequency) || max_frequency <= 0) { + CPU_ZERO(&mask); + return mask; + } + cores.emplace_back(max_frequency, cpu); + } + if (cores.empty()) { + return mask; + } + + std::sort(cores.begin(), cores.end(), + [](const auto& lhs, const auto& rhs) { return lhs.first > rhs.first; }); + + size_t taken = 0; + long cluster_frequency = cores.front().first; + for (const auto& [frequency, cpu] : cores) { + if (frequency != cluster_frequency) { + if (taken >= ANDROID_MINIMUM_PERFORMANCE_CORES) { + break; + } + cluster_frequency = frequency; + } + CPU_SET(cpu, &mask); + ++taken; + } + return mask; +} + +const cpu_set_t& PerformanceCoreMask() { + static const cpu_set_t mask = ComputePerformanceCoreMask(); + return mask; +} + +cpu_set_t ComputeEfficiencyCoreMask() { + cpu_set_t mask; + CPU_ZERO(&mask); + + const cpu_set_t& performance = PerformanceCoreMask(); + if (CPU_COUNT(&performance) == 0) { + return mask; + } + + cpu_set_t allowed; + CPU_ZERO(&allowed); + if (sched_getaffinity(gettid(), sizeof(allowed), &allowed) != 0) { + return mask; + } + + const int total = static_cast(std::thread::hardware_concurrency()); + for (int cpu = 0; cpu < total; ++cpu) { + if (CPU_ISSET(cpu, &allowed) && !CPU_ISSET(cpu, &performance)) { + CPU_SET(cpu, &mask); + } + } + return mask; +} + +const cpu_set_t& EfficiencyCoreMask() { + static const cpu_set_t mask = ComputeEfficiencyCoreMask(); + return mask; +} +} // Anonymous namespace +#endif + #include "common/cpu_features.h" #ifdef ARCHITECTURE_x86_64 #ifdef _MSC_VER @@ -78,6 +183,21 @@ void SetCurrentThreadPriority(ThreadPriority new_priority) { } }(); set_thread_priority(find_thread(NULL), priority); +#elif defined(__ANDROID__) + const int nice_value = [&]() { + switch (new_priority) { + case ThreadPriority::Low: return ANDROID_THREAD_PRIORITY_BACKGROUND; + case ThreadPriority::Normal: return ANDROID_THREAD_PRIORITY_DEFAULT; + case ThreadPriority::High: return ANDROID_THREAD_PRIORITY_DISPLAY; + case ThreadPriority::VeryHigh: return ANDROID_THREAD_PRIORITY_URGENT_DISPLAY; + case ThreadPriority::Critical: return ANDROID_THREAD_PRIORITY_AUDIO; + default: return ANDROID_THREAD_PRIORITY_DEFAULT; + } + }(); + if (setpriority(PRIO_PROCESS, static_cast(gettid()), nice_value) != 0) { + LOG_DEBUG(Common, "Could not set thread nice value to {}: {}", nice_value, + GetLastErrorMsg()); + } #else pthread_t this_thread = pthread_self(); const auto scheduling_type = SCHED_OTHER; @@ -132,29 +252,28 @@ void SetCurrentThreadName(const char* name) { #endif } -void PinCurrentThreadToPerformanceCore(size_t core_id) { - ASSERT(core_id < 4); - // If we set a flag for a CPU that doesn't exist, the thread may not be allowed to - // run in ANY processor! - auto const total_cores = std::thread::hardware_concurrency(); - if (core_id < total_cores) { +void SetCurrentThreadToPerformanceCores() { #if defined(__ANDROID__) - cpu_set_t set; - CPU_ZERO(&set); - CPU_SET(core_id, &set); - sched_setaffinity(pthread_self(), sizeof(set), &set); -#elif defined(__linux__) || defined(__FreeBSD__) - cpu_set_t set; - CPU_ZERO(&set); - CPU_SET(core_id, &set); - pthread_setaffinity_np(pthread_self(), sizeof(set), &set); -#elif defined(_WIN32) - DWORD set = 1UL << core_id; - SetThreadAffinityMask(GetCurrentThread(), set); -#else - // No pin functionality implemented -#endif + const cpu_set_t& mask = PerformanceCoreMask(); + if (CPU_COUNT(&mask) == 0) { + return; } + if (sched_setaffinity(gettid(), sizeof(mask), &mask) != 0) { + LOG_DEBUG(Common, "Could not restrict thread to performance cores: {}", GetLastErrorMsg()); + } +#endif +} + +void SetCurrentThreadToEfficiencyCores() { +#if defined(__ANDROID__) + const cpu_set_t& mask = EfficiencyCoreMask(); + if (CPU_COUNT(&mask) == 0) { + return; + } + if (sched_setaffinity(gettid(), sizeof(mask), &mask) != 0) { + LOG_DEBUG(Common, "Could not restrict thread to efficiency cores: {}", GetLastErrorMsg()); + } +#endif } #ifdef ARCHITECTURE_x86_64 diff --git a/src/common/thread.h b/src/common/thread.h index a75e342802..9e853511c3 100644 --- a/src/common/thread.h +++ b/src/common/thread.h @@ -99,8 +99,14 @@ enum class ThreadPriority : u32 { Critical = 4, }; +enum class ThreadPlacement : u32 { + Default = 0, + Background = 1, +}; + void SetCurrentThreadPriority(ThreadPriority new_priority); void SetCurrentThreadName(const char* name); -void PinCurrentThreadToPerformanceCore(size_t core_id); +void SetCurrentThreadToPerformanceCores(); +void SetCurrentThreadToEfficiencyCores(); } // namespace Common diff --git a/src/common/thread_worker.h b/src/common/thread_worker.h index 6ec9d6a2bd..3f440fcdba 100644 --- a/src/common/thread_worker.h +++ b/src/common/thread_worker.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project @@ -37,10 +37,15 @@ class StatefulThreadWorker { using StateMaker = std::conditional_t, DummyCallable>; public: - explicit StatefulThreadWorker(size_t num_workers, std::string name, StateMaker func = {}) + explicit StatefulThreadWorker(size_t num_workers, std::string name, StateMaker func = {}, + ThreadPlacement placement = ThreadPlacement::Default) : workers_queued{num_workers}, thread_name{std::move(name)} { - const auto lambda = [this, func](std::stop_token stop_token) { + const auto lambda = [this, func, placement](std::stop_token stop_token) { Common::SetCurrentThreadName(thread_name.c_str()); + if (placement == ThreadPlacement::Background) { + Common::SetCurrentThreadPriority(ThreadPriority::Low); + Common::SetCurrentThreadToEfficiencyCores(); + } { [[maybe_unused]] std::conditional_t state{func()}; while (!stop_token.stop_requested()) { diff --git a/src/core/arm/nce/arm_nce.cpp b/src/core/arm/nce/arm_nce.cpp index 63d5638fce..b181839db8 100644 --- a/src/core/arm/nce/arm_nce.cpp +++ b/src/core/arm/nce/arm_nce.cpp @@ -157,6 +157,8 @@ bool ArmNce::HandleGuestAlignmentFault(GuestContext* guest_ctx, void* raw_info, return HandleFailedGuestFault(guest_ctx, raw_info, raw_context); } +constexpr size_t NCE_WRITE_FAULT_CLUSTER_PAGES = 4; + bool ArmNce::HandleGuestAccessFault(GuestContext* guest_ctx, void* raw_info, void* raw_context) { auto* info = static_cast(raw_info); @@ -165,7 +167,7 @@ bool ArmNce::HandleGuestAccessFault(GuestContext* guest_ctx, void* raw_info, voi const Common::ProcessAddress addr = (reinterpret_cast(info->si_addr) & ~Memory::YUZU_PAGEMASK); auto& memory = guest_ctx->parent->m_running_thread->GetOwnerProcess()->GetMemory(); - if (memory.InvalidateNCE(addr, Memory::YUZU_PAGESIZE)) { + if (memory.InvalidateNCE(addr, Memory::YUZU_PAGESIZE * NCE_WRITE_FAULT_CLUSTER_PAGES)) { // We handled the access successfully and are returning to guest code. return true; } diff --git a/src/core/core.cpp b/src/core/core.cpp index 8b9133f963..f8383f01ac 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -118,6 +118,7 @@ struct System::Impl { is_multicore = Settings::values.use_multi_core.GetValue(); extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb; + unified_memory = Settings::values.use_unified_memory.GetValue(); core_timing.SetMulticore(is_multicore); core_timing.Initialize([&system]() { system.RegisterHostThread(); }); @@ -145,7 +146,8 @@ struct System::Impl { !device_memory.has_value() || is_multicore != Settings::values.use_multi_core.GetValue() || extended_memory_layout != (Settings::values.memory_layout_mode.GetValue() != - Settings::MemoryLayout::Memory_4Gb); + Settings::MemoryLayout::Memory_4Gb) || + unified_memory != Settings::values.use_unified_memory.GetValue(); if (!must_reinitialize) { return; @@ -156,6 +158,7 @@ struct System::Impl { is_multicore = Settings::values.use_multi_core.GetValue(); extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb; + unified_memory = Settings::values.use_unified_memory.GetValue(); Initialize(system); } @@ -503,6 +506,7 @@ struct System::Impl { std::atomic_bool is_powered_on{}; bool is_multicore : 1 = false; bool extended_memory_layout : 1 = false; + bool unified_memory : 1 = false; bool exit_locked : 1 = false; bool exit_requested : 1 = false; bool nvdec_active : 1 = false; diff --git a/src/core/cpu_manager.cpp b/src/core/cpu_manager.cpp index 7194fa5022..23d5ac3dfd 100644 --- a/src/core/cpu_manager.cpp +++ b/src/core/cpu_manager.cpp @@ -174,12 +174,7 @@ void CpuManager::RunThread(std::stop_token token, std::size_t core) { std::string name = is_multicore ? ("CPUCore_" + std::to_string(core)) : std::string{"CPUThread"}; Common::SetCurrentThreadName(name.c_str()); Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical); -#ifdef __ANDROID__ - // Aimed specifically for Snapdragon 8 Elite devices - // This kills performance on desktop, but boosts perf for UMA devices - // like the S8E. Mediatek and Mali likely won't suffer. - Common::PinCurrentThreadToPerformanceCore(core); -#endif + Common::SetCurrentThreadToPerformanceCores(); auto& data = core_data[core]; data.host_context = Common::Fiber::ThreadToFiber(); diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 3d97fdcc5c..98cc7babcd 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -20,6 +20,8 @@ #include "common/scratch_buffer.h" #include "common/virtual_buffer.h" +struct AHardwareBuffer; + namespace Core { constexpr size_t DEVICE_PAGEBITS = 12ULL; @@ -95,6 +97,26 @@ public: ApplyOpOnPAddr(address, buffer, operation); } + u8* GetPhysicalBase() noexcept { + return reinterpret_cast(physical_base); + } + + const u8* GetPhysicalBase() const noexcept { + return reinterpret_cast(physical_base); + } + + size_t GetPhysicalSize() const noexcept { + return physical_size; + } + + std::span GetBackingHardwareBuffers() const noexcept { + return ahb_windows; + } + + size_t GetBackingHardwareBufferWindowSize() const noexcept { + return ahb_window_size; + } + PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const { PAddr subbits = PAddr(address & page_mask); auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr; @@ -126,6 +148,10 @@ public: // New batch API to update multiple ranges with a single lock acquisition. void UpdatePagesCachedBatch(std::span> ranges, s32 delta); + void UpdateTexturePagesCount(DAddr addr, size_t size, s32 delta); + + [[nodiscard]] bool IsRegionTextureCached(DAddr addr, size_t size) const noexcept; + private: struct TranslationEntry { DAddr guest_page{}; @@ -171,6 +197,9 @@ private: std::unique_ptr> impl; const uintptr_t physical_base; + const size_t physical_size; + const std::span ahb_windows; + const size_t ahb_window_size; DeviceInterface* device_inter; struct TrackedEntry { @@ -234,6 +263,7 @@ private: (1ULL << (device_virtual_bits - page_bits)) / subentries; using CachedPages = std::array; std::unique_ptr cached_pages; + std::unique_ptr texture_cached_pages; Common::RangeMutex counter_guard; std::mutex mapping_guard; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 43a480cfef..8c247e228b 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,12 +171,16 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , physical_size{device_memory_.buffer.BackingSize()} + , ahb_windows{device_memory_.buffer.BackingHardwareBuffers()} + , ahb_window_size{device_memory_.buffer.BackingHardwareBufferWindowSize()} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); + texture_cached_pages = std::make_unique(); const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; for (size_t i = 0; i < total_virtual; i++) { @@ -625,6 +629,28 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size UpdatePagesCachedCountNoLock(addr, size, delta); } +template +void DeviceMemoryManager::UpdateTexturePagesCount(DAddr addr, size_t size, s32 delta) { + Common::ScopedRangeLock lk(counter_guard, addr, size); + const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); + for (size_t page = addr >> Memory::YUZU_PAGEBITS; page != page_end; ++page) { + CounterAtomicType& count = texture_cached_pages->at(page >> subentries_shift).Count(page); + count.fetch_add(static_cast(delta), std::memory_order_release); + } +} + +template +bool DeviceMemoryManager::IsRegionTextureCached(DAddr addr, size_t size) const noexcept { + const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); + for (size_t page = addr >> Memory::YUZU_PAGEBITS; page != page_end; ++page) { + if (texture_cached_pages->at(page >> subentries_shift).Count(page).load( + std::memory_order_acquire) != 0) { + return true; + } + } + return false; +} + template void DeviceMemoryManager::UpdatePagesCachedBatch(std::span> ranges, s32 delta) { if (ranges.empty()) { diff --git a/src/qt_common/config/shared_translation.cpp b/src/qt_common/config/shared_translation.cpp index 893e4a0854..a81e633597 100644 --- a/src/qt_common/config/shared_translation.cpp +++ b/src/qt_common/config/shared_translation.cpp @@ -230,6 +230,11 @@ std::unique_ptr InitializeTranslations(QObject* parent) { tr("Preserves GPU-modified data by reading it back before uploading.\nSome games require this to render certain effects properly.")); INSERT(Settings, use_asynchronous_shaders, tr("Enable asynchronous shader compilation"), tr("May reduce shader stutter.")); + INSERT(Settings, use_unified_memory, tr("Enable unified memory access (UMA)"), + tr("Lets the GPU write buffer readbacks directly into guest memory.")); + INSERT(Settings, pipeline_worker_count, tr("Pipeline Worker Threads"), + tr("Number of threads used to build Vulkan pipelines.\n" + "Higher values speed up compilation at the cost of heat and power.")); INSERT(Settings, fast_gpu_time, tr("Fast GPU Time"), tr("Overclocks the emulated GPU to increase dynamic resolution and render " "distance.\nUse 256 for maximal performance and 512 for maximal graphics fidelity.")); @@ -287,6 +292,12 @@ std::unique_ptr InitializeTranslations(QObject* parent) { INSERT(Settings, vertex_input_dynamic_state, tr("Vertex Input Dynamic State"), tr("Enables vertex input dynamic state feature for better quality and performance.")); + INSERT(Settings, dynamic_rendering, tr("Dynamic Rendering"), + tr("Renders without render pass and framebuffer objects.\n" + "Results vary by driver: some gain performance, others lose it.")); + + INSERT(Settings, workgroup_memory_explicit_layout, QString(), QString()); + INSERT( Settings, sample_shading, tr("Sample Shading"), tr("Allows the fragment shader to execute per sample in a multi-sampled fragment " diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h index 6572db86dc..6f522ab5c2 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h +++ b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -665,6 +665,8 @@ void EmitShuffleDown(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU3 const IR::Value& clamp, const IR::Value& segmentation_mask); void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 index, const IR::Value& clamp, const IR::Value& segmentation_mask); +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 lane); +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 direction); void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a, ScalarF32 op_b, ScalarU32 swizzle); void EmitDPdxFine(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a); diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp index fb6a597d4e..7c07ec3247 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -97,6 +100,24 @@ void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, Sca Shuffle(ctx, inst, value, index, clamp, segmentation_mask, "XOR"); } +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 lane) { + const Register ret{ctx.reg_alloc.Define(inst)}; + ctx.Add("AND.U RC.x,{}.threadid,~3;" + "AND.U RC.y,{},3;" + "OR.U RC.x,RC.x,RC.y;" + "SHFIDX.U {},{},RC.x,0x1C03;" + "MOV.U {}.x,{}.y;", + ctx.stage_name, lane, ret, value, ret, ret); +} + +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 direction) { + const Register ret{ctx.reg_alloc.Define(inst)}; + ctx.Add("ADD.U RC.x,{},1;" + "SHFXOR.U {},{},RC.x,0x1C03;" + "MOV.U {}.x,{}.y;", + direction, ret, value, ret, ret); +} + void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a, ScalarF32 op_b, ScalarU32 swizzle) { const auto ret{ctx.reg_alloc.Define(inst)}; diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h index d508b84cec..96aac502d5 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h +++ b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -743,6 +743,10 @@ void EmitShuffleDown(EmitContext& ctx, IR::Inst& inst, std::string_view value, void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, std::string_view value, std::string_view index, std::string_view clamp, std::string_view segmentation_mask); +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view lane); +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view direction); void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, std::string_view op_a, std::string_view op_b, std::string_view swizzle); void EmitDPdxFine(EmitContext& ctx, IR::Inst& inst, std::string_view op_a); diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp b/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp index f9be5de1c0..b2f65550fb 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp +++ b/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -200,6 +203,18 @@ void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, std::string_view val ctx.AddU32("{}=shfl_in_bounds?shfl_result:{};", inst, value); } +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view lane) { + const auto src_thread_id{fmt::format("(({}&~3)|({}& 3))", THREAD_ID, lane)}; + ctx.AddU32("{}=readInvocationARB({},{});", inst, value, src_thread_id); +} + +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view direction) { + const auto src_thread_id{fmt::format("({}^({}+1))", THREAD_ID, direction)}; + ctx.AddU32("{}=readInvocationARB({},{});", inst, value, src_thread_id); +} + void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, std::string_view op_a, std::string_view op_b, std::string_view swizzle) { const auto mask{fmt::format("({}>>((gl_SubGroupInvocationARB&3)<<1))&3", swizzle)}; diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp index 53453e6ab7..d9e77db169 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp @@ -322,6 +322,11 @@ void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) { if (ctx.runtime_info.force_early_z) { ctx.AddExecutionMode(main, spv::ExecutionMode::EarlyFragmentTests); } + if (ctx.profile.support_shader_quad_control && program.info.uses_quad_shuffles) { + ctx.AddExtension("SPV_KHR_quad_control"); + ctx.AddCapability(spv::Capability::QuadControlKHR); + ctx.AddExecutionMode(main, spv::ExecutionMode::RequireFullQuadsKHR); + } break; default: throw NotImplementedException("Stage {}", program.stage); @@ -443,6 +448,12 @@ void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ct ctx.AddCapability(spv::Capability::GroupNonUniformVote); } } + if (info.uses_quad_shuffles) { + if (profile.support_quad_shuffles) { + ctx.AddCapability(spv::Capability::GroupNonUniformQuad); + } + ctx.AddCapability(spv::Capability::GroupNonUniformShuffle); + } if (info.uses_int64_bit_atomics && profile.support_int64_atomics) { ctx.AddCapability(spv::Capability::Int64Atomics); } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h index 171977c85c..165ec3fe84 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -622,6 +622,8 @@ Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clam Id segmentation_mask); Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp, Id segmentation_mask); +Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane); +Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction); Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle); Id EmitDPdxFine(EmitContext& ctx, Id op_a); Id EmitDPdyFine(EmitContext& ctx, Id op_a); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp index 242426ec08..ffd5d9c29f 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp @@ -260,6 +260,21 @@ Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id return SelectValue(ctx, in_range, value, src_thread_id); } +Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) { + if (ctx.profile.support_quad_shuffles) { + return ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, lane); + } + const Id base{ctx.OpBitwiseAnd(ctx.U32[1], GetThreadId(ctx), ctx.Const(~3u))}; + const Id local_lane{ctx.OpBitwiseAnd(ctx.U32[1], lane, ctx.Const(3u))}; + const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], base, local_lane)}; + return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id); +} + +Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction) { + const Id xor_mask{ctx.OpIAdd(ctx.U32[1], direction, ctx.Const(1u))}; + return ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, xor_mask); +} + Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle) { const Id three{ctx.Const(3U)}; Id mask{GetThreadId(ctx)}; diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp index 171161ffd1..95815d7d51 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp +++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -2100,6 +2100,14 @@ U32 IREmitter::ShuffleButterfly(const IR::U32& value, const IR::U32& index, cons return Inst(Opcode::ShuffleButterfly, value, index, clamp, seg_mask); } +U32 IREmitter::QuadBroadcast(const IR::U32& value, const IR::U32& lane) { + return Inst(Opcode::QuadBroadcast, value, lane); +} + +U32 IREmitter::QuadSwap(const IR::U32& value, const IR::U32& direction) { + return Inst(Opcode::QuadSwap, value, direction); +} + F32 IREmitter::FSwizzleAdd(const F32& a, const F32& b, const U32& swizzle, FpControl control) { return Inst(Opcode::FSwizzleAdd, Flags{control}, a, b, swizzle); } diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h index 725eac3951..f22f0f75e2 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.h +++ b/src/shader_recompiler/frontend/ir/ir_emitter.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -394,6 +394,8 @@ public: const IR::U32& seg_mask); [[nodiscard]] U32 ShuffleButterfly(const IR::U32& value, const IR::U32& index, const IR::U32& clamp, const IR::U32& seg_mask); + [[nodiscard]] U32 QuadBroadcast(const IR::U32& value, const IR::U32& lane); + [[nodiscard]] U32 QuadSwap(const IR::U32& value, const IR::U32& direction); [[nodiscard]] F32 FSwizzleAdd(const F32& a, const F32& b, const U32& swizzle, FpControl control = {}); diff --git a/src/shader_recompiler/frontend/ir/opcodes.cpp b/src/shader_recompiler/frontend/ir/opcodes.cpp index 7fb6694a9b..6918d4c15d 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.cpp +++ b/src/shader_recompiler/frontend/ir/opcodes.cpp @@ -10,7 +10,7 @@ namespace Shader::IR { namespace Detail { -OpcodeMeta META_TABLE[532] = { +OpcodeMeta META_TABLE[] = { #define OPCODE(name_token, type_token, ...) \ { \ .name{#name_token}, \ @@ -21,7 +21,7 @@ OpcodeMeta META_TABLE[532] = { #undef OPCODE }; -u8 NUM_ARGS[532] = { +u8 NUM_ARGS[] = { #define OPCODE(name_token, type_token, ...) u8(CalculateNumArgsOf(Opcode::name_token)), #include "opcodes.inc" #undef OPCODE diff --git a/src/shader_recompiler/frontend/ir/opcodes.h b/src/shader_recompiler/frontend/ir/opcodes.h index 5549edc9d7..6f5c09dfcb 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.h +++ b/src/shader_recompiler/frontend/ir/opcodes.h @@ -57,12 +57,12 @@ static constexpr Type F64x2{Type::F64x2}; static constexpr Type F64x3{Type::F64x3}; static constexpr Type F64x4{Type::F64x4}; -extern OpcodeMeta META_TABLE[532]; +extern OpcodeMeta META_TABLE[]; constexpr size_t CalculateNumArgsOf(Opcode op) noexcept { const auto& arg_types = META_TABLE[size_t(op)].arg_types; return size_t(std::distance(arg_types.begin(), std::ranges::find(arg_types, Type::Void))); } -extern u8 NUM_ARGS[532]; +extern u8 NUM_ARGS[]; } // namespace Detail /// Get return type of an opcode diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc index 960009d89e..fee0afaf33 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.inc +++ b/src/shader_recompiler/frontend/ir/opcodes.inc @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -579,6 +582,8 @@ OPCODE(ShuffleIndex, U32, U32, OPCODE(ShuffleUp, U32, U32, U32, U32, U32, ) OPCODE(ShuffleDown, U32, U32, U32, U32, U32, ) OPCODE(ShuffleButterfly, U32, U32, U32, U32, U32, ) +OPCODE(QuadBroadcast, U32, U32, U32, ) +OPCODE(QuadSwap, U32, U32, U32, ) OPCODE(FSwizzleAdd, F32, F32, F32, U32, ) OPCODE(DPdxFine, F32, F32, ) OPCODE(DPdyFine, F32, F32, ) diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp index e6345de65e..38cdc1a2d1 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -39,17 +39,6 @@ IR::U32 scaleIndex(IR::IREmitter& ir, IR::U32 index, Shift shift) { } } -IR::U32 skewBytes(IR::IREmitter& ir, SZ sizeRead) { - const IR::U32 lane = ir.LaneId(); - switch (sizeRead) { - case SZ::U8: return lane; - case SZ::U16: return ir.ShiftLeftLogical(lane, ir.Imm32(1)); - case SZ::U32: - case SZ::F32: return ir.ShiftLeftLogical(lane, ir.Imm32(2)); - default: UNREACHABLE(); - } -} - } // Anonymous namespace void TranslatorVisitor::ISBERD(u64 insn) { @@ -68,61 +57,41 @@ void TranslatorVisitor::ISBERD(u64 insn) { BitField<47, 2, Shift> shift; } const isberd{insn}; - IR::U32 index{}; - if (isberd.src_reg_num.Value() == 0xFF) { - index = ir.Imm32(isberd.imm.Value()); - } else { - const IR::U32 scaledIndex = scaleIndex(ir, X(isberd.src_reg.Value()), isberd.shift.Value()); - index = ir.IAdd(scaledIndex, ir.Imm32(isberd.imm.Value())); + if (isberd.skew != 0) { + throw NotImplementedException("ISBERD SKEW"); + } + if (isberd.o != 0) { + throw NotImplementedException("ISBERD O"); + } + if (isberd.sz.Value() > SZ::F32) { + throw NotImplementedException("ISBERD SZ {}", + static_cast(isberd.sz.Value())); + } + if (isberd.shift.Value() > Shift::B32) { + throw NotImplementedException("ISBERD Shift {}", + static_cast(isberd.shift.Value())); } - if (isberd.o.Value()) { - if (isberd.skew.Value()) { - index = ir.IAdd(index, skewBytes(ir, isberd.sz.Value())); + switch (isberd.mode.Value()) { + case Mode::Default: + X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); + return; + case Mode::Attr: { + IR::U32 offset{}; + if (isberd.src_reg_num.Value() == 0xFF) { + offset = ir.Imm32(isberd.imm.Value()); + } else { + const IR::U32 index{ + scaleIndex(ir, X(isberd.src_reg.Value()), isberd.shift.Value())}; + offset = ir.IAdd(index, ir.Imm32(isberd.imm.Value())); } - - const IR::U64 index64 = ir.UConvert(64, index); - IR::U32 globalLoaded{}; - switch (isberd.sz.Value()) { - case SZ::U8: globalLoaded = ir.LoadGlobalU8 (index64); break; - case SZ::U16: globalLoaded = ir.LoadGlobalU16(index64); break; - case SZ::U32: - case SZ::F32: globalLoaded = ir.LoadGlobal32(index64); break; - default: UNREACHABLE(); - } - X(isberd.dest_reg.Value(), globalLoaded); - + X(isberd.dest_reg.Value(), ir.BitCast(ir.GetAttributeIndexed(offset))); return; } - - if (isberd.mode.Value() != Mode::Default) { - if (isberd.skew.Value()) { - index = ir.IAdd(index, skewBytes(ir, SZ::U32)); - } - - IR::F32 float_index{}; - switch (isberd.mode.Value()) { - case Mode::Patch: float_index = ir.GetPatch(index.Patch()); - break; - case Mode::Prim: float_index = ir.GetAttribute(index.Attribute()); - break; - case Mode::Attr: float_index = ir.GetAttributeIndexed(index); - break; - default: UNREACHABLE(); - } - X(isberd.dest_reg.Value(), ir.BitCast(float_index)); - - return; + default: + throw NotImplementedException("ISBERD Mode {}", + static_cast(isberd.mode.Value())); } - - if (isberd.skew.Value()) { - X(isberd.dest_reg.Value(), ir.IAdd(X(isberd.src_reg.Value()), ir.LaneId())); - - return; - } - - // Fallback copy - X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); } } // namespace Shader::Maxwell diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp index 972eec8276..2a19237ab2 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -36,7 +36,10 @@ enum class ShuffleMode : u64 { } } -void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32& mask) { +constexpr u32 QUAD_MASK = (28u << 8) | 3u; + +void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32& mask, + bool index_is_imm, u32 index_imm, bool mask_is_imm, u32 mask_imm) { union { u64 insn; BitField<0, 8, IR::Reg> dest_reg; @@ -45,6 +48,21 @@ void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32 BitField<48, 3, IR::Pred> pred; } const shfl{insn}; + const bool is_quad_candidate{mask_is_imm && mask_imm == QUAD_MASK && index_is_imm && + v.env.ShaderStage() == Stage::Fragment}; + if (is_quad_candidate) { + if (shfl.mode == ShuffleMode::IDX && index_imm <= 3) { + v.X(shfl.dest_reg, v.ir.QuadBroadcast(v.X(shfl.src_reg), v.ir.Imm32(index_imm))); + v.ir.SetPred(shfl.pred, v.ir.Imm1(true)); + return; + } + if (shfl.mode == ShuffleMode::BFLY && index_imm >= 1 && index_imm <= 3) { + v.X(shfl.dest_reg, v.ir.QuadSwap(v.X(shfl.src_reg), v.ir.Imm32(index_imm - 1))); + v.ir.SetPred(shfl.pred, v.ir.Imm1(true)); + return; + } + } + const IR::U32 result{ShuffleOperation(v.ir, v.X(shfl.src_reg), index, mask, shfl.mode)}; v.ir.SetPred(shfl.pred, v.ir.GetInBoundsFromOp(result)); v.X(shfl.dest_reg, result); @@ -59,11 +77,14 @@ void TranslatorVisitor::SHFL(u64 insn) { BitField<29, 1, u64> src_b_flag; BitField<34, 13, u64> src_b_imm; } const flags{insn}; - const IR::U32 src_a{flags.src_a_flag != 0 ? ir.Imm32(static_cast(flags.src_a_imm)) - : GetReg20(insn)}; - const IR::U32 src_b{flags.src_b_flag != 0 ? ir.Imm32(static_cast(flags.src_b_imm)) - : GetReg39(insn)}; - Shuffle(*this, insn, src_a, src_b); + const bool index_is_imm{flags.src_a_flag != 0}; + const bool mask_is_imm{flags.src_b_flag != 0}; + const IR::U32 src_a{index_is_imm ? ir.Imm32(static_cast(flags.src_a_imm)) + : GetReg20(insn)}; + const IR::U32 src_b{mask_is_imm ? ir.Imm32(static_cast(flags.src_b_imm)) + : GetReg39(insn)}; + Shuffle(*this, insn, src_a, src_b, index_is_imm, static_cast(flags.src_a_imm), + mask_is_imm, static_cast(flags.src_b_imm)); } } // namespace Shader::Maxwell diff --git a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp index 5089f3a101..9010c2bf68 100644 --- a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp +++ b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp @@ -498,6 +498,10 @@ void VisitUsages(Info& info, IR::Inst& inst) { case IR::Opcode::ShuffleButterfly: info.uses_subgroup_shuffles = true; break; + case IR::Opcode::QuadBroadcast: + case IR::Opcode::QuadSwap: + info.uses_quad_shuffles = true; + break; case IR::Opcode::GetCbufU8: case IR::Opcode::GetCbufS8: case IR::Opcode::GetCbufU16: diff --git a/src/shader_recompiler/profile.h b/src/shader_recompiler/profile.h index eb7b2a02b2..bd8db0d6f4 100644 --- a/src/shader_recompiler/profile.h +++ b/src/shader_recompiler/profile.h @@ -37,6 +37,8 @@ struct Profile { bool support_explicit_workgroup_layout{}; bool support_workgroup_layout_8bit_access{}; bool support_workgroup_layout_16bit_access{}; + bool support_shader_quad_control{}; + bool support_quad_shuffles{}; bool support_vote{}; u32 supported_subgroup_stages{0x7F}; bool support_viewport_index_layer_non_geometry{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 0c11a0d72a..71309e040e 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -252,6 +252,7 @@ struct Info { bool uses_is_helper_invocation{}; bool uses_subgroup_invocation_id{}; bool uses_subgroup_shuffles{}; + bool uses_quad_shuffles{}; std::array uses_patches{}; std::array interpolation{}; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 5c02b5ed90..999955baf9 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -33,7 +33,7 @@ add_library(video_core STATIC control/channel_state_cache.h control/scheduler.cpp control/scheduler.h - delayed_destruction_ring.h + deferred_destruction_queue.h dirty_flags.cpp dirty_flags.h dma_pusher.cpp @@ -158,6 +158,8 @@ add_library(video_core STATIC renderer_vulkan/vk_compute_pass.h renderer_vulkan/vk_compute_pipeline.cpp renderer_vulkan/vk_compute_pipeline.h + renderer_vulkan/vk_descriptor_buffer.cpp + renderer_vulkan/vk_descriptor_buffer.h renderer_vulkan/vk_descriptor_pool.cpp renderer_vulkan/vk_descriptor_pool.h renderer_vulkan/vk_fence_manager.cpp diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e40aad1fb5..ca6d65c2b0 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include @@ -31,44 +32,84 @@ BufferCache

::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R immediately_free = (Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive); #endif if (!runtime.CanReportMemoryUsage()) { - minimum_memory = DEFAULT_EXPECTED_MEMORY; - critical_memory = DEFAULT_CRITICAL_MEMORY; + memory_budget = FALLBACK_MEMORY_BUDGET; return; } - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - minimum_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); + memory_budget = runtime.GetDeviceLocalMemory(); } template BufferCache

::~BufferCache() = default; template -void BufferCache

::RunGarbageCollector() { - const bool aggressive_gc = total_used_memory >= critical_memory; - const u64 ticks_to_destroy = aggressive_gc ? 60 : 120; - int num_iterations = aggressive_gc ? 64 : 32; - const auto clean_up = [this, &num_iterations](BufferId buffer_id) { - if (num_iterations == 0) { +u64 BufferCache

::DeviceUsage(bool force_refresh) { + if (!runtime.CanReportMemoryUsage()) { + return total_used_memory; + } + if (force_refresh || usage_refresh_countdown == 0) { + cached_device_usage = runtime.GetDeviceAllocationUsage(); + usage_refresh_countdown = USAGE_REFRESH_INTERVAL; + } else { + --usage_refresh_countdown; + } + return cached_device_usage; +} + +template +u64 BufferCache

::ReclaimMemory(u64 target_bytes, bool allow_download) { + if (target_bytes == 0 || in_reclaim) { + return 0; + } + in_reclaim = true; + u64 freed = 0; + const auto clean_up = [&](BufferId buffer_id) { + if (freed >= target_bytes) { return true; } - --num_iterations; auto& buffer = slot_buffers[buffer_id]; + if (!allow_download && IsRegionGpuModified(buffer.CpuAddr(), buffer.SizeBytes())) { + return false; + } + const u64 buffer_bytes = Common::AlignUp(buffer.SizeBytes(), 1024); DownloadBufferMemory(buffer); DeleteBuffer(buffer_id); + freed += buffer_bytes; return false; }; - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up); + const u64 cold_tick = + frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0; + lru_cache.ForEachItemBelow(cold_tick, clean_up); + if (freed == 0) { + lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, clean_up); + } + in_reclaim = false; + usage_refresh_countdown = 0; + reclaim_stalled = freed == 0; + if (freed > 0) { + reclaim_wait_sync_point = runtime.CurrentSyncPoint(); + } + return freed; +} + +template +void BufferCache

::EnsureHeadroom(bool allow_download) { + if (reclaim_stalled) { + return; + } + if (runtime.CompletedSyncPoint() < reclaim_wait_sync_point) { + return; + } + const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0; + const u64 usage = DeviceUsage(false); + if (usage <= limit) { + return; + } + const u64 target = (limit / 100) * RECLAIM_TARGET_PERCENT; + const u64 excess = usage - target; + const u64 usage_mib = (std::max)(usage >> 20, u64{1}); + const u64 share = (((excess >> 20) * (total_used_memory >> 20)) / usage_mib) << 20; + ReclaimMemory((std::min)(share, total_used_memory), allow_download); } template @@ -96,15 +137,11 @@ void BufferCache

::TickFrame() { const bool skip_preferred = hits * 256 < shots * 251; channel_state->uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0; - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); - } - if (total_used_memory >= minimum_memory) { - RunGarbageCollector(); - } + usage_refresh_countdown = 0; + reclaim_stalled = false; + EnsureHeadroom(true); ++frame_tick; - delayed_destruction_ring.Tick(); + sentenced_buffers.Reclaim(runtime.CompletedSyncPoint()); for (auto& buffer : async_buffers_death_ring) { runtime.FreeDeferredStagingBuffer(buffer); @@ -175,9 +212,71 @@ std::optional BufferCache

::GetFlushArea(DA template void BufferCache

::DownloadMemory(DAddr device_addr, u64 size) { - ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) { - DownloadBufferMemory(buffer, device_addr, size); + if constexpr (!USE_MEMORY_MAPS) { + std::scoped_lock lock{mutex}; + ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) { + DownloadBufferMemory(buffer, device_addr, size); + }); + return; + } + + boost::container::small_vector, 8> downloads; + u64 total_size_bytes = 0; + u64 largest_copy = 0; + + std::unique_lock lock{mutex}; + ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) { + memory_tracker.ForEachDownloadRangeAndClear( + device_addr, size, [&](u64 device_addr_out, u64 range_size) { + const DAddr buffer_addr = buffer.CpuAddr(); + const auto add_download = [&](DAddr start, DAddr end) { + const u64 new_offset = start - buffer_addr; + const u64 new_size = end - start; + downloads.push_back({ + BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }, + buffer_id, + }); + constexpr u64 align = 64ULL; + constexpr u64 mask = ~(align - 1ULL); + total_size_bytes += (new_size + align - 1) & mask; + largest_copy = (std::max)(largest_copy, new_size); + }; + gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, add_download); + ClearDownload(device_addr_out, range_size); + gpu_modified_ranges.Subtract(device_addr_out, range_size); + }); }); + if (total_size_bytes == 0) { + return; + } + + auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); + boost::container::small_vector writebacks; + runtime.PreCopyBarrier(); + for (auto& [copy, buffer_id] : downloads) { + copy.dst_offset += download_staging.offset; + Buffer& buffer = slot_buffers[buffer_id]; + buffer.MarkUsage(copy.src_offset, copy.size); + const std::array copies{copy}; + runtime.CopyBuffer(download_staging.buffer, buffer, copies, false); + BufferCopy writeback{copy}; + writeback.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; + writebacks.push_back(writeback); + } + runtime.PostCopyBarrier(); + lock.unlock(); + + runtime.Finish(); + const u8* const base = download_staging.mapped_span.data(); + for (const BufferCopy& writeback : writebacks) { + const u64 staging_offset = writeback.dst_offset - download_staging.offset; + device_memory.WriteBlockUnsafe(static_cast(writeback.src_offset), + base + staging_offset, writeback.size); + } } template @@ -214,7 +313,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am auto& src_buffer = slot_buffers[buffer_a]; auto& dest_buffer = slot_buffers[buffer_b]; SynchronizeBuffer(src_buffer, *cpu_src_address, static_cast(amount)); - SynchronizeBuffer(dest_buffer, *cpu_dest_address, static_cast(amount)); + memory_tracker.UnmarkRegionAsCpuModified(*cpu_dest_address, static_cast(amount)); std::array copies{BufferCopy{ .src_offset = src_buffer.Offset(*cpu_src_address), .dst_offset = dest_buffer.Offset(*cpu_dest_address), @@ -673,32 +772,44 @@ void BufferCache

::PopAsyncFlushes() { template void BufferCache

::PopAsyncBuffers() { - if (async_buffers.empty()) { - return; - } - if (!async_buffers.front().has_value()) { + struct Writeback { + DAddr addr; + const u8* src; + u64 size; + }; + boost::container::small_vector writebacks; + { + std::scoped_lock lock{mutex}; + if (async_buffers.empty()) { + return; + } + if (!async_buffers.front().has_value()) { + async_buffers.pop_front(); + return; + } + auto& downloads = pending_downloads.front(); + auto& async_buffer = async_buffers.front(); + const u8* base = async_buffer->mapped_span.data(); + const size_t base_offset = async_buffer->offset; + for (const auto& copy : downloads) { + const DAddr device_addr = static_cast(copy.src_offset); + const u64 dst_offset = copy.dst_offset - base_offset; + const u8* read_mapped_memory = base + dst_offset; + async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) { + writebacks.push_back( + {start, &read_mapped_memory[start - device_addr], end - start}); + }); + async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + }); + } + async_buffers_death_ring.emplace_back(*async_buffer); async_buffers.pop_front(); - return; + pending_downloads.pop_front(); } - auto& downloads = pending_downloads.front(); - auto& async_buffer = async_buffers.front(); - u8* base = async_buffer->mapped_span.data(); - const size_t base_offset = async_buffer->offset; - for (const auto& copy : downloads) { - const DAddr device_addr = static_cast(copy.src_offset); - const u64 dst_offset = copy.dst_offset - base_offset; - const u8* read_mapped_memory = base + dst_offset; - async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) { - device_memory.WriteBlockUnsafe(start, &read_mapped_memory[start - device_addr], - end - start); - }); - async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { - gpu_modified_ranges.Subtract(start, end - start); - }); + for (const auto& wb : writebacks) { + device_memory.WriteBlockUnsafe(wb.addr, wb.src, wb.size); } - async_buffers_death_ring.emplace_back(*async_buffer); - async_buffers.pop_front(); - pending_downloads.pop_front(); } template @@ -809,46 +920,46 @@ void BufferCache

::BindHostVertexBuffers() { if (use_optimized_vertex_buffers) { auto& flags = maxwell3d->dirty.flags; - u32 enabled_mask = enabled_vertex_buffers_mask; - HostBindings bindings{}; - u32 last_index = (std::numeric_limits::max)(); - const auto flush_bindings = [&]() { - if (bindings.buffers.empty()) { - return; - } - bindings.max_index = bindings.min_index + static_cast(bindings.buffers.size()); - runtime.BindVertexBuffers(bindings); - bindings = HostBindings{}; - last_index = (std::numeric_limits::max)(); - }; - while (enabled_mask != 0) { - const u32 index = std::countr_zero(enabled_mask); - enabled_mask &= (enabled_mask - 1); + const u32 enabled_mask = enabled_vertex_buffers_mask; + bool any_dirty = false; + u32 pending_mask = enabled_mask; + while (pending_mask != 0) { + const u32 index = std::countr_zero(pending_mask); + pending_mask &= (pending_mask - 1); const Binding& binding = VertexBufferSlot(index); Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); SynchronizeBuffer(buffer, binding.device_addr, binding.size); - if (!flags[Dirty::VertexBuffer0 + index]) { - flush_bindings(); - continue; - } + any_dirty |= flags[Dirty::VertexBuffer0 + index]; + } + if (enabled_mask == 0 || !any_dirty) { + return; + } + const u32 min_index = static_cast(std::countr_zero(enabled_mask)); + const u32 max_index = 32u - static_cast(std::countl_zero(enabled_mask)); + HostBindings bindings{}; + bindings.min_index = min_index; + bindings.max_index = max_index; + for (u32 index = min_index; index < max_index; ++index) { flags[Dirty::VertexBuffer0 + index] = false; const u32 stride = maxwell3d->regs.vertex_streams[index].stride; + if ((enabled_mask & (1u << index)) == 0) { + bindings.buffers.push_back(&slot_buffers[NULL_BUFFER_ID]); + bindings.offsets.push_back(0); + bindings.sizes.push_back(0); + bindings.strides.push_back(stride); + continue; + } + const Binding& binding = VertexBufferSlot(index); + Buffer& buffer = slot_buffers[binding.buffer_id]; const u32 offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, binding.size); - if (!bindings.buffers.empty() && index != last_index + 1) { - flush_bindings(); - } - if (bindings.buffers.empty()) { - bindings.min_index = index; - } bindings.buffers.push_back(&buffer); bindings.offsets.push_back(offset); bindings.sizes.push_back(binding.size); bindings.strides.push_back(stride); - last_index = index; } - flush_bindings(); + runtime.BindVertexBuffers(bindings); } else { HostBindings host_bindings; bool any_valid{false}; @@ -921,7 +1032,6 @@ void BufferCache

::BindHostGraphicsUniformBuffers(size_t stage) { template void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { - ++channel_state->uniform_cache_shots[0]; const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); @@ -940,8 +1050,12 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 return alignment > 1 && (offset % alignment) != 0; } }(); + + const bool cached_buffer_is_current = + has_host_buffer && !memory_tracker.IsRegionCpuModified(device_addr, size); const bool use_fast_buffer = needs_alignment_stream - || (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size + || (has_host_buffer && !cached_buffer_is_current + && size <= channel_state->uniform_buffer_skip_cache_size && !memory_tracker.IsRegionGpuModified(device_addr, size)); if (use_fast_buffer) { if constexpr (IS_OPENGL) { @@ -968,7 +1082,7 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 device_memory.ReadBlockUnsafe(device_addr, span.data(), size); return; } - // Classic cached path + ++channel_state->uniform_cache_shots[0]; if (SynchronizeBuffer(buffer, device_addr, size)) { ++channel_state->uniform_cache_hits[0]; } @@ -1576,6 +1690,7 @@ void BufferCache

::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, template BufferId BufferCache

::CreateBuffer(DAddr device_addr, u32 wanted_size) { + EnsureHeadroom(false); DAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE); device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE); wanted_size = static_cast(device_addr_end - device_addr); @@ -1613,7 +1728,7 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { total_used_memory += Common::AlignUp(size, 1024); buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick)); } else { - total_used_memory -= Common::AlignUp(size, 1024); + total_used_memory -= std::min(total_used_memory, Common::AlignUp(size, 1024)); lru_cache.Free(buffer.getLRUID()); } const DAddr device_addr_begin = buffer.CpuAddr(); @@ -1699,6 +1814,80 @@ void BufferCache

::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer, } } +template +bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, + [[maybe_unused]] std::span copies) { + if constexpr (USE_UNIFIED_MEMORY) { + const u8* const physical_base = device_memory.GetPhysicalBase(); + const u64 unified_size = runtime.UnifiedMemorySize(); + const u64 window_size = runtime.UnifiedMemoryWindowSize(); + if (window_size == 0) { + return false; + } + boost::container::small_vector window_ids; + boost::container::small_vector, 4> groups; + const auto group_for = [&](u64 window) -> boost::container::small_vector& { + for (size_t i = 0; i < window_ids.size(); ++i) { + if (window_ids[i] == window) { + return groups[i]; + } + } + window_ids.push_back(window); + groups.emplace_back(); + return groups.back(); + }; + for (const BufferCopy& copy : copies) { + const DAddr device_addr = buffer.CpuAddr() + copy.src_offset; + u64 downloaded = 0; + while (downloaded < copy.size) { + const DAddr page_addr = device_addr + downloaded; + const u8* const ptr = device_memory.GetPointer(page_addr); + if (ptr == nullptr) { + return false; + } + const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK; + u64 chunk = (std::min)(copy.size - downloaded, + static_cast(Core::DEVICE_PAGESIZE) - page_offset); + const u64 phys_offset = static_cast(ptr - physical_base); + if (phys_offset + chunk > unified_size) { + return false; + } + const u64 window = phys_offset / window_size; + const u64 local_offset = phys_offset % window_size; + chunk = (std::min)(chunk, window_size - local_offset); + auto& group = group_for(window); + if (!group.empty()) { + BufferCopy& last = group.back(); + if (last.src_offset + last.size == copy.src_offset + downloaded && + last.dst_offset + last.size == local_offset) { + last.size += chunk; + downloaded += chunk; + continue; + } + } + group.push_back(BufferCopy{ + .src_offset = copy.src_offset + downloaded, + .dst_offset = local_offset, + .size = chunk, + }); + downloaded += chunk; + } + } + for (const BufferCopy& copy : copies) { + buffer.MarkUsage(copy.src_offset, copy.size); + } + for (size_t i = 0; i < window_ids.size(); ++i) { + const std::span group_span(groups[i].data(), groups[i].size()); + runtime.CopyBuffer(runtime.UnifiedMemoryWindowBuffer(window_ids[i]), buffer, + group_span, true); + } + runtime.Finish(); + return true; + } else { + return false; + } +} + template void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] u64 total_size_bytes, @@ -1802,6 +1991,12 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 } if constexpr (USE_MEMORY_MAPS) { + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory() && + TryUnifiedDownloadMemory(buffer, std::span(copies.data(), copies.size()))) { + return; + } + } auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); const u8* const mapped_memory = download_staging.mapped_span.data(); const std::span copies_span(copies.data(), copies.data() + copies.size()); @@ -1872,7 +2067,7 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { #ifdef YUZU_LEGACY if (!do_not_mark || !immediately_free) #endif - delayed_destruction_ring.Push(std::move(slot_buffers[buffer_id])); + sentenced_buffers.Push(std::move(slot_buffers[buffer_id]), runtime.CurrentSyncPoint()); slot_buffers.erase(buffer_id); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 14ab3e6ebc..c3f39d39cb 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,7 @@ #include "common/slot_vector.h" #include "video_core/buffer_cache/buffer_base.h" #include "video_core/control/channel_state_cache.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/dirty_flags.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" @@ -180,15 +181,18 @@ class BufferCache : public VideoCommon::ChannelSetupCaches copies); + bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); + void DownloadBufferMemory(Buffer& buffer_id); void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size); @@ -475,12 +485,7 @@ private: Tegra::MaxwellDeviceMemoryManager& device_memory; Common::SlotVector slot_buffers; -#ifdef YUZU_LEGACY - static constexpr size_t TICKS_TO_DESTROY = 6; -#else - static constexpr size_t TICKS_TO_DESTROY = 8; -#endif - DelayedDestructionRing delayed_destruction_ring; + DeferredDestructionQueue sentenced_buffers; const Tegra::Engines::Maxwell3D::DrawManager::IndirectParams* current_draw_indirect{}; @@ -515,8 +520,14 @@ private: Common::LeastRecentlyUsedCache lru_cache; u64 frame_tick = 0; u64 total_used_memory = 0; - u64 minimum_memory = 0; - u64 critical_memory = 0; + u64 memory_budget = 0; + u64 cached_device_usage = 0; + /// Sync point the last reclaim's evictions were queued at. Their memory is not back with the + /// device until this completes, so reclaiming again before then measures stale usage. + u64 reclaim_wait_sync_point = 0; + u32 usage_refresh_countdown = 0; + bool in_reclaim = false; + bool reclaim_stalled = false; BufferId inline_buffer_id; #ifdef YUZU_LEGACY bool immediately_free = false; diff --git a/src/video_core/deferred_destruction_queue.h b/src/video_core/deferred_destruction_queue.h new file mode 100644 index 0000000000..d5faf1e6aa --- /dev/null +++ b/src/video_core/deferred_destruction_queue.h @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include +#include + +#include "common/common_types.h" + +namespace VideoCommon { + +template +class DeferredDestructionQueue { +public: + void Push(T&& object, u64 sync_point) { + entries.emplace_back(std::move(object), sync_point); + } + + void Reclaim(u64 completed_sync_point) { + while (!entries.empty() && entries.front().sync_point <= completed_sync_point) { + entries.pop_front(); + } + } + + void Clear() { + entries.clear(); + } + + [[nodiscard]] size_t Size() const noexcept { + return entries.size(); + } + + [[nodiscard]] bool Empty() const noexcept { + return entries.empty(); + } + +private: + struct Entry { + Entry(T&& object_, u64 sync_point_) noexcept + : object{std::move(object_)}, sync_point{sync_point_} {} + + T object; + u64 sync_point; + }; + + using EntryDequeOptions = + boost::container::deque_options>::type; + + boost::container::deque entries; +}; + +} // namespace VideoCommon diff --git a/src/video_core/delayed_destruction_ring.h b/src/video_core/delayed_destruction_ring.h deleted file mode 100644 index 1f7af86d26..0000000000 --- a/src/video_core/delayed_destruction_ring.h +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include -#include -#include -#include - -namespace VideoCommon { - -/// Container to push objects to be destroyed a few ticks in the future -template -class DelayedDestructionRing { -public: - void Tick() { - index = (index + 1) % TICKS_TO_DESTROY; - elements[index].clear(); - } - - void Push(T&& object) { - elements[index].push_back(std::move(object)); - } - -private: - size_t index = 0; - std::array, TICKS_TO_DESTROY> elements; -}; - -} // namespace VideoCommon diff --git a/src/video_core/engines/fermi_2d.cpp b/src/video_core/engines/fermi_2d.cpp index 8b02c91e03..b1f92fe9ca 100644 --- a/src/video_core/engines/fermi_2d.cpp +++ b/src/video_core/engines/fermi_2d.cpp @@ -71,7 +71,8 @@ void Fermi2D::Blit() { constexpr s64 null_derivative = 1ULL << 32; Surface src = regs.src; const auto bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(src.format)); - const bool delegate_to_gpu = src.width > 512 && src.height > 512 && bytes_per_pixel <= 8 && + const u64 src_area = static_cast(src.width) * static_cast(src.height); + const bool delegate_to_gpu = src_area > 512ULL * 512ULL && bytes_per_pixel <= 8 && src.format != regs.dst.format; auto srcX = args.src_x0; diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 16f19bf009..a7d45acc61 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -18,7 +18,7 @@ #include "common/common_types.h" #include "common/settings.h" #include "common/thread.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" #include "video_core/host1x/syncpoint_manager.h" @@ -50,7 +50,8 @@ public: /// Notify the fence manager about a new frame void TickFrame() { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Tick(); + ++retire_tick; + sentenced_fences.Reclaim(retire_tick > RETIRE_DELAY ? retire_tick - RETIRE_DELAY : 0); } // Unlike other fences, this one doesn't @@ -91,9 +92,6 @@ public: func(); } fences.push(std::move(new_fence)); - if (should_flush) { - rasterizer.FlushCommands(); - } if constexpr (can_async_check) { guard.unlock(); cv.notify_all(); @@ -186,7 +184,7 @@ private: } { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Push(std::move(current_fence)); + sentenced_fences.Push(std::move(current_fence), retire_tick); } fences.pop(); } @@ -219,7 +217,7 @@ private: } { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Push(std::move(current_fence)); + sentenced_fences.Push(std::move(current_fence), retire_tick); } } } @@ -238,10 +236,10 @@ private: void PopAsyncFlushes() { { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + std::scoped_lock lock{texture_cache.mutex}; texture_cache.PopAsyncFlushes(); - buffer_cache.PopAsyncFlushes(); } + buffer_cache.PopAsyncFlushes(); query_cache.PopAsyncFlushes(); } @@ -264,7 +262,9 @@ private: std::jthread fence_thread; - DelayedDestructionRing delayed_destruction_ring; + static constexpr u64 RETIRE_DELAY = 8; + u64 retire_tick = 1; + DeferredDestructionQueue sentenced_fences; }; } // namespace VideoCommon diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 50570b596d..2acf9ac458 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -30,6 +30,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten thread = std::jthread([&](std::stop_token stop_token) { Common::SetCurrentThreadName("GPU"); Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical); + Common::SetCurrentThreadToPerformanceCores(); system.RegisterHostThread(); auto current_context = context.Acquire(); diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 0d342c463d..9949e90e09 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -17,11 +17,13 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp + ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d_buffer.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_bcn.comp + ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_buffer.comp ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d24s8.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d32f.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_d32f_to_abgr8.frag @@ -32,6 +34,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.comp ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.frag + ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_s8d24_to_abgr8.frag ${CMAKE_CURRENT_SOURCE_DIR}/full_screen_triangle.vert ${CMAKE_CURRENT_SOURCE_DIR}/fxaa.frag diff --git a/src/video_core/host_shaders/astc_decoder.comp b/src/video_core/host_shaders/astc_decoder.comp index ce03e589ad..86a9c0a4d2 100644 --- a/src/video_core/host_shaders/astc_decoder.comp +++ b/src/video_core/host_shaders/astc_decoder.comp @@ -77,14 +77,8 @@ uvec4 local_buff; uvec4 color_endpoint_data; int color_bitsread = 0; -// Global "vector" to be pushed into when decoding -// At most will require BLOCK_WIDTH x BLOCK_HEIGHT in single plane mode -// At most will require BLOCK_WIDTH x BLOCK_HEIGHT x 2 in dual plane mode -// So the maximum would be 144 (12 x 12) elements, x 2 for two planes -#define DIVCEIL(number, divisor) (number + divisor - 1) / divisor -#define ARRAY_NUM_ELEMENTS 144 -#define VECTOR_ARRAY_SIZE DIVCEIL(ARRAY_NUM_ELEMENTS * 2, 4) -uint result_vector[ARRAY_NUM_ELEMENTS * 2]; +#define MAX_WEIGHT_VALUES 64 +uint result_vector[MAX_WEIGHT_VALUES]; int result_index = 0; uint result_vector_max_index; @@ -492,7 +486,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o A = ReplicateBitTo9((bitval & 1)); switch (encoding) { case JUST_BITS: - color_values[++out_index] = FastReplicateTo8(bitval, bitlen); + color_values[out_index++] = FastReplicateTo8(bitval, bitlen); break; case TRIT: { D = QuintTritValue(val); @@ -571,7 +565,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o uint T = (D * C) + B; T ^= A; T = (A & 0x80) | (T >> 2); - color_values[++out_index] = T; + color_values[out_index++] = T; } } } @@ -753,12 +747,12 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui #define READ_UINT_VALUES(N) \ uvec4 V[2]; \ for (uint i = 0; i < N; i++) { \ - V[i / 4][i % 4] = color_values[++colvals_index]; \ + V[i / 4][i % 4] = color_values[colvals_index++]; \ } #define READ_INT_VALUES(N) \ ivec4 V[2]; \ for (uint i = 0; i < N; i++) { \ - V[i / 4][i % 4] = int(color_values[++colvals_index]); \ + V[i / 4][i % 4] = int(color_values[colvals_index++]); \ } switch (color_endpoint_mode) { @@ -1225,6 +1219,10 @@ void DecompressBlock(ivec3 coord) { FillError(coord); return; } + if (GetNumWeightValues(size_params, dual_plane) > MAX_WEIGHT_VALUES) { + FillError(coord); + return; + } uint partition_index = 1; uvec4 color_endpoint_mode = uvec4(0); uint ced_pointer = 0; @@ -1384,11 +1382,7 @@ void DecompressBlock(ivec3 coord) { p = Cf / 65535.0f; } -#ifdef VULKAN - imageStore(dest_image, coord + ivec3(i, j, 0), p.gbar); -#else imageStore(dest_image, coord + ivec3(i, j, 0), clamp(p, 0.0f, 1.0f).gbar); -#endif } } } diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d_buffer.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d_buffer.comp new file mode 100644 index 0000000000..3fc71e05b6 --- /dev/null +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d_buffer.comp @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 430 + +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_8bit_storage : require + +#define BINDING_INPUT_BUFFER 0 +#define BINDING_OUTPUT_BUFFER 1 + +layout(push_constant) uniform PushConstants { + uvec3 dim; + uint bytes_per_block_log2; + + uvec3 origin; + uint layer_stride; + + uint block_size; + uint x_shift; + uint block_height; + uint block_height_mask; +} pc; + +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; }; + +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer { + uint out_u32[]; +}; + +layout(local_size_x = 16, local_size_y = 8, local_size_z = 1) in; + +const uint GOB_SIZE_X = 64; +const uint GOB_SIZE_Y = 8; + +const uint GOB_SIZE_X_SHIFT = 6; +const uint GOB_SIZE_Y_SHIFT = 3; +const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT; + +const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u); + +uint SwizzleTable(uint pos) { + const uint t[8] = uint[]( + 0x12100200, 0x13110301, 0x16140604, 0x17150705, + 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d + ); + const uint i = pos >> 4; + const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; + return (h << 4) | (pos & 0xf); +} + +uint SwizzleOffset(uvec2 pos) { + pos = pos & SWIZZLE_MASK; + return SwizzleTable(pos.y * 64u + pos.x); +} + +uvec4 ReadTexel(uint offset) { + switch (pc.bytes_per_block_log2) { + case 2u: + return uvec4(u32data[offset / 4u], 0u, 0u, 0u); + case 3u: + return uvec4(u64data[offset / 8u], 0u, 0u); + case 4u: + return u128data[offset / 16u]; + } + return uvec4(0u); +} + +void main() { + uvec3 coord = gl_GlobalInvocationID; + if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) { + return; + } + + uvec3 pos = coord + pc.origin; + pos.x <<= pc.bytes_per_block_log2; + + uint swizzle = SwizzleOffset(pos.xy); + uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; + + uint offset = 0u; + offset += pos.z * pc.layer_stride; + offset += (block_y >> pc.block_height) * pc.block_size; + offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT; + offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift; + offset += swizzle; + + uvec4 texel = ReadTexel(offset); + + uint words = 1u << (pc.bytes_per_block_log2 - 2u); + uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y; + uint out_idx = linear_index * words; + + out_u32[out_idx] = texel.x; + if (words > 1u) { + out_u32[out_idx + 1u] = texel.y; + } + if (words > 2u) { + out_u32[out_idx + 2u] = texel.z; + out_u32[out_idx + 3u] = texel.w; + } +} diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_buffer.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_buffer.comp new file mode 100644 index 0000000000..7333568eb7 --- /dev/null +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_buffer.comp @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 430 + +#define BINDING_INPUT_BUFFER 0 +#define BINDING_OUTPUT_BUFFER 1 + +layout(push_constant) uniform PushConstants { + uvec3 dim; + uint bytes_per_block_log2; + + uvec3 origin; + uint slice_size; + + uint block_size; + uint x_shift; + uint block_height; + uint block_height_mask; + + uint block_depth; + uint block_depth_mask; +} pc; + +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; }; + +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer { + uint out_u32[]; +}; + +layout(local_size_x = 8, local_size_y = 8, local_size_z = 4) in; + +const uint GOB_SIZE_X = 64; +const uint GOB_SIZE_Y = 8; + +const uint GOB_SIZE_X_SHIFT = 6; +const uint GOB_SIZE_Y_SHIFT = 3; +const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT; + +const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u); + +uint SwizzleTable(uint pos) { + const uint t[8] = uint[]( + 0x12100200, 0x13110301, 0x16140604, 0x17150705, + 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d + ); + const uint i = pos >> 4; + const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; + return (h << 4) | (pos & 0xf); +} + +uint SwizzleOffset(uvec2 pos) { + pos = pos & SWIZZLE_MASK; + return SwizzleTable(pos.y * 64u + pos.x); +} + +uvec4 ReadTexel(uint offset) { + switch (pc.bytes_per_block_log2) { + case 2u: + return uvec4(u32data[offset / 4u], 0u, 0u, 0u); + case 3u: + return uvec4(u64data[offset / 8u], 0u, 0u); + case 4u: + return u128data[offset / 16u]; + } + return uvec4(0u); +} + +void main() { + uvec3 coord = gl_GlobalInvocationID; + if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) { + return; + } + + uvec3 pos = coord + pc.origin; + pos.x <<= pc.bytes_per_block_log2; + + uint swizzle = SwizzleOffset(pos.xy); + uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; + + uint offset = 0u; + offset += (pos.z >> pc.block_depth) * pc.slice_size; + offset += (pos.z & pc.block_depth_mask) << (GOB_SIZE_SHIFT + pc.block_height); + offset += (block_y >> pc.block_height) * pc.block_size; + offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT; + offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift; + offset += swizzle; + + uvec4 texel = ReadTexel(offset); + + uint words = 1u << (pc.bytes_per_block_log2 - 2u); + uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y; + uint out_idx = linear_index * words; + + out_u32[out_idx] = texel.x; + if (words > 1u) { + out_u32[out_idx + 1u] = texel.y; + } + if (words > 2u) { + out_u32[out_idx + 2u] = texel.z; + out_u32[out_idx + 3u] = texel.w; + } +} diff --git a/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag new file mode 100644 index 0000000000..f883b3196f --- /dev/null +++ b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 450 core + +layout(binding = 0) uniform sampler2D img_in; + +layout(push_constant) uniform PushConstants { + ivec2 dst_offset; + ivec2 src_offset; + ivec2 scale; +}; + +void main() { + const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset; + const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x); + const ivec2 coord = msaa_coord * scale + sample_offset + src_offset; + gl_FragDepth = texelFetch(img_in, coord, 0).r; +} diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index c583fbdf7d..a78486c39e 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -370,8 +370,37 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si template void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which) const { + const u8* run_src{nullptr}; + u8* run_dst{nullptr}; + std::size_t run_size{0}; + auto flush_run = [&] { + if (run_size == 0) { + return; + } + std::memcpy(run_dst, run_src, run_size); + run_src = nullptr; + run_dst = nullptr; + run_size = 0; + }; + auto append_run = [&](const u8* physical, std::size_t copy_amount) { + if (physical == nullptr) [[unlikely]] { + flush_run(); + std::memset(dest_buffer, 0, copy_amount); + return; + } + if (run_size != 0 && run_src + run_size == physical && + run_dst + run_size == static_cast(dest_buffer)) { + run_size += copy_amount; + return; + } + flush_run(); + run_src = physical; + run_dst = static_cast(dest_buffer); + run_size = copy_amount; + }; auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + flush_run(); std::memset(dest_buffer, 0, copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; @@ -381,8 +410,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: if constexpr (is_safe) { rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(dest_buffer, physical, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -392,10 +420,10 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } if (!IsBigPageContinuous(page_index)) [[unlikely]] { + flush_run(); memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); } else { - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(dest_buffer, physical, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); } dest_buffer = static_cast(dest_buffer) + copy_amount; }; @@ -405,6 +433,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: MemoryOperation(base, copy_amount, mapped_normal, set_to_zero, set_to_zero); }; MemoryOperation(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages); + flush_run(); } void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, @@ -420,8 +449,36 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, template void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which) { + const u8* run_src{nullptr}; + u8* run_dst{nullptr}; + std::size_t run_size{0}; + auto flush_run = [&] { + if (run_size == 0) { + return; + } + std::memcpy(run_dst, run_src, run_size); + run_src = nullptr; + run_dst = nullptr; + run_size = 0; + }; + auto append_run = [&](u8* physical, std::size_t copy_amount) { + if (physical == nullptr) [[unlikely]] { + flush_run(); + return; + } + if (run_size != 0 && run_dst + run_size == physical && + run_src + run_size == static_cast(src_buffer)) { + run_size += copy_amount; + return; + } + flush_run(); + run_src = static_cast(src_buffer); + run_dst = physical; + run_size = copy_amount; + }; auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + flush_run(); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -430,8 +487,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if constexpr (is_safe) { rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(physical, src_buffer, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -441,10 +497,10 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } if (!IsBigPageContinuous(page_index)) [[unlikely]] { + flush_run(); memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); } else { - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(physical, src_buffer, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); } src_buffer = static_cast(src_buffer) + copy_amount; }; @@ -454,6 +510,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe MemoryOperation(base, copy_amount, mapped_normal, just_advance, just_advance); }; MemoryOperation(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages); + flush_run(); } void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, @@ -605,7 +662,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const { if (GetEntry(gpu_addr) == EntryType::Mapped) [[likely]] { size_t page_index = gpu_addr >> big_page_bits; if (IsBigPageContinuous(page_index)) [[likely]] { - const std::size_t page{(page_index & big_page_mask) + size}; + const std::size_t page{(gpu_addr & big_page_mask) + size}; return page <= big_page_size; } const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size}; diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index a0acfc48c1..7c8d4de287 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -93,7 +93,17 @@ public: void PostCopyBarrier(); void Finish(); - void TickFrame(Common::SlotVector&) noexcept {} + void TickFrame(Common::SlotVector&) noexcept { + ++sync_point; + } + + u64 CurrentSyncPoint() const noexcept { + return sync_point; + } + + u64 CompletedSyncPoint() const noexcept { + return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0; + } void ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value); @@ -128,6 +138,10 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const { + return GetDeviceMemoryUsage(); + } + void BindFastUniformBuffer(size_t stage, u32 binding_index, u32 size) { const GLuint handle = fast_uniforms[stage][binding_index].handle; const GLsizeiptr gl_size = static_cast(size); @@ -213,9 +227,13 @@ private: GL_FRAGMENT_PROGRAM_PARAMETER_BUFFER_NV, }; + static constexpr u64 SYNC_POINT_DELAY = 8; + const Device& device; StagingBufferPool& staging_buffer_pool; + u64 sync_point = 1; + bool has_fast_buffer_sub_data = false; bool use_assembly_shaders = false; bool has_unified_vertex_buffers = false; @@ -261,6 +279,7 @@ struct BufferCacheParams { // TODO: Investigate why OpenGL seems to perform worse with persistently mapped buffer uploads static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false; + static constexpr bool USE_UNIFIED_MEMORY = false; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 1d77d28c46..e660f75dae 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -485,7 +485,6 @@ void RasterizerOpenGL::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType texture_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::BufferCache))) { - std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::QueryCache))) { diff --git a/src/video_core/renderer_opengl/gl_texture_cache.h b/src/video_core/renderer_opengl/gl_texture_cache.h index dfcef4b0b6..942ffdd514 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.h +++ b/src/video_core/renderer_opengl/gl_texture_cache.h @@ -87,6 +87,10 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const { + return GetDeviceMemoryUsage(); + } + bool CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -139,7 +143,19 @@ public: bool HasNativeASTC() const noexcept; - void TickFrame() {} + void TickFrame() { + ++sync_point; + } + + u64 CurrentSyncPoint() const noexcept { + return sync_point; + } + + u64 CompletedSyncPoint() const noexcept { + return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0; + } + + void WaitSyncPoint(u64) {} StateTracker& GetStateTracker() { return state_tracker; @@ -174,6 +190,9 @@ private: std::array rescale_read_fbos; const Settings::ResolutionScalingInfo& resolution; u64 device_access_memory; + + static constexpr u64 SYNC_POINT_DELAY = 8; + u64 sync_point = 1; }; class Image : public VideoCommon::ImageBase { @@ -370,6 +389,7 @@ struct TextureCacheParams { static constexpr bool HAS_EMULATED_COPIES = true; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true; + static constexpr bool HAS_TIMELINE_SYNC_POINTS = false; using Runtime = OpenGL::TextureCacheRuntime; using Image = OpenGL::Image; diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index 89083d29fb..f4af87fb00 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -21,6 +21,7 @@ #include "video_core/host_shaders/convert_depth_to_float_frag_spv.h" #include "video_core/host_shaders/convert_float_to_depth_frag_spv.h" #include "video_core/host_shaders/convert_msaa_to_non_msaa_frag_spv.h" +#include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_frag_spv.h" #include "video_core/host_shaders/convert_non_msaa_to_msaa_frag_spv.h" #include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h" #include "video_core/host_shaders/full_screen_triangle_vert_spv.h" @@ -519,7 +520,8 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) } [[nodiscard]] vk::ImageView MakeMSAACopyView(const vk::Device& device, VkImage image, - VkFormat format, u32 base_level) { + VkFormat format, u32 base_level, + VkImageAspectFlags aspect_mask) { return device.CreateImageView(VkImageViewCreateInfo{ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .pNext = nullptr, @@ -534,7 +536,7 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) .a = VK_COMPONENT_SWIZZLE_IDENTITY, }, .subresourceRange{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect_mask, .baseMipLevel = base_level, .levelCount = 1, .baseArrayLayer = 0, @@ -545,6 +547,10 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) { const VkRenderPass render_pass = framebuffer->RenderPass(); + if (!render_pass) { + framebuffer->BeginRendering(cmdbuf); + return; + } const VkFramebuffer framebuffer_handle = framebuffer->Handle(); const VkExtent2D render_area = framebuffer->RenderArea(); const VkRenderPassBeginInfo renderpass_bi{ @@ -561,6 +567,31 @@ void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) }; cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); } + +void EndRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) { + if (framebuffer->RenderPass()) { + cmdbuf.EndRenderPass(); + } else { + cmdbuf.EndRendering(); + } +} + +[[nodiscard]] VkPipelineRenderingCreateInfo MakePipelineRenderingCreateInfo( + const Framebuffer* framebuffer) { + return VkPipelineRenderingCreateInfo{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, + .pNext = nullptr, + .viewMask = 0, + .colorAttachmentCount = framebuffer->NumColorAttachments(), + .pColorAttachmentFormats = framebuffer->ColorAttachmentFormats().data(), + .depthAttachmentFormat = framebuffer->HasAspectDepthBit() + ? framebuffer->DepthAttachmentFormat() + : VK_FORMAT_UNDEFINED, + .stencilAttachmentFormat = framebuffer->HasAspectStencilBit() + ? framebuffer->DepthAttachmentFormat() + : VK_FORMAT_UNDEFINED, + }; +} } // Anonymous namespace BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, @@ -610,6 +641,8 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, convert_s8d24_to_abgr8_frag(BuildShader(device, CONVERT_S8D24_TO_ABGR8_FRAG_SPV)), convert_msaa_to_non_msaa_frag(BuildShader(device, CONVERT_MSAA_TO_NON_MSAA_FRAG_SPV)), convert_non_msaa_to_msaa_frag(BuildShader(device, CONVERT_NON_MSAA_TO_MSAA_FRAG_SPV)), + convert_non_msaa_to_msaa_depth_frag( + BuildShader(device, CONVERT_NON_MSAA_TO_MSAA_DEPTH_FRAG_SPV)), linear_sampler(device.GetLogical().CreateSampler(SAMPLER_CREATE_INFO)), nearest_sampler(device.GetLogical().CreateSampler(SAMPLER_CREATE_INFO)) {} @@ -623,10 +656,12 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, const ImageV const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = operation, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *one_texture_pipeline_layout; const VkSampler sampler = is_linear ? *linear_sampler : *nearest_sampler; - const VkPipeline pipeline = FindOrEmplaceColorPipeline(key); + const VkPipeline pipeline = FindOrEmplaceColorPipeline(key, dst_framebuffer); const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D); RecordShaderReadBarrier(scheduler, src_image_view); @@ -651,9 +686,11 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *one_texture_pipeline_layout; - const VkPipeline pipeline = FindOrEmplaceColorPipeline(key); + const VkPipeline pipeline = FindOrEmplaceColorPipeline(key, dst_framebuffer); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, dst_framebuffer, src_image_view, src_image, src_sampler, dst_region, src_region, src_size, pipeline, layout](vk::CommandBuffer cmdbuf) { @@ -666,7 +703,7 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView nullptr); BindBlitState(cmdbuf, layout, dst_region, src_region, src_size); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + EndRenderPass(cmdbuf, dst_framebuffer); }); } @@ -676,10 +713,12 @@ void BlitImageHelper::BlitColorMSAA(const Framebuffer* dst_framebuffer, const BlitMSAAPipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .samples = dst_framebuffer->Samples(), + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *one_texture_pipeline_layout; const VkSampler sampler = *nearest_sampler; - const VkPipeline pipeline = FindOrEmplaceBlitColorMSAAPipeline(key); + const VkPipeline pipeline = FindOrEmplaceBlitColorMSAAPipeline(key, dst_framebuffer); const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D); RecordShaderReadBarrier(scheduler, src_image_view); @@ -703,7 +742,7 @@ void BlitImageHelper::ResolveDepthStencil(const Framebuffer* dst_framebuffer, const bool resolve_stencil = dst_framebuffer->HasAspectStencilBit() && device.IsExtShaderStencilExportSupported(); const VkPipeline pipeline = - FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer->RenderPass(), resolve_stencil); + FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer, resolve_stencil); const VkPipelineLayout layout = resolve_stencil ? *two_textures_pipeline_layout : *one_texture_pipeline_layout; const VkSampler sampler = *nearest_sampler; @@ -747,10 +786,12 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = operation, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *two_textures_pipeline_layout; const VkSampler sampler = *nearest_sampler; - const VkPipeline pipeline = FindOrEmplaceDepthStencilPipeline(key); + const VkPipeline pipeline = FindOrEmplaceDepthStencilPipeline(key, dst_framebuffer); const VkImageView src_depth_view = src_image_view.DepthView(); const VkImageView src_stencil_view = src_image_view.StencilView(); @@ -772,25 +813,25 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, void BlitImageHelper::ConvertD32ToR32(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer->RenderPass()); + ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer); Convert(*convert_d32_to_r32_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertR32ToD32(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer->RenderPass()); + ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer); Convert(*convert_r32_to_d32_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD16ToR16(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer->RenderPass()); + ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer); Convert(*convert_d16_to_r16_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertR16ToD16(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer->RenderPass()); + ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer); Convert(*convert_r16_to_d16_pipeline, dst_framebuffer, src_image_view); } @@ -801,35 +842,35 @@ void BlitImageHelper::ConvertABGR8ToD24S8(const Framebuffer* dst_framebuffer, LOG_WARNING(Render_Vulkan, "ConvertABGR8ToD24S8 requires shader_stencil_export, skipping"); return; } - ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer, convert_abgr8_to_d24s8_frag); Convert(*convert_abgr8_to_d24s8_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertABGR8ToD32F(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer, convert_abgr8_to_d32f_frag); Convert(*convert_abgr8_to_d32f_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD32FToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer, convert_d32f_to_abgr8_frag); ConvertDepthStencil(*convert_d32f_to_abgr8_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD24S8ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer, convert_d24s8_to_abgr8_frag); ConvertDepthStencil(*convert_d24s8_to_abgr8_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertS8D24ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer, convert_s8d24_to_abgr8_frag); ConvertDepthStencil(*convert_s8d24_to_abgr8_pipeline, dst_framebuffer, src_image_view); } @@ -840,8 +881,10 @@ void BlitImageHelper::ClearColor(const Framebuffer* dst_framebuffer, u8 color_ma const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = Tegra::Engines::Fermi2D::Operation::BlendPremult, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; - const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key); + const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key, dst_framebuffer); const VkPipelineLayout layout = *clear_color_pipeline_layout; scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record( @@ -867,8 +910,10 @@ void BlitImageHelper::ClearDepthStencil(const Framebuffer* dst_framebuffer, bool .stencil_mask = stencil_mask, .stencil_compare_mask = stencil_compare_mask, .stencil_ref = stencil_ref, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; - const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key); + const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key, dst_framebuffer); const VkPipelineLayout layout = *clear_color_pipeline_layout; scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record([pipeline, layout, clear_depth, dst_region](vk::CommandBuffer cmdbuf) { @@ -895,16 +940,34 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i const s32 scale_y = 1 << samples_y; const VkSampleCountFlagBits samples = msaa_to_non_msaa ? VK_SAMPLE_COUNT_1_BIT : SampleCountFlag(num_samples); + const auto dst_surface_type = VideoCore::Surface::GetFormatType(dst_format); + const bool is_depth = dst_surface_type == VideoCore::Surface::SurfaceType::Depth || + dst_surface_type == VideoCore::Surface::SurfaceType::DepthStencil; + const bool has_stencil = dst_surface_type == VideoCore::Surface::SurfaceType::DepthStencil; + const VkImageAspectFlags view_aspect = + is_depth ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT; + VkImageAspectFlags barrier_aspect = VK_IMAGE_ASPECT_COLOR_BIT; + if (is_depth) { + barrier_aspect = VK_IMAGE_ASPECT_DEPTH_BIT; + if (has_stencil) { + barrier_aspect |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + } RenderPassKey renderpass_key{}; renderpass_key.color_formats.fill(VideoCore::Surface::PixelFormat::Invalid); - renderpass_key.color_formats[0] = dst_format; - renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid; + if (is_depth) { + renderpass_key.depth_format = dst_format; + } else { + renderpass_key.color_formats[0] = dst_format; + renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid; + } renderpass_key.samples = samples; const VkRenderPass renderpass = render_pass_cache.Get(renderpass_key); const MSAACopyPipelineKey key{ .renderpass = renderpass, .samples = samples, .msaa_to_non_msaa = msaa_to_non_msaa, + .is_depth = is_depth, }; const VkPipeline pipeline = FindOrEmplaceMSAACopyPipeline(key); const VkPipelineLayout layout = *msaa_copy_pipeline_layout; @@ -920,10 +983,10 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i ASSERT(copy.dst_subresource.num_layers == 1); vk::ImageView src_view = MakeMSAACopyView(device.GetLogical(), src_image, src_vk_format, - static_cast(copy.src_subresource.base_level)); + static_cast(copy.src_subresource.base_level), view_aspect); vk::ImageView dst_view = MakeMSAACopyView(device.GetLogical(), dst_image, dst_vk_format, - static_cast(copy.dst_subresource.base_level)); + static_cast(copy.dst_subresource.base_level), view_aspect); const VkOffset2D dst_offset{copy.dst_offset.x, copy.dst_offset.y}; const VkExtent2D dst_extent{copy.extent.width, copy.extent.height}; const VkRect2D render_area{ @@ -949,50 +1012,64 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, pipeline, layout, sampler, renderpass, framebuffer_handle = *framebuffer, src_view_handle = *src_view, - src = src_image, dst = dst_image, render_area, + src = src_image, dst = dst_image, render_area, is_depth, barrier_aspect, push_constants](vk::CommandBuffer cmdbuf) { - constexpr VkImageSubresourceRange color_range{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + const VkImageSubresourceRange src_range{ + .aspectMask = barrier_aspect, .baseMipLevel = 0, .levelCount = VK_REMAINING_MIP_LEVELS, .baseArrayLayer = 0, .layerCount = VK_REMAINING_ARRAY_LAYERS, }; + const VkImageSubresourceRange dst_range = src_range; + const VkAccessFlags attachment_read = + is_depth ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT + : VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + const VkAccessFlags attachment_write = + is_depth ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + : VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const VkPipelineStageFlags depth_stage = + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; + const VkPipelineStageFlags attachment_stage = + is_depth ? depth_stage + : static_cast( + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); const std::array pre_barriers{ VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .srcAccessMask = attachment_write | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = src, - .subresourceRange = color_range, + .subresourceRange = src_range, }, VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .srcAccessMask = attachment_write | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = attachment_read | attachment_write, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst, - .subresourceRange = color_range, + .subresourceRange = dst_range, }, }; cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | attachment_stage, 0, nullptr, nullptr, pre_barriers); const VkRenderPassBeginInfo renderpass_bi{ .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, @@ -1025,16 +1102,16 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i const VkImageMemoryBarrier post_barrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .srcAccessMask = attachment_write, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst, - .subresourceRange = color_range, + .subresourceRange = dst_range, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + cmdbuf.PipelineBarrier(attachment_stage, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, @@ -1140,12 +1217,14 @@ void BlitImageHelper::ConvertDepthStencil(VkPipeline pipeline, const Framebuffer scheduler.InvalidateState(); } -VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(blit_color_keys, key); if (it != blit_color_keys.end()) { return *blit_color_pipelines[std::distance(blit_color_keys.begin(), it)]; } blit_color_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *blit_color_to_color_frag); const VkPipelineColorBlendAttachmentState blend_attachment{ @@ -1173,7 +1252,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); blit_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1191,21 +1270,23 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *blit_color_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(blit_depth_stencil_keys, key); if (it != blit_depth_stencil_keys.end()) { return *blit_depth_stencil_pipelines[std::distance(blit_depth_stencil_keys.begin(), it)]; } blit_depth_stencil_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *blit_depth_stencil_frag); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); blit_depth_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1223,42 +1304,50 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *blit_depth_stencil_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(clear_color_keys, key); if (it != clear_color_keys.end()) { return *clear_color_pipelines[std::distance(clear_color_keys.begin(), it)]; } clear_color_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*clear_color_vert, *clear_color_frag); - const VkPipelineColorBlendAttachmentState color_blend_attachment_state{ - .blendEnable = VK_TRUE, - .srcColorBlendFactor = VK_BLEND_FACTOR_CONSTANT_COLOR, - .dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR, - .colorBlendOp = VK_BLEND_OP_ADD, - .srcAlphaBlendFactor = VK_BLEND_FACTOR_CONSTANT_ALPHA, - .dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA, - .alphaBlendOp = VK_BLEND_OP_ADD, - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | - VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - }; + const u32 num_color = framebuffer->NumColorAttachments(); + constexpr VkColorComponentFlags full_write_mask = + VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | + VK_COLOR_COMPONENT_A_BIT; + std::array blend_attachments{}; + for (u32 index = 0; index < num_color; ++index) { + blend_attachments[index] = VkPipelineColorBlendAttachmentState{ + .blendEnable = index == 0 ? VK_TRUE : VK_FALSE, + .srcColorBlendFactor = VK_BLEND_FACTOR_CONSTANT_COLOR, + .dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR, + .colorBlendOp = VK_BLEND_OP_ADD, + .srcAlphaBlendFactor = VK_BLEND_FACTOR_CONSTANT_ALPHA, + .dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA, + .alphaBlendOp = VK_BLEND_OP_ADD, + .colorWriteMask = index == 0 ? full_write_mask : VkColorComponentFlags{0}, + }; + } const VkPipelineColorBlendStateCreateInfo color_blend_state_generic_create_info{ .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, .pNext = nullptr, .flags = 0, .logicOpEnable = VK_FALSE, .logicOp = VK_LOGIC_OP_CLEAR, - .attachmentCount = 1, - .pAttachments = &color_blend_attachment_state, + .attachmentCount = num_color, + .pAttachments = blend_attachments.data(), .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, }; const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); clear_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1276,18 +1365,31 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *clear_color_pipelines.back(); } VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( - const BlitDepthStencilPipelineKey& key) { + const BlitDepthStencilPipelineKey& key, const Framebuffer* framebuffer) { const auto it = std::ranges::find(clear_stencil_keys, key); if (it != clear_stencil_keys.end()) { return *clear_stencil_pipelines[std::distance(clear_stencil_keys.begin(), it)]; } clear_stencil_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*clear_color_vert, *clear_stencil_frag); + const u32 num_color = framebuffer->NumColorAttachments(); + std::array blend_attachments{}; + const VkPipelineColorBlendStateCreateInfo color_blend_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = num_color, + .pAttachments = blend_attachments.data(), + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; const auto stencil = VkStencilOpState{ .failOp = VK_STENCIL_OP_KEEP, .passOp = VK_STENCIL_OP_REPLACE, @@ -1314,7 +1416,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); clear_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1325,23 +1427,25 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, .pDepthStencilState = &depth_stencil_ci, - .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, + .pColorBlendState = &color_blend_ci, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *clear_color_pipeline_layout, .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *clear_stencil_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(blit_msaa_color_keys, key); if (it != blit_msaa_color_keys.end()) { return *blit_msaa_color_pipelines[std::distance(blit_msaa_color_keys.begin(), it)]; } blit_msaa_color_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *blit_color_msaa_frag); const VkPipelineMultisampleStateCreateInfo multisample_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, @@ -1357,7 +1461,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); blit_msaa_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1375,26 +1479,32 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *blit_msaa_color_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass, - bool resolve_stencil) { +VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline( + const Framebuffer* framebuffer, bool resolve_stencil) { + const VkRenderPass renderpass = framebuffer->RenderPass(); + const ResolveDepthStencilPipelineKey key{ + .renderpass = renderpass, + .depth_format = framebuffer->DepthAttachmentFormat(), + }; auto& keys = resolve_stencil ? resolve_depth_stencil_keys : resolve_depth_keys; auto& pipelines = resolve_stencil ? resolve_depth_stencil_pipelines : resolve_depth_pipelines; - const auto it = std::ranges::find(keys, renderpass); + const auto it = std::ranges::find(keys, key); if (it != keys.end()) { return *pipelines[std::distance(keys.begin(), it)]; } - keys.push_back(renderpass); + keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, resolve_stencil ? *blit_depth_stencil_msaa_frag : *blit_depth_msaa_frag); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1413,7 +1523,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPas .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *pipelines.back(); } @@ -1423,9 +1533,36 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline return *msaa_copy_pipelines[std::distance(msaa_copy_keys.begin(), it)]; } msaa_copy_keys.push_back(key); - const std::array stages = MakeStages(*clear_color_vert, key.msaa_to_non_msaa - ? *convert_msaa_to_non_msaa_frag - : *convert_non_msaa_to_msaa_frag); + const VkShaderModule frag_module = + key.msaa_to_non_msaa + ? *convert_msaa_to_non_msaa_frag + : (key.is_depth ? *convert_non_msaa_to_msaa_depth_frag + : *convert_non_msaa_to_msaa_frag); + const std::array stages = MakeStages(*clear_color_vert, frag_module); + const VkPipelineDepthStencilStateCreateInfo depth_stencil_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .depthTestEnable = VK_TRUE, + .depthWriteEnable = VK_TRUE, + .depthCompareOp = VK_COMPARE_OP_ALWAYS, + .depthBoundsTestEnable = VK_FALSE, + .stencilTestEnable = VK_FALSE, + .front = {}, + .back = {}, + .minDepthBounds = 0.0f, + .maxDepthBounds = 0.0f, + }; + static constexpr VkPipelineColorBlendStateCreateInfo no_color_blend_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = 0, + .pAttachments = nullptr, + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; const VkPipelineMultisampleStateCreateInfo multisample_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, .pNext = nullptr, @@ -1450,37 +1587,42 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pMultisampleState = &multisample_ci, - .pDepthStencilState = nullptr, - .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, + .pDepthStencilState = key.is_depth ? &depth_stencil_ci : nullptr, + .pColorBlendState = key.is_depth ? &no_color_blend_ci + : &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *msaa_copy_pipeline_layout, .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *msaa_copy_pipelines.back(); } -void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) { - ConvertPipeline(pipeline, renderpass, false); +void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline, + const Framebuffer* framebuffer) { + ConvertPipeline(pipeline, framebuffer, false); } -void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) { - ConvertPipeline(pipeline, renderpass, true); +void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline, + const Framebuffer* framebuffer) { + ConvertPipeline(pipeline, framebuffer, true); } -void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module, bool single_texture, bool is_target_depth) { if (pipeline) { return; } + const VkRenderPass renderpass = framebuffer->RenderPass(); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *module); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1499,31 +1641,35 @@ void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass ren .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } -void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, + const Framebuffer* framebuffer, vk::ShaderModule& module) { - ConvertPipelineEx(pipeline, renderpass, module, false, false); + ConvertPipelineEx(pipeline, framebuffer, module, false, false); } -void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, + const Framebuffer* framebuffer, vk::ShaderModule& module) { - ConvertPipelineEx(pipeline, renderpass, module, true, true); + ConvertPipelineEx(pipeline, framebuffer, module, true, true); } -void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer, bool is_target_depth) { if (pipeline) { return; } + const VkRenderPass renderpass = framebuffer->RenderPass(); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); VkShaderModule frag_shader = is_target_depth ? *convert_float_to_depth_frag : *convert_depth_to_float_frag; const std::array stages = MakeStages(*full_screen_vert, frag_shader); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1542,7 +1688,7 @@ void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass rende .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 67294d594b..defd1caf4e 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -33,6 +33,8 @@ struct BlitImagePipelineKey { VkRenderPass renderpass; Tegra::Engines::Fermi2D::Operation operation; + std::array color_formats; + VkFormat depth_format; }; struct BlitDepthStencilPipelineKey { @@ -43,6 +45,8 @@ struct BlitDepthStencilPipelineKey { u8 stencil_mask; u32 stencil_compare_mask; u32 stencil_ref; + std::array color_formats; + VkFormat depth_format; }; struct MSAACopyPipelineKey { @@ -51,6 +55,7 @@ struct MSAACopyPipelineKey { VkRenderPass renderpass; VkSampleCountFlagBits samples; bool msaa_to_non_msaa; + bool is_depth; }; struct BlitMSAAPipelineKey { @@ -58,6 +63,15 @@ struct BlitMSAAPipelineKey { VkRenderPass renderpass; VkSampleCountFlagBits samples; + std::array color_formats; + VkFormat depth_format; +}; + +struct ResolveDepthStencilPipelineKey { + constexpr auto operator<=>(const ResolveDepthStencilPipelineKey&) const noexcept = default; + + VkRenderPass renderpass; + VkFormat depth_format; }; class BlitImageHelper { @@ -123,31 +137,36 @@ private: void ConvertDepthStencil(VkPipeline pipeline, const Framebuffer* dst_framebuffer, ImageView& src_image_view); - [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer); - [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer); - [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer); [[nodiscard]] VkPipeline FindOrEmplaceClearStencilPipeline( - const BlitDepthStencilPipelineKey& key); + const BlitDepthStencilPipelineKey& key, const Framebuffer* framebuffer); [[nodiscard]] VkPipeline FindOrEmplaceMSAACopyPipeline(const MSAACopyPipelineKey& key); - [[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key); - [[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass, + [[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key, + const Framebuffer* framebuffer); + [[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(const Framebuffer* framebuffer, bool resolve_stencil); - void ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, bool is_target_depth); + void ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer, + bool is_target_depth); - void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass); + void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer); - void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass); + void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer); - void ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass, + void ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module, bool single_texture, bool is_target_depth); - void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, + void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module); - void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, + void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module); const Device& device; @@ -180,6 +199,7 @@ private: vk::ShaderModule convert_s8d24_to_abgr8_frag; vk::ShaderModule convert_msaa_to_non_msaa_frag; vk::ShaderModule convert_non_msaa_to_msaa_frag; + vk::ShaderModule convert_non_msaa_to_msaa_depth_frag; vk::Sampler linear_sampler; vk::Sampler nearest_sampler; @@ -195,9 +215,9 @@ private: std::vector msaa_copy_pipelines; std::vector blit_msaa_color_keys; std::vector blit_msaa_color_pipelines; - std::vector resolve_depth_keys; + std::vector resolve_depth_keys; std::vector resolve_depth_pipelines; - std::vector resolve_depth_stencil_keys; + std::vector resolve_depth_stencil_keys; std::vector resolve_depth_stencil_pipelines; struct MSAACopyResources { u64 tick; diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp index 0ced890929..02ee3e1e6e 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -164,7 +164,9 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe } provoking_vertex_last.Assign(use_last_provoking_vertex ? 1 : 0); - conservative_raster_enable.Assign(regs.conservative_raster_enable != 0 ? 1 : 0); + if (!features.has_dynamic_state3_conservative_raster_mode) { + conservative_raster_enable.Assign(regs.conservative_raster_enable != 0 ? 1 : 0); + } smooth_lines.Assign(regs.line_anti_alias_enable != 0 ? 1 : 0); alpha_to_coverage_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_coverage != 0 ? 1 : 0); alpha_to_one_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_one != 0 ? 1 : 0); @@ -360,18 +362,35 @@ void FixedPipelineState::DynamicState::Refresh2(const Maxwell& regs, depth_bias_enable.Assign(enabled_lut[POLYGON_OFFSET_ENABLE_LUT[topology_index]] != 0 ? 1 : 0); } +bool IsDepthClipEnabled(const Maxwell& regs) { + const auto clip = regs.viewport_clip_control.geometry_clip.Value(); + return clip == Maxwell::ViewportClipControl::GeometryClip::Passthrough || + clip == Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ || + clip == Maxwell::ViewportClipControl::GeometryClip::FrustumZ; +} + +bool IsDepthClampEnabled(const Maxwell& regs, bool has_depth_clip_enable) { + if (!IsDepthClipEnabled(regs)) { + return true; + } + if (!has_depth_clip_enable) { + return false; + } + return regs.viewport_clip_control.pixel_min_z.Value() != 0 || + regs.viewport_clip_control.pixel_max_z.Value() != 0; +} + void FixedPipelineState::DynamicState::Refresh3(const Maxwell& regs, const DynamicFeatures& features) { if (!features.has_dynamic_state3_logic_op_enable) { logic_op_enable.Assign(regs.logic_op.enable != 0 ? 1 : 0); } + if (features.has_depth_clip_enable) { + depth_clip_disabled.Assign(IsDepthClipEnabled(regs) ? 0 : 1); + } if (!features.has_dynamic_state3_depth_clamp_enable) { - depth_clamp_disabled.Assign(regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::Passthrough || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumZ); + depth_clamp_disabled.Assign( + IsDepthClampEnabled(regs, features.has_depth_clip_enable) ? 0 : 1); } if (!features.has_dynamic_state3_line_stipple_enable) { line_stipple_enable.Assign(regs.line_stipple_enable); diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.h b/src/video_core/renderer_vulkan/fixed_pipeline_state.h index 36a4b4b7a2..45920042f9 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.h +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.h @@ -30,6 +30,8 @@ struct DynamicFeatures { bool has_extended_dynamic_state_3_blend; bool has_extended_dynamic_state_3_enables; bool has_dynamic_state3_depth_clamp_enable; + bool has_dynamic_state3_conservative_raster_mode; + bool has_depth_clip_enable; bool has_dynamic_state3_logic_op_enable; bool has_dynamic_state3_line_stipple_enable; bool has_dynamic_vertex_input; @@ -165,6 +167,7 @@ struct FixedPipelineState { BitField<10, 1, u32> logic_op_enable; BitField<11, 1, u32> depth_clamp_disabled; BitField<12, 1, u32> line_stipple_enable; + BitField<13, 1, u32> depth_clip_disabled; }; union { u32 raw2; @@ -298,6 +301,9 @@ static_assert(std::has_unique_object_representations_v); static_assert(std::is_trivially_copyable_v); static_assert(std::is_trivially_constructible_v); +bool IsDepthClipEnabled(const Maxwell& regs); +bool IsDepthClampEnabled(const Maxwell& regs, bool has_depth_clip_enable); + } // namespace Vulkan namespace std { diff --git a/src/video_core/renderer_vulkan/pipeline_helper.h b/src/video_core/renderer_vulkan/pipeline_helper.h index 882372aee2..0773277e5c 100644 --- a/src/video_core/renderer_vulkan/pipeline_helper.h +++ b/src/video_core/renderer_vulkan/pipeline_helper.h @@ -47,6 +47,93 @@ using Shader::Backend::SPIRV::NUM_TEXTURE_AND_IMAGE_SCALING_WORDS; return std::nullopt; } +[[nodiscard]] inline VkDeviceSize DescriptorSizeForType(const Device& device, + VkDescriptorType type) { + const auto& props = device.DescriptorBufferProperties(); + const bool robust = device.IsRobustBufferAccessEnabled(); + switch (type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + return robust ? props.robustUniformBufferDescriptorSize : props.uniformBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + return robust ? props.robustStorageBufferDescriptorSize : props.storageBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + return robust ? props.robustUniformTexelBufferDescriptorSize + : props.uniformTexelBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + return robust ? props.robustStorageTexelBufferDescriptorSize + : props.storageTexelBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + return props.combinedImageSamplerDescriptorSize; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + return props.storageImageDescriptorSize; + default: + return 0; + } +} + +struct DescriptorBufferBinding { + VkDescriptorType type; + u32 count; + VkDeviceSize offset; + VkDeviceSize stride; +}; + +struct DescriptorBufferLayout { + VkDeviceSize size{}; + boost::container::small_vector bindings; + + [[nodiscard]] bool Empty() const noexcept { + return bindings.empty(); + } +}; + +inline void WriteDescriptorBuffer(const Device& device, const DescriptorBufferLayout& layout, + const DescriptorUpdateEntry* payload, u8* host) { + const vk::Device& dev = device.GetLogical(); + for (const DescriptorBufferBinding& binding : layout.bindings) { + for (u32 index = 0; index < binding.count; ++index) { + const DescriptorUpdateEntry& entry = *(payload++); + const VkDescriptorAddressInfoEXT address_info{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT, + .pNext = nullptr, + .address = entry.address.address, + .range = entry.address.range, + .format = entry.address.format, + }; + VkDescriptorGetInfoEXT get_info{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT, + .pNext = nullptr, + .type = binding.type, + .data{}, + }; + switch (binding.type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + get_info.data.pUniformBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + get_info.data.pStorageBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + get_info.data.pUniformTexelBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + get_info.data.pStorageTexelBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + get_info.data.pCombinedImageSampler = &entry.image; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + get_info.data.pStorageImage = &entry.image; + break; + default: + continue; + } + dev.GetDescriptorEXT(get_info, binding.stride, + host + binding.offset + index * binding.stride); + } + } +} + [[nodiscard]] inline u32 NumDescriptorEntries(const Shader::Info& info) { return Shader::NumDescriptors(info.constant_buffer_descriptors) + Shader::NumDescriptors(info.storage_buffers_descriptors) + @@ -65,16 +152,59 @@ public: num_descriptors <= device->MaxPushDescriptors(); } - // TODO(crueter): utilize layout binding flags - vk::DescriptorSetLayout CreateDescriptorSetLayout(bool use_push_descriptor) const { + bool CanUseDescriptorBuffer() const noexcept { + return device->IsExtDescriptorBufferSupported() && !bindings.empty() && + !CanUsePushDescriptor() && + device->DescriptorBufferProperties().combinedImageSamplerDescriptorSingleArray; + } + + DescriptorBufferLayout MakeDescriptorBufferLayout(VkDescriptorSetLayout layout) const { + DescriptorBufferLayout result; + if (!layout) { + return result; + } + const vk::Device& dev = device->GetLogical(); + result.size = dev.GetDescriptorSetLayoutSizeEXT(layout); + result.bindings.reserve(bindings.size()); + for (const VkDescriptorSetLayoutBinding& binding : bindings) { + result.bindings.push_back(DescriptorBufferBinding{ + .type = binding.descriptorType, + .count = binding.descriptorCount, + .offset = dev.GetDescriptorSetLayoutBindingOffsetEXT(layout, binding.binding), + .stride = DescriptorSizeForType(*device, binding.descriptorType), + }); + } + return result; + } + + vk::DescriptorSetLayout CreateDescriptorSetLayout(bool use_push_descriptor, + bool use_descriptor_buffer = false) const { if (bindings.empty()) { return nullptr; } - const VkDescriptorSetLayoutCreateFlags flags = - use_push_descriptor ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0; + VkDescriptorSetLayoutCreateFlags flags = 0; + if (use_push_descriptor) { + flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR; + } + if (use_descriptor_buffer) { + flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + } + boost::container::small_vector binding_flags; + VkDescriptorSetLayoutBindingFlagsCreateInfo binding_flags_ci{}; + const void* pnext = nullptr; + if (!use_push_descriptor && device->IsDescriptorBindingPartiallyBoundSupported()) { + binding_flags.assign(bindings.size(), VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT); + binding_flags_ci = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO, + .pNext = nullptr, + .bindingCount = static_cast(binding_flags.size()), + .pBindingFlags = binding_flags.data(), + }; + pnext = &binding_flags_ci; + } return device->GetLogical().CreateDescriptorSetLayout({ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, - .pNext = nullptr, + .pNext = pnext, .flags = flags, .bindingCount = static_cast(bindings.size()), .pBindings = bindings.data(), diff --git a/src/video_core/renderer_vulkan/present/util.cpp b/src/video_core/renderer_vulkan/present/util.cpp index 2b83902bc0..33b69c06d4 100644 --- a/src/video_core/renderer_vulkan/present/util.cpp +++ b/src/video_core/renderer_vulkan/present/util.cpp @@ -491,7 +491,7 @@ static vk::Pipeline CreateWrappedPipelineImpl( .subpass = 0, .basePipelineHandle = 0, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass, diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 4402550af9..6c9ae97b03 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -69,6 +69,9 @@ vk::Buffer CreateBuffer(const Device& device, const MemoryAllocator& memory_allo if (device.IsExtConditionalRendering()) { flags |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + flags |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } const VkBufferCreateInfo buffer_ci = { .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = nullptr, @@ -91,6 +94,9 @@ Buffer::Buffer(BufferCacheRuntime& runtime, VideoCommon::NullBufferParams null_p device = &runtime.device; buffer = runtime.CreateNullBuffer(); is_null = true; + if (device->IsBufferDeviceAddressSupported()) { + device_address = device->GetLogical().GetBufferDeviceAddress(*buffer); + } } Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_) @@ -100,6 +106,9 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_) if (runtime.device.HasDebuggingToolAttached()) { buffer.SetObjectNameEXT(fmt::format("Buffer 0x{:x}", CpuAddr()).c_str()); } + if (device->IsBufferDeviceAddressSupported()) { + device_address = device->GetLogical().GetBufferDeviceAddress(*buffer); + } } void Buffer::MarkUsage(u64 offset, u64 size) noexcept { @@ -246,7 +255,6 @@ protected: StagingBufferPool& staging_pool; vk::Buffer buffer{}; - MemoryCommit memory_commit{}; VkIndexType index_type{}; u32 num_indices = 0; }; @@ -356,6 +364,16 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m scheduler_, staging_pool_); } +void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window) { + unified_memory = std::make_unique(device, base, size, hardware_buffers, + hardware_buffer_window); + if (!unified_memory->IsValid()) { + unified_memory.reset(); + } +} + StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { return staging_pool.Request(size, MemoryUsage::Upload); } @@ -364,6 +382,10 @@ StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size, bool def return staging_pool.Request(size, MemoryUsage::Download, deferred); } +VkFormat BufferCacheRuntime::TexelBufferFormat(VideoCore::Surface::PixelFormat format) const { + return MaxwellToVK::SurfaceFormat(device, FormatType::Buffer, false, format).format; +} + void BufferCacheRuntime::FreeDeferredStagingBuffer(StagingBufferRef& ref) { staging_pool.FreeDeferred(ref); } @@ -376,6 +398,10 @@ u64 BufferCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 BufferCacheRuntime::GetDeviceAllocationUsage() const { + return device.GetMemoryBudgetInfo().allocation_bytes; +} + bool BufferCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -404,6 +430,16 @@ u64 BufferCacheRuntime::KnownGpuTick() { return scheduler.GetMasterSemaphore().KnownGpuTick(); } +u64 BufferCacheRuntime::CurrentSyncPoint() const noexcept { + return scheduler.GetMasterSemaphore().CurrentTick(); +} + +u64 BufferCacheRuntime::CompletedSyncPoint() const { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + return master_semaphore.KnownGpuTick(); +} + void BufferCacheRuntime::Wait(u64 buffer_tick) { scheduler.Wait(buffer_tick); } @@ -641,6 +677,7 @@ void BufferCacheRuntime::BindTransformFeedbackBuffer(u32 index, VkBuffer buffer, offset = 0; size = 0; } + scheduler.MarkTransformFeedbackUsed(); scheduler.Record([index, buffer, offset, size](vk::CommandBuffer cmdbuf) { const VkDeviceSize vk_offset = offset; const VkDeviceSize vk_size = size; @@ -653,19 +690,26 @@ void BufferCacheRuntime::BindTransformFeedbackBuffers(VideoCommon::HostBindings< // Already logged in the rasterizer return; } - boost::container::static_vector buffer_handles(bindings.buffers.size()); - for (u32 i = 0; i < bindings.buffers.size(); ++i) { + const u32 count = std::min(static_cast(bindings.buffers.size()), + VideoCommon::NUM_TRANSFORM_FEEDBACK_BUFFERS); + std::array handles{}; + std::array offsets{}; + std::array sizes{}; + for (u32 i = 0; i < count; ++i) { auto handle = bindings.buffers[i]->Handle(); if (handle == VK_NULL_HANDLE) { ReserveNullBuffer(); handle = *null_buffer; - bindings.offsets[i] = 0; - bindings.sizes[i] = 0; + } else { + offsets[i] = bindings.offsets[i]; + sizes[i] = bindings.sizes[i]; } - buffer_handles[i] = handle; + handles[i] = handle; } - scheduler.Record([bindings_ = std::move(bindings), buffer_handles_ = std::move(buffer_handles)](vk::CommandBuffer cmdbuf) { - cmdbuf.BindTransformFeedbackBuffersEXT(0, u32(buffer_handles_.size()), buffer_handles_.data(), bindings_.offsets.data(), bindings_.sizes.data()); + scheduler.MarkTransformFeedbackUsed(); + scheduler.Record([count, handles, offsets, sizes](vk::CommandBuffer cmdbuf) { + cmdbuf.BindTransformFeedbackBuffersEXT(0, count, handles.data(), offsets.data(), + sizes.data()); }); } @@ -690,6 +734,9 @@ vk::Buffer BufferCacheRuntime::CreateNullBuffer() { if (device.IsExtTransformFeedbackSupported()) { create_info.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + create_info.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } vk::Buffer ret = memory_allocator.CreateBuffer(create_info, MemoryUsage::DeviceLocal); if (device.HasDebuggingToolAttached()) { ret.SetObjectNameEXT("Null buffer"); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d4ad156073..9cf435a8ac 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" @@ -39,6 +41,10 @@ public: return *buffer; } + [[nodiscard]] VkDeviceAddress DeviceAddress() const noexcept { + return device_address; + } + [[nodiscard]] bool IsRegionUsed(u64 offset, u64 size) const noexcept { return tracker.IsUsed(offset, size); } @@ -70,6 +76,7 @@ private: vk::Buffer buffer; std::vector views; VideoCommon::UsageTracker tracker; + VkDeviceAddress device_address{}; u64 last_usage_tick{}; bool is_null{}; }; @@ -92,6 +99,26 @@ public: void TickFrame(Common::SlotVector& slot_buffers) noexcept; + void TryEnableUnifiedMemory(void* base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window); + + [[nodiscard]] bool HasUnifiedMemory() const noexcept { + return unified_memory != nullptr && unified_memory->IsValid(); + } + + [[nodiscard]] u64 UnifiedMemorySize() const noexcept { + return unified_memory ? unified_memory->GetSize() : 0; + } + + [[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept { + return unified_memory ? unified_memory->GetWindowSize() : 0; + } + + [[nodiscard]] VkBuffer UnifiedMemoryWindowBuffer(size_t index) const noexcept { + return unified_memory ? unified_memory->GetWindowBuffer(index) : VK_NULL_HANDLE; + } + u64 CurrentTick(); u64 KnownGpuTick(); @@ -100,10 +127,16 @@ public: void Finish(); + u64 CurrentSyncPoint() const noexcept; + + u64 CompletedSyncPoint() const; + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const; + bool CanReportMemoryUsage() const; u32 GetUniformBufferAlignment() const; @@ -145,22 +178,25 @@ public: [[maybe_unused]] u32 binding_index, u32 size) { const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload); - BindBuffer(ref.buffer, static_cast(ref.offset), size); + guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, + static_cast(ref.offset), size); return ref.mapped_span; } - void BindUniformBuffer(VkBuffer buffer, u32 offset, u32 size) { + void BindUniformBuffer(const Buffer& buffer, u32 offset, u32 size) { BindBuffer(buffer, offset, size); } - void BindStorageBuffer(VkBuffer buffer, u32 offset, u32 size, + void BindStorageBuffer(const Buffer& buffer, u32 offset, u32 size, [[maybe_unused]] bool is_written) { BindBuffer(buffer, offset, size); } void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size, VideoCore::Surface::PixelFormat format) { - guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format)); + guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format), + buffer.DeviceAddress(), offset, size, + TexelBufferFormat(format)); } bool ShouldLimitDynamicStorageBuffers() const { @@ -172,14 +208,17 @@ public: } private: - void BindBuffer(VkBuffer buffer, u32 offset, u32 size) { - if (buffer == VK_NULL_HANDLE) { - guest_descriptor_queue.AddBuffer(buffer, 0, VK_WHOLE_SIZE); + void BindBuffer(const Buffer& buffer, u32 offset, u32 size) { + const VkBuffer handle = buffer.Handle(); + if (handle == VK_NULL_HANDLE) { + guest_descriptor_queue.AddBuffer(handle, 0, 0, VK_WHOLE_SIZE); } else { - guest_descriptor_queue.AddBuffer(buffer, offset, size); + guest_descriptor_queue.AddBuffer(handle, buffer.DeviceAddress(), offset, size); } } + VkFormat TexelBufferFormat(VideoCore::Surface::PixelFormat format) const; + void ReserveNullBuffer(); vk::Buffer CreateNullBuffer(); @@ -193,6 +232,7 @@ private: std::shared_ptr quad_strip_index_buffer; vk::Buffer null_buffer; + std::unique_ptr unified_memory; std::unique_ptr uint8_pass; QuadIndexedPass quad_index_pass; @@ -215,6 +255,7 @@ struct BufferCacheParams { static constexpr bool USE_MEMORY_MAPS = true; static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false; static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true; + static constexpr bool USE_UNIFIED_MEMORY = true; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_vulkan/vk_command_pool.cpp b/src/video_core/renderer_vulkan/vk_command_pool.cpp index d0dbf7ca54..7ba35e0e20 100644 --- a/src/video_core/renderer_vulkan/vk_command_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_command_pool.cpp @@ -1,9 +1,13 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #include #include "video_core/renderer_vulkan/vk_command_pool.h" +#include "video_core/renderer_vulkan/vk_master_semaphore.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -14,32 +18,52 @@ constexpr size_t COMMAND_BUFFER_POOL_SIZE = 4; struct CommandPool::Pool { vk::CommandPool handle; vk::CommandBuffers cmdbufs; + u64 tick; }; CommandPool::CommandPool(MasterSemaphore& master_semaphore_, const Device& device_) - : ResourcePool(master_semaphore_, COMMAND_BUFFER_POOL_SIZE), device{device_} {} + : master_semaphore{master_semaphore_}, device{device_} {} CommandPool::~CommandPool() = default; -void CommandPool::Allocate(size_t begin, size_t end) { - // Command buffers are going to be committed, recorded, executed every single usage cycle. - // They are also going to be reset when committed. +void CommandPool::AllocatePool() { Pool& pool = pools.emplace_back(); pool.handle = device.GetLogical().CreateCommandPool({ .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .pNext = nullptr, - .flags = - VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT, .queueFamilyIndex = device.GetGraphicsFamily(), }); pool.cmdbufs = pool.handle.Allocate(COMMAND_BUFFER_POOL_SIZE); + pool.tick = 0; +} + +void CommandPool::AcquirePool() { + if (!pools.empty()) { + master_semaphore.Refresh(); + const u64 gpu_tick = master_semaphore.KnownGpuTick(); + for (size_t i = 0; i < pools.size(); ++i) { + const size_t candidate = (current_pool + 1 + i) % pools.size(); + if (gpu_tick >= pools[candidate].tick) { + current_pool = candidate; + current_index = 0; + pools[current_pool].handle.Reset(); + return; + } + } + } + AllocatePool(); + current_pool = pools.size() - 1; + current_index = 0; } VkCommandBuffer CommandPool::Commit() { - const size_t index = CommitResource(); - const auto pool_index = index / COMMAND_BUFFER_POOL_SIZE; - const auto sub_index = index % COMMAND_BUFFER_POOL_SIZE; - return pools[pool_index].cmdbufs[sub_index]; + if (pools.empty() || current_index >= COMMAND_BUFFER_POOL_SIZE) { + AcquirePool(); + } + Pool& pool = pools[current_pool]; + pool.tick = master_semaphore.CurrentTick(); + return pool.cmdbufs[current_index++]; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_command_pool.h b/src/video_core/renderer_vulkan/vk_command_pool.h index ec1647f012..362ec0610d 100644 --- a/src/video_core/renderer_vulkan/vk_command_pool.h +++ b/src/video_core/renderer_vulkan/vk_command_pool.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -6,7 +9,7 @@ #include #include -#include "video_core/renderer_vulkan/vk_resource_pool.h" +#include "common/common_types.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { @@ -14,20 +17,24 @@ namespace Vulkan { class Device; class MasterSemaphore; -class CommandPool final : public ResourcePool { +class CommandPool final { public: explicit CommandPool(MasterSemaphore& master_semaphore_, const Device& device_); - ~CommandPool() override; - - void Allocate(size_t begin, size_t end) override; + ~CommandPool(); VkCommandBuffer Commit(); private: struct Pool; + void AllocatePool(); + void AcquirePool(); + + MasterSemaphore& master_semaphore; const Device& device; std::vector pools; + size_t current_pool = 0; + size_t current_index = 0; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 80c434dc8f..985b77a70c 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -5,10 +5,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include #include +#include #include "video_core/renderer_vulkan/vk_texture_cache.h" @@ -22,7 +24,9 @@ #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" #include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h" #include "video_core/host_shaders/vulkan_uint8_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_2d_buffer_comp_spv.h" #include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_3d_buffer_comp_spv.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" #include "video_core/surface.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" @@ -31,6 +35,7 @@ #include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/texture_cache/accelerated_swizzle.h" #include "video_core/texture_cache/types.h" +#include "video_core/texture_cache/util.h" #include "video_core/textures/decoders.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -268,7 +273,7 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor .layout = *layout, .basePipelineHandle = {}, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } ComputePass::~ComputePass() = default; @@ -570,7 +575,7 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; // To unswizzle the ASTC data - const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(swizzle, image.info); ASSERT(params.origin == (std::array{0, 0, 0})); ASSERT(params.destination == (std::array{0, 0, 0})); ASSERT(params.bytes_per_block_log2 == 4); @@ -713,7 +718,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( ASSERT(swizzles.size() == 1); const auto& sw = swizzles[0]; - const auto params = MakeBlockLinearSwizzle3DParams(sw, image.info); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info); const u32 blocks_x = (image.info.size.width + 3) / 4; const u32 blocks_y = (image.info.size.height + 3) / 4; @@ -872,4 +877,626 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk( }); } +namespace { +constexpr u32 BL2D_BINDING_INPUT_BUFFER = 0; +constexpr u32 BL2D_BINDING_OUTPUT_BUFFER = 1; + +struct alignas(16) BlockLinearUnswizzle2DPushConstants { + std::array dim; + u32 bytes_per_block_log2; + std::array origin; + u32 layer_stride; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; +}; +static_assert(sizeof(BlockLinearUnswizzle2DPushConstants) <= 128); + +constexpr std::array BL2D_BINDINGS{{ + { + .binding = BL2D_BINDING_INPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = BL2D_BINDING_OUTPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, +}}; + +constexpr std::array BL2D_TEMPLATE{{ + { + .dstBinding = BL2D_BINDING_INPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL2D_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = BL2D_BINDING_OUTPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL2D_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, +}}; + +constexpr DescriptorBankInfo BL2D_BANK_INFO{ + .uniform_buffers = 0, + .storage_buffers = 2, + .texture_buffers = 0, + .image_buffers = 0, + .textures = 0, + .images = 0, + .score = 2, +}; + +constexpr bool BL2D_VERIFY_AGAINST_CPU = false; +} // Anonymous namespace + +BlockLinearUnswizzle2DPass::BlockLinearUnswizzle2DPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, BL2D_BINDINGS, BL2D_TEMPLATE, + BL2D_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_2D_BUFFER_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle2DPass::~BlockLinearUnswizzle2DPass() = default; + +bool BlockLinearUnswizzle2DPass::IsSupported(const VideoCommon::ImageInfo& info) { + if (info.type != VideoCommon::ImageType::e2D) { + return false; + } + if (info.resources.levels != 1) { + return false; + } + if (info.num_samples > 1) { + return false; + } + if (VideoCore::Surface::IsPixelFormatASTC(info.format) || + VideoCore::Surface::IsPixelFormatBCn(info.format)) { + return false; + } + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format); + if (bytes_per_block != 4 && bytes_per_block != 8 && bytes_per_block != 16) { + return false; + } + return VideoCore::Surface::DefaultBlockWidth(info.format) == 1 && + VideoCore::Surface::DefaultBlockHeight(info.format) == 1; +} + +void BlockLinearUnswizzle2DPass::Unswizzle( + Image& image, const StagingBufferRef& swizzled, + std::span swizzles) { + using namespace VideoCommon::Accelerated; + + if (swizzles.empty()) { + return; + } + const VideoCommon::SwizzleParameters& sw = swizzles.front(); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(sw, image.info); + + const u32 width = sw.num_tiles.width; + const u32 height = sw.num_tiles.height; + const u32 depth = image.info.resources.layers; + const u32 bytes_per_block = 1u << params.bytes_per_block_log2; + const VkDeviceSize output_size = + static_cast(width) * height * depth * bytes_per_block; + + const StagingBufferRef output = staging_buffer_pool.Request( + static_cast(output_size), MemoryUsage::DeviceLocal); + + BlockLinearUnswizzle2DPushConstants pc{}; + pc.dim = {width, height, depth}; + pc.bytes_per_block_log2 = params.bytes_per_block_log2; + pc.origin = params.origin; + pc.layer_stride = params.layer_stride; + pc.block_size = params.block_size; + pc.x_shift = params.x_shift; + pc.block_height = params.block_height; + pc.block_height_mask = params.block_height_mask; + + scheduler.RequestOutsideRenderPassOperationContext(); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, + sw.buffer_offset + swizzled.offset, + image.guest_size_bytes - sw.buffer_offset); + compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size); + + const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); + const VkDescriptorSet set = descriptor_allocator.Commit(); + + const u32 gx = Common::DivCeil(width, 16u); + const u32 gy = Common::DivCeil(height, 8u); + const bool is_initialized = image.ExchangeInitialization(); + + const VkBuffer out_buffer = output.buffer; + const VkDeviceSize out_offset = output.offset; + const VkImage dst_image = image.Handle(); + const VkImageAspectFlags aspect = image.AspectMask(); + + scheduler.Record([this, set, descriptor_data, pc, gx, gy, depth, output_size, out_buffer, + out_offset, dst_image, aspect, width, height, + is_initialized](vk::CommandBuffer cmdbuf) { + if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { + return; + } + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); + cmdbuf.Dispatch(gx, gy, depth); + + const VkBufferMemoryBarrier buffer_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = out_offset, + .size = output_size, + }; + const VkImageMemoryBarrier pre_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast( + is_initialized ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + (is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE + : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy); + + const VkBufferImageCopy copy{ + .bufferOffset = out_offset, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource{ + .aspectMask = aspect, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = depth, + }, + .imageOffset = {0, 0, 0}, + .imageExtent = {width, height, 1}, + }; + cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); + + const VkImageMemoryBarrier post_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, post_copy); + }); + + if constexpr (BL2D_VERIFY_AGAINST_CPU) { + VerifyAgainstCpu(swizzled, sw, output, output_size, width, height, depth, + bytes_per_block); + } +} + +void BlockLinearUnswizzle2DPass::VerifyAgainstCpu(const StagingBufferRef& swizzled, + const VideoCommon::SwizzleParameters& sw, + const StagingBufferRef& gpu_output, + VkDeviceSize output_size, u32 width, u32 height, + u32 depth, u32 bytes_per_block) { + const StagingBufferRef readback = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::Download); + + const VkBuffer src = gpu_output.buffer; + const VkDeviceSize src_offset = gpu_output.offset; + const VkBuffer dst = readback.buffer; + const VkDeviceSize dst_offset = readback.offset; + scheduler.Record([src, src_offset, dst, dst_offset, output_size](vk::CommandBuffer cmdbuf) { + const VkBufferMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = src, + .offset = src_offset, + .size = output_size, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, {}, barrier, {}); + const VkBufferCopy copy{ + .srcOffset = src_offset, + .dstOffset = dst_offset, + .size = output_size, + }; + cmdbuf.CopyBuffer(src, dst, copy); + }); + scheduler.Finish(); + + const size_t size = static_cast(output_size); + std::vector reference(size); + const std::span input{swizzled.mapped_span.data() + sw.buffer_offset, + swizzled.mapped_span.size() - sw.buffer_offset}; + Tegra::Texture::UnswizzleTexture(reference, input, bytes_per_block, width, height, depth, + sw.block.height, sw.block.depth); + + const u8* gpu_data = readback.mapped_span.data(); + if (std::memcmp(reference.data(), gpu_data, size) == 0) { + LOG_INFO(Render_Vulkan, "BL2D verify OK: {}x{}x{} bpb={} ({} bytes)", width, height, depth, + bytes_per_block, size); + return; + } + size_t first_diff = size; + size_t num_diff = 0; + for (size_t i = 0; i < size; ++i) { + if (reference[i] != gpu_data[i]) { + if (first_diff == size) { + first_diff = i; + } + ++num_diff; + } + } + LOG_CRITICAL(Render_Vulkan, + "BL2D verify FAILED: {}x{}x{} bpb={} block_height={} first_diff={} " + "num_diff={}/{} cpu=0x{:02x} gpu=0x{:02x}", + width, height, depth, bytes_per_block, sw.block.height, first_diff, + num_diff, size, reference[first_diff], gpu_data[first_diff]); +} + +namespace { +constexpr u32 BL3DB_BINDING_INPUT_BUFFER = 0; +constexpr u32 BL3DB_BINDING_OUTPUT_BUFFER = 1; + +struct alignas(16) BlockLinearUnswizzle3DBufferPushConstants { + std::array dim; + u32 bytes_per_block_log2; + std::array origin; + u32 slice_size; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; + u32 block_depth; + u32 block_depth_mask; +}; +static_assert(sizeof(BlockLinearUnswizzle3DBufferPushConstants) <= 128); + +constexpr std::array BL3DB_BINDINGS{{ + { + .binding = BL3DB_BINDING_INPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = BL3DB_BINDING_OUTPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, +}}; + +constexpr std::array BL3DB_TEMPLATE{{ + { + .dstBinding = BL3DB_BINDING_INPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL3DB_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = BL3DB_BINDING_OUTPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL3DB_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, +}}; + +constexpr DescriptorBankInfo BL3DB_BANK_INFO{ + .uniform_buffers = 0, + .storage_buffers = 2, + .texture_buffers = 0, + .image_buffers = 0, + .textures = 0, + .images = 0, + .score = 2, +}; + +constexpr bool BL3DB_VERIFY_AGAINST_CPU = false; +} // Anonymous namespace + +BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, BL3DB_BINDINGS, BL3DB_TEMPLATE, + BL3DB_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_3D_BUFFER_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle3DBufferPass::~BlockLinearUnswizzle3DBufferPass() = default; + +bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device, + const VideoCommon::ImageInfo& info) { + if (info.type != VideoCommon::ImageType::e3D) { + return false; + } + if (info.resources.levels != 1 || info.resources.layers != 1) { + return false; + } + if (info.num_samples > 1) { + return false; + } + if (info.size.depth <= 1) { + return false; + } + if (VideoCore::Surface::IsPixelFormatASTC(info.format)) { + return false; + } + if (VideoCore::Surface::IsPixelFormatBCn(info.format) && !device.IsOptimalBcnSupported()) { + return false; + } + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format); + return bytes_per_block == 4 || bytes_per_block == 8 || bytes_per_block == 16; +} + +void BlockLinearUnswizzle3DBufferPass::Unswizzle( + Image& image, const StagingBufferRef& swizzled, + std::span swizzles) { + if (swizzles.empty()) { + return; + } + + const VideoCommon::SwizzleParameters& sw = swizzles.front(); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info); + + const u32 blocks_x = sw.num_tiles.width; + const u32 blocks_y = sw.num_tiles.height; + const u32 blocks_z = sw.num_tiles.depth; + const u32 bytes_per_block = 1u << params.bytes_per_block_log2; + const VkDeviceSize output_size = + static_cast(blocks_x) * blocks_y * blocks_z * bytes_per_block; + + const StagingBufferRef output = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::DeviceLocal); + + BlockLinearUnswizzle3DBufferPushConstants pc{}; + pc.dim = {blocks_x, blocks_y, blocks_z}; + pc.bytes_per_block_log2 = params.bytes_per_block_log2; + pc.origin = params.origin; + pc.slice_size = params.slice_size; + pc.block_size = params.block_size; + pc.x_shift = params.x_shift; + pc.block_height = params.block_height; + pc.block_height_mask = params.block_height_mask; + pc.block_depth = params.block_depth; + pc.block_depth_mask = params.block_depth_mask; + + scheduler.RequestOutsideRenderPassOperationContext(); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset, + image.guest_size_bytes - sw.buffer_offset); + compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size); + + const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); + const VkDescriptorSet set = descriptor_allocator.Commit(); + + const u32 gx = Common::DivCeil(blocks_x, 8u); + const u32 gy = Common::DivCeil(blocks_y, 8u); + const u32 gz = Common::DivCeil(blocks_z, 4u); + const bool is_initialized = image.ExchangeInitialization(); + + const VkBuffer out_buffer = output.buffer; + const VkDeviceSize out_offset = output.offset; + const VkImage dst_image = image.Handle(); + const VkImageAspectFlags aspect = image.AspectMask(); + const VkExtent3D extent{ + .width = image.info.size.width, + .height = image.info.size.height, + .depth = image.info.size.depth, + }; + + scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer, + out_offset, dst_image, aspect, extent, + is_initialized](vk::CommandBuffer cmdbuf) { + if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { + return; + } + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); + cmdbuf.Dispatch(gx, gy, gz); + + const VkBufferMemoryBarrier buffer_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = out_offset, + .size = output_size, + }; + const VkImageMemoryBarrier pre_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast( + is_initialized ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + (is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE + : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy); + + const VkBufferImageCopy copy{ + .bufferOffset = out_offset, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource{ + .aspectMask = aspect, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }, + .imageOffset = {0, 0, 0}, + .imageExtent = extent, + }; + cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); + + const VkImageMemoryBarrier post_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, post_copy); + }); + + if constexpr (BL3DB_VERIFY_AGAINST_CPU) { + VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y, + blocks_z, bytes_per_block); + } +} + +void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu( + const StagingBufferRef& swizzled, const VideoCommon::SwizzleParameters& sw, + const VideoCommon::ImageInfo& info, const StagingBufferRef& gpu_output, + VkDeviceSize output_size, u32 blocks_x, u32 blocks_y, u32 blocks_z, u32 bytes_per_block) { + const StagingBufferRef readback = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::Download); + + const VkBuffer src = gpu_output.buffer; + const VkDeviceSize src_offset = gpu_output.offset; + const VkBuffer dst = readback.buffer; + const VkDeviceSize dst_offset = readback.offset; + scheduler.Record([src, src_offset, dst, dst_offset, output_size](vk::CommandBuffer cmdbuf) { + const VkBufferMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = src, + .offset = src_offset, + .size = output_size, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, {}, barrier, {}); + const VkBufferCopy copy{ + .srcOffset = src_offset, + .dstOffset = dst_offset, + .size = output_size, + }; + cmdbuf.CopyBuffer(src, dst, copy); + }); + scheduler.Finish(); + + const size_t size = static_cast(output_size); + std::vector reference(size); + const std::span input{swizzled.mapped_span.data() + sw.buffer_offset, + swizzled.mapped_span.size() - sw.buffer_offset}; + const u32 stride_alignment = VideoCommon::CalculateLevelStrideAlignment(info, sw.level); + Tegra::Texture::UnswizzleTexture(reference, input, bytes_per_block, blocks_x, blocks_y, blocks_z, + sw.block.height, sw.block.depth, stride_alignment); + + const u8* gpu_data = readback.mapped_span.data(); + if (std::memcmp(reference.data(), gpu_data, size) == 0) { + LOG_INFO(Render_Vulkan, "BL3D verify OK: {}x{}x{} bpb={} ({} bytes)", blocks_x, blocks_y, + blocks_z, bytes_per_block, size); + return; + } + size_t first_diff = size; + size_t num_diff = 0; + for (size_t i = 0; i < size; ++i) { + if (reference[i] != gpu_data[i]) { + if (first_diff == size) { + first_diff = i; + } + ++num_diff; + } + } + LOG_CRITICAL(Render_Vulkan, + "BL3D verify FAILED: {}x{}x{} bpb={} block_height={} block_depth={} " + "first_diff={} num_diff={}/{} cpu=0x{:02x} gpu=0x{:02x}", + blocks_x, blocks_y, blocks_z, bytes_per_block, sw.block.height, sw.block.depth, + first_diff, num_diff, size, reference[first_diff], gpu_data[first_diff]); +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 3d30aa6bdc..70a02c8d8a 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -164,4 +164,54 @@ private: ComputePassDescriptorQueue& compute_pass_descriptor_queue; }; +class BlockLinearUnswizzle2DPass final : public ComputePass { +public: + explicit BlockLinearUnswizzle2DPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle2DPass(); + + [[nodiscard]] static bool IsSupported(const VideoCommon::ImageInfo& info); + + void Unswizzle(Image& image, const StagingBufferRef& swizzled, + std::span swizzles); + +private: + void VerifyAgainstCpu(const StagingBufferRef& swizzled, + const VideoCommon::SwizzleParameters& sw, + const StagingBufferRef& gpu_output, VkDeviceSize output_size, u32 width, + u32 height, u32 depth, u32 bytes_per_block); + + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzle3DBufferPass final : public ComputePass { +public: + explicit BlockLinearUnswizzle3DBufferPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle3DBufferPass(); + + [[nodiscard]] static bool IsSupported(const Device& device, + const VideoCommon::ImageInfo& info); + + void Unswizzle(Image& image, const StagingBufferRef& swizzled, + std::span swizzles); + +private: + void VerifyAgainstCpu(const StagingBufferRef& swizzled, + const VideoCommon::SwizzleParameters& sw, + const VideoCommon::ImageInfo& info, + const StagingBufferRef& gpu_output, VkDeviceSize output_size, + u32 blocks_x, u32 blocks_y, u32 blocks_z, u32 bytes_per_block); + + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 006bfc0a09..545c19b273 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -34,12 +34,14 @@ using Tegra::Texture::TexturePair; ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk::PipelineCache& pipeline_cache_, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue_, + DescriptorBufferRing& descriptor_buffer_ring_, Common::ThreadWorker* thread_worker, PipelineStatistics* pipeline_statistics, VideoCore::ShaderNotify* shader_notify, const Shader::Info& info_, vk::ShaderModule spv_module_, u64 shader_hash_) : device{device_}, - pipeline_cache(pipeline_cache_), guest_descriptor_queue{guest_descriptor_queue_}, info{info_}, + pipeline_cache(pipeline_cache_), guest_descriptor_queue{guest_descriptor_queue_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, info{info_}, shader_hash{shader_hash_}, spv_module(std::move(spv_module_)) { if (shader_notify) { shader_notify->MarkShaderBuilding(); @@ -48,18 +50,35 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk uniform_buffer_sizes.begin()); num_descriptor_entries = NumDescriptorEntries(info); - auto func{[this, &scheduler, &descriptor_pool, shader_notify, pipeline_statistics] { - DescriptorLayoutBuilder builder{device}; - builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT); + DescriptorLayoutBuilder builder{device}; + builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT); - uses_push_descriptor = builder.CanUsePushDescriptor(); - descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor); - pipeline_layout = builder.CreatePipelineLayout(*descriptor_set_layout); + uses_push_descriptor = builder.CanUsePushDescriptor(); + uses_descriptor_buffer = builder.CanUseDescriptorBuffer() && descriptor_buffer_ring.IsValid(); + descriptor_set_layout = + builder.CreateDescriptorSetLayout(uses_push_descriptor, uses_descriptor_buffer); + if (uses_descriptor_buffer) { + descriptor_buffer_layout = builder.MakeDescriptorBufferLayout(*descriptor_set_layout); + if (descriptor_buffer_layout.size > DescriptorBufferRing::MaxAllocationSize()) { + LOG_WARNING(Render_Vulkan, + "Compute shader {:016X} needs {} descriptor bytes, falling back to sets", + shader_hash, descriptor_buffer_layout.size); + uses_descriptor_buffer = false; + descriptor_buffer_layout = {}; + descriptor_set_layout = builder.CreateDescriptorSetLayout(false); + } + } + pipeline_layout = builder.CreatePipelineLayout(*descriptor_set_layout); + if (!uses_descriptor_buffer) { descriptor_update_template = builder.CreateTemplate(*descriptor_set_layout, *pipeline_layout, uses_push_descriptor); if (!uses_push_descriptor) { - descriptor_allocator = descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, info); + descriptor_allocator = + descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, info); } + } + + auto func{[this, shader_notify, pipeline_statistics] { const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT subgroup_size_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT, .pNext = nullptr, @@ -69,9 +88,20 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) { flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR; } + if (uses_descriptor_buffer) { + flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + } + VkPipelineCreationFeedback creation_feedback{}; + const VkPipelineCreationFeedbackCreateInfo feedback_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO, + .pNext = nullptr, + .pPipelineCreationFeedback = &creation_feedback, + .pipelineStageCreationFeedbackCount = 0, + .pPipelineStageCreationFeedbacks = nullptr, + }; const VkComputePipelineCreateInfo compute_ci{ .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = device.IsExtPipelineCreationFeedbackSupported() ? &feedback_ci : nullptr, .flags = flags, .stage{ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, @@ -101,6 +131,14 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk return; } + if ((creation_feedback.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) != 0) { + const bool cache_hit = + (creation_feedback.flags & + VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; + LOG_DEBUG(Render_Vulkan, "Compute pipeline {:016X} cache_hit={} duration={}us", + shader_hash, cache_hit, creation_feedback.duration / 1000); + } + // Log compute pipeline creation if (GPU::Logging::IsActive()) { GPU::Logging::GPULogger::GetInstance().LogPipelineStateChange( @@ -128,7 +166,7 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, Tegra::MemoryManager& gpu_memory, Scheduler& scheduler, BufferCache& buffer_cache, TextureCache& texture_cache) { - guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries); + guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); buffer_cache.SetComputeUniformBufferState(info.constant_buffer_mask, &uniform_buffer_sizes); buffer_cache.UnbindComputeStorageBuffers(); @@ -249,9 +287,22 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, GPU::Logging::GPULogger::GetInstance().LogPipelineBind(true, "compute pipeline"); } - const void* const descriptor_data{guest_descriptor_queue.UpdateData()}; + const DescriptorUpdateEntry* const descriptor_data{guest_descriptor_queue.UpdateData()}; + VkDeviceSize descriptor_buffer_offset{}; + bool descriptor_buffer_ready{false}; + if (uses_descriptor_buffer) { + const DescriptorBufferRing::Allocation alloc{ + descriptor_buffer_ring.Allocate(scheduler, descriptor_buffer_layout.size)}; + if (alloc.host) { + WriteDescriptorBuffer(device, descriptor_buffer_layout, descriptor_data, alloc.host); + descriptor_buffer_offset = alloc.offset; + descriptor_buffer_ready = true; + } + } + const bool is_rescaling = !info.texture_descriptors.empty() || !info.image_descriptors.empty(); - scheduler.Record([this, descriptor_data, is_rescaling, + scheduler.Record([this, descriptor_data, is_rescaling, descriptor_buffer_offset, + descriptor_buffer_ready, rescaling_data = rescaling.Data()](vk::CommandBuffer cmdbuf) { if (!pipeline) { return; @@ -265,7 +316,17 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, RESCALING_LAYOUT_WORDS_OFFSET, sizeof(rescaling_data), rescaling_data.data()); } - if (uses_push_descriptor) { + if (uses_descriptor_buffer) { + if (!descriptor_buffer_ready) { + return; + } + const VkDescriptorBufferBindingInfoEXT binding_info{ + descriptor_buffer_ring.BindingInfo()}; + cmdbuf.BindDescriptorBuffersEXT(binding_info); + const u32 buffer_index{}; + cmdbuf.SetDescriptorBufferOffsetsEXT(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline_layout, + 0, buffer_index, descriptor_buffer_offset); + } else if (uses_push_descriptor) { cmdbuf.PushDescriptorSetWithTemplateKHR(*descriptor_update_template, *pipeline_layout, 0, descriptor_data); } else { diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.h b/src/video_core/renderer_vulkan/vk_compute_pipeline.h index 073e4b5079..56c4707e69 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.h @@ -13,7 +13,9 @@ #include "common/common_types.h" #include "common/thread_worker.h" #include "shader_recompiler/shader_info.h" +#include "video_core/renderer_vulkan/pipeline_helper.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" @@ -34,6 +36,7 @@ public: explicit ComputePipeline(const Device& device, Scheduler& scheduler, vk::PipelineCache& pipeline_cache, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue, + DescriptorBufferRing& descriptor_buffer_ring, Common::ThreadWorker* thread_worker, PipelineStatistics* pipeline_statistics, VideoCore::ShaderNotify* shader_notify, const Shader::Info& info, @@ -56,6 +59,7 @@ private: const Device& device; vk::PipelineCache& pipeline_cache; GuestDescriptorQueue& guest_descriptor_queue; + DescriptorBufferRing& descriptor_buffer_ring; Shader::Info info; u64 shader_hash{}; u32 num_descriptor_entries{}; @@ -65,6 +69,8 @@ private: vk::ShaderModule spv_module; vk::DescriptorSetLayout descriptor_set_layout; bool uses_push_descriptor{false}; + bool uses_descriptor_buffer{false}; + DescriptorBufferLayout descriptor_buffer_layout; DescriptorAllocator descriptor_allocator; vk::PipelineLayout pipeline_layout; vk::DescriptorUpdateTemplate descriptor_update_template; diff --git a/src/video_core/renderer_vulkan/vk_descriptor_buffer.cpp b/src/video_core/renderer_vulkan/vk_descriptor_buffer.cpp new file mode 100644 index 0000000000..60047faec0 --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_descriptor_buffer.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/logging.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" +#include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/vulkan_common/vulkan_device.h" + +namespace Vulkan { + +DescriptorBufferRing::DescriptorBufferRing(const Device& device_, + MemoryAllocator& memory_allocator) + : device{device_} { + if (!device.IsExtDescriptorBufferSupported() || !device.IsBufferDeviceAddressSupported()) { + return; + } + alignment = std::max( + device.DescriptorBufferProperties().descriptorBufferOffsetAlignment, 1); + const VkDeviceSize total = FRAME_SIZE * FRAMES_IN_FLIGHT + alignment; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = total, + .usage = VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + buffer = memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::Upload); + if (!buffer.IsHostVisible()) { + LOG_WARNING(Render_Vulkan, "Descriptor buffer is not host visible, disabling"); + buffer.reset(); + return; + } + if (device.HasDebuggingToolAttached()) { + buffer.SetObjectNameEXT("Descriptor buffer"); + } + const VkDeviceAddress raw_address = device.GetLogical().GetBufferDeviceAddress(*buffer); + base_address = Common::AlignUp(raw_address, alignment); + base_host = buffer.Mapped().data() + (base_address - raw_address); +} + +DescriptorBufferRing::~DescriptorBufferRing() = default; + +void DescriptorBufferRing::TickFrame() { + if (++frame_index >= FRAMES_IN_FLIGHT) { + frame_index = 0; + } + frame_start = static_cast(frame_index) * FRAME_SIZE; + cursor = 0; + frame_reused = true; +} + +DescriptorBufferRing::Allocation DescriptorBufferRing::Allocate(Scheduler& scheduler, + VkDeviceSize size) { + ASSERT(buffer); + const VkDeviceSize needed = Common::AlignUp(size, alignment); + if (needed > FRAME_SIZE) { + LOG_ERROR(Render_Vulkan, "Descriptor set of {} bytes exceeds frame capacity {}", needed, + FRAME_SIZE); + return Allocation{}; + } + if (frame_reused) { + frame_reused = false; + scheduler.Wait(frame_ticks[frame_index]); + } + if (cursor + needed > FRAME_SIZE) { + LOG_WARNING(Render_Vulkan, "Descriptor buffer frame exhausted, stalling on the GPU"); + scheduler.Finish(); + cursor = 0; + } + const VkDeviceSize offset = frame_start + cursor; + cursor += needed; + frame_ticks[frame_index] = scheduler.CurrentTick(); + return Allocation{ + .host = base_host + offset, + .offset = offset, + }; +} + +VkDescriptorBufferBindingInfoEXT DescriptorBufferRing::BindingInfo() const noexcept { + return VkDescriptorBufferBindingInfoEXT{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_BUFFER_BINDING_INFO_EXT, + .pNext = nullptr, + .address = base_address, + .usage = VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT, + }; +} + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_descriptor_buffer.h b/src/video_core/renderer_vulkan/vk_descriptor_buffer.h new file mode 100644 index 0000000000..25e0d1380f --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_descriptor_buffer.h @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include "common/common_types.h" +#include "video_core/vulkan_common/vulkan_memory_allocator.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +namespace Vulkan { + +class Device; +class Scheduler; + +class DescriptorBufferRing final { + static constexpr size_t FRAMES_IN_FLIGHT = 8; + static constexpr VkDeviceSize FRAME_SIZE = 512 * 1024; + +public: + explicit DescriptorBufferRing(const Device& device_, MemoryAllocator& memory_allocator); + ~DescriptorBufferRing(); + + struct Allocation { + u8* host{}; + VkDeviceSize offset{}; + }; + + [[nodiscard]] static constexpr VkDeviceSize MaxAllocationSize() noexcept { + return FRAME_SIZE; + } + + void TickFrame(); + + [[nodiscard]] Allocation Allocate(Scheduler& scheduler, VkDeviceSize size); + + [[nodiscard]] VkDescriptorBufferBindingInfoEXT BindingInfo() const noexcept; + + [[nodiscard]] bool IsValid() const noexcept { + return static_cast(buffer); + } + +private: + const Device& device; + vk::Buffer buffer; + VkDeviceAddress base_address{}; + u8* base_host{}; + VkDeviceSize alignment{1}; + size_t frame_index{}; + VkDeviceSize frame_start{}; + VkDeviceSize cursor{}; + std::array frame_ticks{}; + bool frame_reused{}; +}; + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_fence_manager.cpp b/src/video_core/renderer_vulkan/vk_fence_manager.cpp index fad9e38324..bdda8582e9 100644 --- a/src/video_core/renderer_vulkan/vk_fence_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_fence_manager.cpp @@ -37,7 +37,7 @@ void InnerFence::Wait() { if (is_stubbed) { return; } - scheduler.Wait(wait_tick); + scheduler.WaitSubmitted(wait_tick); } FenceManager::FenceManager(VideoCore::RasterizerInterface& rasterizer_, Tegra::GPU& gpu_, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 757e0f3ccf..3db07ad5e1 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include @@ -250,13 +251,15 @@ GraphicsPipeline::GraphicsPipeline( Scheduler& scheduler_, BufferCache& buffer_cache_, TextureCache& texture_cache_, vk::PipelineCache& pipeline_cache_, VideoCore::ShaderNotify* shader_notify, const Device& device_, DescriptorPool& descriptor_pool, - GuestDescriptorQueue& guest_descriptor_queue_, Common::ThreadWorker* worker_thread, + GuestDescriptorQueue& guest_descriptor_queue_, DescriptorBufferRing& descriptor_buffer_ring_, + Common::ThreadWorker* worker_thread, PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache, const GraphicsPipelineCacheKey& key_, std::array stages, const std::array& infos) : key{key_}, device{device_}, texture_cache{texture_cache_}, buffer_cache{buffer_cache_}, pipeline_cache(pipeline_cache_), scheduler{scheduler_}, - guest_descriptor_queue{guest_descriptor_queue_}, spv_modules{std::move(stages)} { + guest_descriptor_queue{guest_descriptor_queue_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, spv_modules{std::move(stages)} { if (shader_notify) { shader_notify->MarkShaderBuilding(); } @@ -276,21 +279,40 @@ GraphicsPipeline::GraphicsPipeline( num_descriptor_entries += NumDescriptorEntries(*info); } fragment_has_color0_output = stage_infos[NUM_STAGES - 1].stores_frag_color[0]; - auto func{[this, shader_notify, &render_pass_cache, &descriptor_pool, pipeline_statistics] { - DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)}; - uses_push_descriptor = builder.CanUsePushDescriptor(); - descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor); - if (!uses_push_descriptor) { - descriptor_allocator = descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, stage_infos); + DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)}; + uses_push_descriptor = builder.CanUsePushDescriptor(); + uses_descriptor_buffer = builder.CanUseDescriptorBuffer() && descriptor_buffer_ring.IsValid(); + descriptor_set_layout = + builder.CreateDescriptorSetLayout(uses_push_descriptor, uses_descriptor_buffer); + if (uses_descriptor_buffer) { + descriptor_buffer_layout = builder.MakeDescriptorBufferLayout(*descriptor_set_layout); + if (descriptor_buffer_layout.size > DescriptorBufferRing::MaxAllocationSize()) { + LOG_WARNING(Render_Vulkan, + "Graphics pipeline {:016X} needs {} descriptor bytes, falling back to sets", + key.Hash(), descriptor_buffer_layout.size); + uses_descriptor_buffer = false; + descriptor_buffer_layout = {}; + descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor); } + } - const VkDescriptorSetLayout set_layout{*descriptor_set_layout}; - pipeline_layout = builder.CreatePipelineLayout(set_layout); + const VkDescriptorSetLayout set_layout{*descriptor_set_layout}; + pipeline_layout = builder.CreatePipelineLayout(set_layout); + if (!uses_descriptor_buffer) { descriptor_update_template = builder.CreateTemplate(set_layout, *pipeline_layout, uses_push_descriptor); + if (!uses_push_descriptor) { + descriptor_allocator = + descriptor_pool.Allocator(device, scheduler, set_layout, stage_infos); + } + } - const VkRenderPass render_pass{render_pass_cache.Get(MakeRenderPassKey(key.state, device))}; + auto func{[this, shader_notify, &render_pass_cache, pipeline_statistics] { + VkRenderPass render_pass{}; + if (!device.IsKhrDynamicRenderingSupported()) { + render_pass = render_pass_cache.Get(MakeRenderPassKey(key.state, device)); + } Validate(); try { MakePipeline(render_pass); @@ -496,7 +518,7 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { buffer_cache.UpdateGraphicsBuffers(is_indexed); buffer_cache.BindHostGeometryBuffers(is_indexed); - guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries); + guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); RescalingPushConstant rescaling; RenderAreaPushConstant render_area; @@ -565,8 +587,33 @@ void GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, } const void* const descriptor_data{guest_descriptor_queue.UpdateData()}; - scheduler.Record([this, descriptor_data, bind_pipeline, rescaling_data = rescaling.Data(), - is_rescaling, update_rescaling, + bool update_descriptors = true; + if (descriptor_set_layout && !uses_push_descriptor && !uses_descriptor_buffer) { + const auto* const entries = static_cast(descriptor_data); + update_descriptors = + bind_pipeline || last_descriptor_payload.size() != num_descriptor_entries || + std::memcmp(last_descriptor_payload.data(), entries, + num_descriptor_entries * sizeof(DescriptorUpdateEntry)) != 0; + if (update_descriptors) { + last_descriptor_payload.assign(entries, entries + num_descriptor_entries); + } + } + VkDeviceSize descriptor_buffer_offset{}; + bool descriptor_buffer_ready{false}; + if (descriptor_set_layout && uses_descriptor_buffer) { + const DescriptorBufferRing::Allocation alloc{ + descriptor_buffer_ring.Allocate(scheduler, descriptor_buffer_layout.size)}; + if (alloc.host) { + WriteDescriptorBuffer(device, descriptor_buffer_layout, + static_cast(descriptor_data), + alloc.host); + descriptor_buffer_offset = alloc.offset; + descriptor_buffer_ready = true; + } + } + scheduler.Record([this, descriptor_data, bind_pipeline, update_descriptors, + descriptor_buffer_offset, descriptor_buffer_ready, + rescaling_data = rescaling.Data(), is_rescaling, update_rescaling, uses_render_area = render_area.uses_render_area, render_area_data = render_area.words](vk::CommandBuffer cmdbuf) { if (bind_pipeline) { @@ -593,10 +640,20 @@ void GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, if (!descriptor_set_layout) { return; } - if (uses_push_descriptor) { + if (uses_descriptor_buffer) { + if (!descriptor_buffer_ready) { + return; + } + const VkDescriptorBufferBindingInfoEXT binding_info{ + descriptor_buffer_ring.BindingInfo()}; + cmdbuf.BindDescriptorBuffersEXT(binding_info); + const u32 buffer_index{}; + cmdbuf.SetDescriptorBufferOffsetsEXT(VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline_layout, + 0, buffer_index, descriptor_buffer_offset); + } else if (uses_push_descriptor) { cmdbuf.PushDescriptorSetWithTemplateKHR(*descriptor_update_template, *pipeline_layout, 0, descriptor_data); - } else { + } else if (update_descriptors) { const VkDescriptorSet descriptor_set{descriptor_allocator.Commit()}; const vk::Device& dev{device.GetLogical()}; dev.UpdateDescriptorSet(descriptor_set, *descriptor_update_template, descriptor_data); @@ -757,16 +814,13 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .lineWidth = 1.0f, // TODO(alekpop): Transfer from regs }; - const bool smooth_lines_supported = - device.IsExtLineRasterizationSupported() && device.SupportsSmoothLines(); - const bool stippled_lines_supported = - device.IsExtLineRasterizationSupported() && device.SupportsStippledRectangularLines(); + const VkLineRasterizationModeEXT line_raster_mode = + device.GetLineRasterizationMode(key.state.smooth_lines != 0); + const bool stippled_lines_supported = device.SupportsStippleForMode(line_raster_mode); VkPipelineRasterizationLineStateCreateInfoEXT line_state{ .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_LINE_STATE_CREATE_INFO_EXT, .pNext = nullptr, - .lineRasterizationMode = key.state.smooth_lines != 0 && smooth_lines_supported - ? VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT - : VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT, + .lineRasterizationMode = line_raster_mode, .stippledLineEnable = (dynamic.line_stipple_enable && stippled_lines_supported) ? VK_TRUE : VK_FALSE, .lineStippleFactor = key.state.line_stipple_factor, @@ -805,6 +859,16 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { if (device.IsExtProvokingVertexSupported()) { provoking_vertex.pNext = std::exchange(rasterization_ci.pNext, &provoking_vertex); } + VkPipelineRasterizationDepthClipStateCreateInfoEXT depth_clip_state{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_DEPTH_CLIP_STATE_CREATE_INFO_EXT, + .pNext = nullptr, + .flags = 0, + .depthClipEnable = static_cast(dynamic.depth_clip_disabled == 0 ? VK_TRUE + : VK_FALSE), + }; + if (device.IsExtDepthClipEnableSupported()) { + depth_clip_state.pNext = std::exchange(rasterization_ci.pNext, &depth_clip_state); + } const bool supports_alpha_output = fragment_has_color0_output; const bool alpha_to_one_supported = device.SupportsAlphaToOne(); @@ -995,10 +1059,65 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) { flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR; } + if (uses_descriptor_buffer) { + flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + } + + const RenderPassKey renderpass_key{MakeRenderPassKey(key.state, device)}; + std::array color_attachment_formats{}; + for (size_t index = 0; index < renderpass_key.color_formats.size(); ++index) { + const PixelFormat pixel_format{renderpass_key.color_formats[index]}; + if (pixel_format == PixelFormat::Invalid) { + color_attachment_formats[index] = VK_FORMAT_UNDEFINED; + continue; + } + color_attachment_formats[index] = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, pixel_format).format; + } + VkFormat depth_attachment_format{VK_FORMAT_UNDEFINED}; + VkFormat stencil_attachment_format{VK_FORMAT_UNDEFINED}; + if (renderpass_key.depth_format != PixelFormat::Invalid) { + const VkFormat format{ + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, + renderpass_key.depth_format) + .format}; + const auto surface_type{VideoCore::Surface::GetFormatType(renderpass_key.depth_format)}; + if (surface_type == VideoCore::Surface::SurfaceType::Depth || + surface_type == VideoCore::Surface::SurfaceType::DepthStencil) { + depth_attachment_format = format; + } + if (surface_type == VideoCore::Surface::SurfaceType::Stencil || + surface_type == VideoCore::Surface::SurfaceType::DepthStencil) { + stencil_attachment_format = format; + } + } + const VkPipelineRenderingCreateInfo rendering_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, + .pNext = nullptr, + .viewMask = 0, + .colorAttachmentCount = static_cast(NumAttachments(key.state)), + .pColorAttachmentFormats = color_attachment_formats.data(), + .depthAttachmentFormat = depth_attachment_format, + .stencilAttachmentFormat = stencil_attachment_format, + }; + + VkPipelineCreationFeedback creation_feedback{}; + const VkPipelineCreationFeedbackCreateInfo feedback_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO, + .pNext = device.IsKhrDynamicRenderingSupported() ? &rendering_ci : nullptr, + .pPipelineCreationFeedback = &creation_feedback, + .pipelineStageCreationFeedbackCount = 0, + .pPipelineStageCreationFeedbacks = nullptr, + }; + const void* const create_next = + device.IsExtPipelineCreationFeedbackSupported() + ? static_cast(&feedback_ci) + : (device.IsKhrDynamicRenderingSupported() ? static_cast(&rendering_ci) + : nullptr); pipeline = device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = create_next, .flags = flags, .stageCount = static_cast(shader_stages.size()), .pStages = shader_stages.data(), @@ -1018,6 +1137,14 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .basePipelineIndex = 0, }, *pipeline_cache); + if ((creation_feedback.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) != 0) { + const bool cache_hit = + (creation_feedback.flags & + VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; + LOG_DEBUG(Render_Vulkan, "Graphics pipeline {:016X} cache_hit={} duration={}us", + key.Hash(), cache_hit, creation_feedback.duration / 1000); + } + // Log graphics pipeline creation if (GPU::Logging::IsActive()) { const std::string pipeline_info = fmt::format( diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index d1caeaee8e..6e3481d609 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -12,14 +12,18 @@ #include #include #include +#include #include "common/thread_worker.h" #include "shader_recompiler/shader_info.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/fixed_pipeline_state.h" +#include "video_core/renderer_vulkan/pipeline_helper.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" +#include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace VideoCore { @@ -76,7 +80,8 @@ public: Scheduler& scheduler, BufferCache& buffer_cache, TextureCache& texture_cache, vk::PipelineCache& pipeline_cache, VideoCore::ShaderNotify* shader_notify, const Device& device, DescriptorPool& descriptor_pool, - GuestDescriptorQueue& guest_descriptor_queue, Common::ThreadWorker* worker_thread, + GuestDescriptorQueue& guest_descriptor_queue, + DescriptorBufferRing& descriptor_buffer_ring, Common::ThreadWorker* worker_thread, PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache, const GraphicsPipelineCacheKey& key, std::array stages, const std::array& infos); @@ -148,6 +153,7 @@ private: vk::PipelineCache& pipeline_cache; Scheduler& scheduler; GuestDescriptorQueue& guest_descriptor_queue; + DescriptorBufferRing& descriptor_buffer_ring; bool (*configure_func)(GraphicsPipeline*, bool){}; @@ -170,10 +176,14 @@ private: vk::DescriptorUpdateTemplate descriptor_update_template; vk::Pipeline pipeline; + DescriptorBufferLayout descriptor_buffer_layout; + std::vector last_descriptor_payload; + std::condition_variable build_condvar; std::mutex build_mutex; std::atomic_bool is_built{false}; bool uses_push_descriptor{false}; + bool uses_descriptor_buffer{false}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 0ff4e52c54..a4d7ccf0a7 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -17,6 +17,7 @@ #include "common/cityhash.h" #include "common/fs/fs.h" #include "common/fs/path_util.h" +#include "common/settings.h" #include "common/thread_worker.h" #include "core/core.h" #include "shader_recompiler/backend/spirv/emit_spirv.h" @@ -45,10 +46,6 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" #include "video_core/gpu_logging/gpu_logging.h" -#ifdef __ANDROID__ -#include "../../android/app/src/main/jni/android_settings.h" -#endif - namespace Vulkan { namespace { @@ -63,6 +60,8 @@ using VideoCommon::GenericEnvironment; using VideoCommon::GraphicsEnvironment; constexpr u32 CACHE_VERSION = 18; +constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128; +constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30; constexpr std::array VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'}; template @@ -304,12 +303,8 @@ size_t GetTotalPipelineWorkers() { const size_t max_core_threads = std::max(static_cast(std::thread::hardware_concurrency()), 2ULL) - 1ULL; #ifdef __ANDROID__ - const int configured = AndroidSettings::values.pipeline_worker_count.GetValue(); - const int clamped = std::clamp(configured, 4, 8); - const size_t desired = static_cast(clamped); - if (desired == 0) { - return 1ULL; - } + const s32 configured = Settings::values.pipeline_worker_count.GetValue(); + const size_t desired = static_cast(std::clamp(configured, 2, 8)); return std::min(max_core_threads, desired); #else return max_core_threads; @@ -340,17 +335,20 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, GuestDescriptorQueue& guest_descriptor_queue_, + DescriptorBufferRing& descriptor_buffer_ring_, RenderPassCache& render_pass_cache_, BufferCache& buffer_cache_, TextureCache& texture_cache_, VideoCore::ShaderNotify& shader_notify_) : VideoCommon::ShaderCache{device_memory_}, device{device_}, scheduler{scheduler_}, descriptor_pool{descriptor_pool_}, guest_descriptor_queue{guest_descriptor_queue_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_}, texture_cache{texture_cache_}, shader_notify{shader_notify_}, use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()}, use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()}, workers(device.HasBrokenParallelShaderCompiling() ? 1ULL : GetTotalPipelineWorkers(), "VkPipelineBuilder"), - serialization_thread(1, "VkPipelineSerialization") { + serialization_thread(1, "VkPipelineSerialization", {}, + Common::ThreadPlacement::Background) { const auto& float_control{device.FloatControlProperties()}; const VkDriverId driver_id{device.GetDriverID()}; const VkShaderStageFlags subgroup_stages{device.GetSubgroupSupportedStages()}; @@ -401,6 +399,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.IsWorkgroupMemoryExplicitLayout8BitAccessSupported(), .support_workgroup_layout_16bit_access = device.IsWorkgroupMemoryExplicitLayout16BitAccessSupported(), + .support_shader_quad_control = device.IsKhrShaderQuadControlSupported(), + .support_quad_shuffles = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_QUAD_BIT), .support_vote = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_VOTE_BIT), .supported_subgroup_stages = supported_subgroup_stages, .support_viewport_index_layer_non_geometry = @@ -514,6 +514,11 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, dynamic_features.has_dynamic_state3_depth_clamp_enable = dynamic_features.has_extended_dynamic_state_3_enables && device.SupportsDynamicState3DepthClampEnable(); + dynamic_features.has_dynamic_state3_conservative_raster_mode = + dynamic_features.has_extended_dynamic_state_3_enables && + device.SupportsDynamicState3ConservativeRasterizationMode(); + dynamic_features.has_depth_clip_enable = + device.IsExtDepthClipEnableSupported(); dynamic_features.has_dynamic_state3_logic_op_enable = dynamic_features.has_extended_dynamic_state_3_enables && device.SupportsDynamicState3LogicOpEnable(); @@ -526,7 +531,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.IsExtVertexInputDynamicStateSupported() && Settings::values.vertex_input_dynamic_state.GetValue(); - dynamic_features.has_provoking_vertex = device.IsExtProvokingVertexSupported(); + dynamic_features.has_provoking_vertex = + device.IsExtProvokingVertexSupported(); dynamic_features.has_provoking_vertex_first_mode = device.SupportsProvokingVertexFirstMode(); dynamic_features.has_provoking_vertex_last_mode = @@ -696,6 +702,10 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading if (use_vulkan_pipeline_cache) { SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache, CACHE_VERSION); + size_t size = 0; + vulkan_pipeline_cache.Read(&size, nullptr); + last_cache_size.store(size, std::memory_order_relaxed); + last_flush = std::chrono::steady_clock::now(); } if (state.statistics) { @@ -703,6 +713,35 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading } } +void PipelineCache::QueueVulkanPipelineCacheFlush() { + if (!use_vulkan_pipeline_cache || vulkan_pipeline_cache_filename.empty()) { + return; + } + if (++pipelines_since_flush < VULKAN_CACHE_FLUSH_PIPELINES) { + return; + } + const auto now = std::chrono::steady_clock::now(); + const auto megabytes = last_cache_size.load(std::memory_order_relaxed) / (1024 * 1024); + const std::chrono::seconds interval{ + std::max(VULKAN_CACHE_FLUSH_MIN_SECONDS, megabytes)}; + if (last_flush.time_since_epoch().count() != 0 && now - last_flush < interval) { + return; + } + if (flush_in_flight.exchange(true, std::memory_order_acq_rel)) { + return; + } + pipelines_since_flush = 0; + last_flush = now; + serialization_thread.QueueWork([this] { + SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache, + CACHE_VERSION); + size_t size = 0; + vulkan_pipeline_cache.Read(&size, nullptr); + last_cache_size.store(size, std::memory_order_relaxed); + flush_in_flight.store(false, std::memory_order_release); + }); +} + GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() { const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)}; auto& pipeline{pair->second}; @@ -741,7 +780,7 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline( std::span envs, PipelineStatistics* statistics, bool build_in_parallel) try { auto hash = key.Hash(); - LOG_INFO(Render_Vulkan, "0x{:016x}", hash); + LOG_DEBUG(Render_Vulkan, "{:#016x}", hash); size_t env_index{0}; std::array programs; const bool uses_vertex_a{key.unique_hashes[0] != 0}; @@ -836,8 +875,8 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline( Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr}; return std::make_unique( scheduler, buffer_cache, texture_cache, vulkan_pipeline_cache, &shader_notify, device, - descriptor_pool, guest_descriptor_queue, thread_worker, statistics, render_pass_cache, key, - std::move(modules), infos); + descriptor_pool, guest_descriptor_queue, descriptor_buffer_ring, thread_worker, statistics, + render_pass_cache, key, std::move(modules), infos); } catch (const Shader::Exception& exception) { auto hash = key.Hash(); @@ -877,6 +916,7 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline() { } SerializePipeline(key, env_ptrs, pipeline_cache_filename, CACHE_VERSION); }); + QueueVulkanPipelineCacheFlush(); return pipeline; } @@ -896,6 +936,7 @@ std::unique_ptr PipelineCache::CreateComputePipeline( SerializePipeline(key, std::array{&env_}, pipeline_cache_filename, CACHE_VERSION); }); + QueueVulkanPipelineCacheFlush(); return pipeline; } @@ -904,11 +945,11 @@ std::unique_ptr PipelineCache::CreateComputePipeline( PipelineStatistics* statistics, bool build_in_parallel) try { auto hash = key.Hash(); if (device.HasBrokenCompute()) { - LOG_ERROR(Render_Vulkan, "Skipping 0x{:016x}", hash); + LOG_ERROR(Render_Vulkan, "Skipping {:#016x}", hash); return nullptr; } - LOG_INFO(Render_Vulkan, "0x{:016x}", hash); + LOG_DEBUG(Render_Vulkan, "{:#016x}", hash); Shader::Maxwell::Flow::CFG cfg{env, pools.flow_block, env.StartAddress()}; @@ -925,7 +966,7 @@ std::unique_ptr PipelineCache::CreateComputePipeline( const u32 max_shared_memory = device.GetMaxComputeSharedMemorySize(); if (needs_shared_mem_clamp && program.shared_memory_size > max_shared_memory) { LOG_WARNING(Render_Vulkan, - "Compute shader 0x{:016x} requests {}KB shared memory but device max is {}KB - clamping", + "Compute shader {:#016x} requests {}KB shared memory but device max is {}KB - clamping", key.unique_hash, program.shared_memory_size / 1024, max_shared_memory / 1024); @@ -957,7 +998,8 @@ std::unique_ptr PipelineCache::CreateComputePipeline( } Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr}; return std::make_unique(device, scheduler, vulkan_pipeline_cache, descriptor_pool, - guest_descriptor_queue, thread_worker, statistics, + guest_descriptor_queue, descriptor_buffer_ring, + thread_worker, statistics, &shader_notify, program.info, std::move(spv_module), key.unique_hash); diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h index 490f9da81a..9d66b663f5 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include #include #include @@ -105,6 +107,7 @@ public: explicit PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue, + DescriptorBufferRing& descriptor_buffer_ring, RenderPassCache& render_pass_cache, BufferCache& buffer_cache, TextureCache& texture_cache, VideoCore::ShaderNotify& shader_notify_); ~PipelineCache(); @@ -143,10 +146,13 @@ private: vk::PipelineCache LoadVulkanPipelineCache(const std::filesystem::path& filename, u32 expected_cache_version); + void QueueVulkanPipelineCacheFlush(); + const Device& device; Scheduler& scheduler; DescriptorPool& descriptor_pool; GuestDescriptorQueue& guest_descriptor_queue; + DescriptorBufferRing& descriptor_buffer_ring; RenderPassCache& render_pass_cache; BufferCache& buffer_cache; TextureCache& texture_cache; @@ -169,6 +175,10 @@ private: std::filesystem::path vulkan_pipeline_cache_filename; vk::PipelineCache vulkan_pipeline_cache; + size_t pipelines_since_flush{}; + std::chrono::steady_clock::time_point last_flush{}; + std::atomic last_cache_size{}; + std::atomic_bool flush_in_flight{}; Common::ThreadWorker workers; Common::ThreadWorker serialization_thread; diff --git a/src/video_core/renderer_vulkan/vk_present_manager.cpp b/src/video_core/renderer_vulkan/vk_present_manager.cpp index 53c3f6759f..13838794de 100644 --- a/src/video_core/renderer_vulkan/vk_present_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_present_manager.cpp @@ -296,9 +296,6 @@ void PresentManager::RecreateSwapchain(Frame* frame) { } void PresentManager::SetImageCount() { - // We cannot have more than 7 images in flight at any given time. - // FRAMES_IN_FLIGHT is 8, and the cache TICKS_TO_DESTROY is 8. - // Mali drivers will give us 6. image_count = std::min(swapchain.GetImageCount(), 7); } diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 709f65dc25..6d8e2309cd 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -235,11 +235,23 @@ public: return; } PauseCounter(); - const auto driver_id = device.GetDriverID(); - if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || - driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { - pending_sync.clear(); + if (!CanResolveHostQueries()) { sync_values_stash.clear(); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([buffer = *accumulation_buffer](vk::CommandBuffer cmdbuf) { + cmdbuf.FillBuffer(buffer, 0, 8, 0); + }); + std::function func([this] { + amend_value = 0; + accumulation_value = 0; + }); + rasterizer->SyncOperation(std::move(func)); + AbandonCurrentQuery(); + num_slots_used = 0; + first_accumulation_checkpoint = (std::numeric_limits::max)(); + last_accumulation_checkpoint = 0; + accumulation_since_last_sync = false; + pending_sync.clear(); return; } sync_values_stash.clear(); @@ -406,6 +418,13 @@ public: } private: + bool CanResolveHostQueries() const { + const auto driver_id = device.GetDriverID(); + return driver_id != VK_DRIVER_ID_QUALCOMM_PROPRIETARY && + driver_id != VK_DRIVER_ID_ARM_PROPRIETARY && + driver_id != VK_DRIVER_ID_MESA_TURNIP; + } + template void ApplyBankOp(VideoCommon::HostQueryBase* query, Func&& func) { size_t size_slots = query->size_slots; @@ -919,7 +938,7 @@ private: return; } has_flushed_end_pending = true; - // Refresh buffers state before beginning transform feedback so counters are up-to-date + scheduler.MarkTransformFeedbackUsed(); UpdateBuffers(); if (!has_started || buffers_count == 0) { // No counter buffers available: begin without counters @@ -1423,18 +1442,12 @@ void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::Lo return; } } - bool was_running = impl->is_hcr_running; - if (was_running) { - PauseHostConditionalRendering(); - } + PauseHostConditionalRendering(); impl->hcr_setup.buffer = impl->hcr_buffer; impl->hcr_setup.offset = impl->hcr_offset; impl->hcr_setup.flags = is_equal ? VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT : 0; impl->hcr_is_set = true; impl->is_hcr_running = false; - if (was_running) { - ResumeHostConditionalRendering(); - } } void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, bool is_equal, @@ -1451,10 +1464,7 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo to_resolve = buffer->Handle(); to_resolve_offset = static_cast(offset); } - bool was_running = impl->is_hcr_running; - if (was_running) { - PauseHostConditionalRendering(); - } + PauseHostConditionalRendering(); impl->conditional_resolve_pass->Resolve(*impl->hcr_resolve_buffer, to_resolve, to_resolve_offset, compare_to_zero); impl->hcr_setup.buffer = *impl->hcr_resolve_buffer; @@ -1462,9 +1472,6 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo impl->hcr_setup.flags = is_equal ? 0 : VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT; impl->hcr_is_set = true; impl->is_hcr_running = false; - if (was_running) { - ResumeHostConditionalRendering(); - } } bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::LookupData object_1, @@ -1472,6 +1479,12 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::Lookup if (!impl->device.IsExtConditionalRendering()) { return false; } + const auto driver_id = impl->device.GetDriverID(); + if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || + driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { + EndHostConditionalRendering(); + return true; + } HostConditionalRenderingCompareBCImpl(object_1.address, true, true); return true; } @@ -1518,10 +1531,12 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValues(VideoCommon::Looku return false; } - auto driver_id = impl->device.GetDriverID(); + const auto driver_id = impl->device.GetDriverID(); const bool is_gpu_high = Settings::IsGPULevelHigh(); - if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { + if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || + driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || + driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { EndHostConditionalRendering(); return true; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 720f3b868c..f7046777db 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include @@ -102,8 +104,20 @@ VkViewport GetViewportState(const Device& device, const Maxwell& regs, size_t in .maxDepth = src.translate_z + src.scale_z, }; if (!device.IsExtDepthRangeUnrestrictedSupported()) { + const float unclamped_min = viewport.minDepth; + const float unclamped_max = viewport.maxDepth; viewport.minDepth = std::clamp(viewport.minDepth, 0.0f, 1.0f); viewport.maxDepth = std::clamp(viewport.maxDepth, 0.0f, 1.0f); + if (viewport.minDepth != unclamped_min || viewport.maxDepth != unclamped_max) { + static std::atomic reported{0}; + if (reported.fetch_add(1, std::memory_order_relaxed) < 32) { + LOG_WARNING(Render_Vulkan, + "Depth range clamped: viewport={} mode={} translate_z={} scale_z={} " + "range=[{}, {}] -> [{}, {}]", + index, static_cast(regs.depth_mode), src.translate_z, src.scale_z, + unclamped_min, unclamped_max, viewport.minDepth, viewport.maxDepth); + } + } } return viewport; } @@ -203,7 +217,10 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra : gpu{gpu_}, device_memory{device_memory_}, device{device_}, memory_allocator{memory_allocator_}, state_tracker{state_tracker_}, scheduler{scheduler_}, staging_pool(device, memory_allocator, scheduler), descriptor_pool(device, scheduler), - guest_descriptor_queue(device), compute_pass_descriptor_queue(device), + guest_descriptor_queue(device, UpdateDescriptorQueue::GUEST_FRAME_PAYLOAD_SIZE, + device.IsExtDescriptorBufferSupported()), + compute_pass_descriptor_queue(device, UpdateDescriptorQueue::COMPUTE_FRAME_PAYLOAD_SIZE), + descriptor_buffer_ring(device, memory_allocator), blit_image(device, scheduler, state_tracker, descriptor_pool), render_pass_cache(device), texture_cache_runtime{ device, scheduler, memory_allocator, staging_pool, @@ -216,14 +233,35 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra staging_pool, compute_pass_descriptor_queue, descriptor_pool, texture_cache), query_cache(gpu, *this, device_memory, query_cache_runtime), pipeline_cache(device_memory, device, scheduler, descriptor_pool, guest_descriptor_queue, - render_pass_cache, buffer_cache, texture_cache, gpu.ShaderNotify()), + descriptor_buffer_ring, render_pass_cache, buffer_cache, texture_cache, + gpu.ShaderNotify()), accelerate_dma(buffer_cache, texture_cache, scheduler), fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler), wfi_event(device.GetLogical().CreateEvent()) { scheduler.SetQueryCache(query_cache); + if (Settings::values.use_unified_memory.GetValue()) { + buffer_cache_runtime.TryEnableUnifiedMemory( + device_memory.GetPhysicalBase(), device_memory.GetPhysicalSize(), + device_memory.GetBackingHardwareBuffers(), + device_memory.GetBackingHardwareBufferWindowSize()); + } + memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 { + u64 freed = staging_pool.ReclaimMemory(bytes); + if (freed < bytes) { + freed += texture_cache.ReclaimMemory(bytes - freed, false); + } + if (freed < bytes) { + freed += buffer_cache.ReclaimMemory(bytes - freed, false); + } + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + vk::TickDeletionQueue(master_semaphore.KnownGpuTick()); + return freed; + }); } RasterizerVulkan::~RasterizerVulkan() { + memory_allocator.SetReclaimCallback(nullptr); scheduler.WaitWorker(); scheduler.Finish(); } @@ -241,11 +279,13 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { if (!pipeline) { return; } - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; - // update engine as channel may be different. - pipeline->SetEngine(maxwell3d, gpu_memory); - if (!pipeline->Configure(is_indexed)) - return; + { + std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + pipeline->SetEngine(maxwell3d, gpu_memory); + if (!pipeline->Configure(is_indexed)) { + return; + } + } UpdateDynamicStates(); @@ -359,7 +399,6 @@ void RasterizerVulkan::DrawTexture() { UpdateDynamicStates(); - query_cache.NotifySegment(true); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); const auto& draw_texture_state = maxwell3d->draw_manager.draw_texture_state; const auto& sampler = texture_cache.GetSampler(draw_texture_state.src_sampler, false); @@ -420,15 +459,21 @@ void RasterizerVulkan::Clear(u32 layer_count) { const bool ds_deferrable = !ds_used || ((!framebuffer->HasAspectDepthBit() || use_depth) && (!framebuffer->HasAspectStencilBit() || use_stencil) && !stencil_partial); - const bool can_defer_clear = ENABLE_DEFERRED_CLEAR && !regs.clear_control.use_scissor && - regs.clear_surface.layer == 0 && - !scheduler.IsRenderPassActive() && - (!use_color || color_full_channels) && ds_deferrable; + const bool clear_shape_deferrable = ENABLE_DEFERRED_CLEAR && + !regs.clear_control.use_scissor && + regs.clear_surface.layer == 0 && + (!use_color || color_full_channels) && ds_deferrable; + // An open pass normally blocks deferral, which drops the clear to ClearAttachments inside the + // pass and also loses the MSAA store discard, since that only applies when a clear is folded + // into the begin. A pass whose BeginRendering has not been recorded yet can be retracted for + // free, so the clear becomes a load op after all. Only retract when it will actually be used. + const bool can_defer_clear = + clear_shape_deferrable && + (!scheduler.IsRenderPassActive() || scheduler.RetractUnrecordedRenderPass()); if (!can_defer_clear) { scheduler.RequestRenderpass(framebuffer); } - query_cache.NotifySegment(true); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); u32 up_scale = 1; u32 down_shift = 0; @@ -673,7 +718,6 @@ void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType texture_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::BufferCache))) { - std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::QueryCache))) { @@ -771,12 +815,15 @@ bool RasterizerVulkan::OnCPUWrite(DAddr addr, u64 size) { return false; } +static constexpr bool ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP = true; + void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) { if (addr == 0 || size == 0) { return; } - { + if (!ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP || + device_memory.IsRegionTextureCached(addr, size)) { std::scoped_lock lock{texture_cache.mutex}; texture_cache.WriteMemory(addr, size); } @@ -880,8 +927,12 @@ void RasterizerVulkan::FlushCommands() { void RasterizerVulkan::TickFrame() { draw_counter = 0; + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + vk::TickDeletionQueue(master_semaphore.KnownGpuTick()); guest_descriptor_queue.TickFrame(); compute_pass_descriptor_queue.TickFrame(); + descriptor_buffer_ring.TickFrame(); fence_manager.TickFrame(); staging_pool.TickFrame(); { @@ -969,11 +1020,11 @@ void RasterizerVulkan::LoadDiskResources(u64 title_id, std::stop_token stop_load void RasterizerVulkan::FlushWork() { #ifdef __ANDROID__ - static constexpr u32 DRAWS_TO_DISPATCH = 512; - static constexpr u32 CHECK_MASK = 3; + static constexpr u32 DRAWS_TO_DISPATCH = 1024; + static constexpr u32 CHECK_MASK = 63; #else static constexpr u32 DRAWS_TO_DISPATCH = 4096; - static constexpr u32 CHECK_MASK = 7; + static constexpr u32 CHECK_MASK = 31; #endif // __ANDROID__ static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0); @@ -1279,6 +1330,13 @@ void RasterizerVulkan::UpdateDepthBias(Tegra::Engines::Maxwell3D::Regs& regs) { regs.zeta.format == Tegra::DepthFormat::S8Z24_UNORM || regs.zeta.format == Tegra::DepthFormat::V8Z24_UNORM; + const bool is_float_depth = regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT || + regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT_X24S8_UINT; + + if (is_float_depth && !device.IsExtDepthBiasControlSupported()) { + units /= static_cast(1ULL << (32 - 24)); + } + if (is_d24 && !device.SupportsD24DepthBuffer()) { static constexpr const size_t length = sizeof(NEEDS_D24) / sizeof(NEEDS_D24[0]); @@ -1451,7 +1509,10 @@ void RasterizerVulkan::UpdateLineWidth(Tegra::Engines::Maxwell3D::Regs& regs) { } const float width = regs.line_anti_alias_enable ? regs.line_width_smooth : regs.line_width_aliased; - scheduler.Record([width](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineWidth(width); }); + const float clamped_width = device.ClampLineWidth(width); + scheduler.Record([clamped_width](vk::CommandBuffer cmdbuf) { + cmdbuf.SetLineWidth(clamped_width); + }); } void RasterizerVulkan::UpdateCullMode(Tegra::Engines::Maxwell3D::Regs& regs) { @@ -1548,7 +1609,10 @@ void RasterizerVulkan::UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs& return; } - scheduler.Record([enable = regs.line_stipple_enable](vk::CommandBuffer cmdbuf) { + const VkLineRasterizationModeEXT mode = + device.GetLineRasterizationMode(regs.line_anti_alias_enable != 0); + const bool enable = regs.line_stipple_enable != 0 && device.SupportsStippleForMode(mode); + scheduler.Record([enable](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineStippleEnableEXT(enable); }); } @@ -1562,28 +1626,24 @@ void RasterizerVulkan::UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Re } if (!device.SupportsDynamicState3LineRasterizationMode()) { - static std::once_flag warn_missing_rect; - std::call_once(warn_missing_rect, [] { + static std::once_flag warn_missing_dynamic_state; + std::call_once(warn_missing_dynamic_state, [] { LOG_WARNING(Render_Vulkan, - "Driver lacks rectangular line rasterization support; skipping dynamic " - "line state updates"); + "Driver lacks dynamic line rasterization mode; the pipeline static value " + "is used instead"); }); return; } const bool wants_smooth = regs.line_anti_alias_enable != 0; - VkLineRasterizationModeEXT mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT; - if (wants_smooth) { - if (device.SupportsSmoothLines()) { - mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT; - } else { - static std::once_flag warn_missing_smooth; - std::call_once(warn_missing_smooth, [] { - LOG_WARNING(Render_Vulkan, - "Line anti-aliasing requested but smoothLines feature unavailable; " - "using rectangular rasterization"); - }); - } + const VkLineRasterizationModeEXT mode = device.GetLineRasterizationMode(wants_smooth); + if (wants_smooth && mode != VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT) { + static std::once_flag warn_missing_smooth; + std::call_once(warn_missing_smooth, [] { + LOG_WARNING(Render_Vulkan, + "Line anti-aliasing requested but smoothLines feature unavailable; " + "falling back to the closest supported mode"); + }); } scheduler.Record([mode](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineRasterizationModeEXT(mode); @@ -1643,12 +1703,7 @@ void RasterizerVulkan::UpdateDepthClampEnable(Tegra::Engines::Maxwell3D::Regs& r if (!device.SupportsDynamicState3DepthClampEnable()) { return; } - bool is_enabled = !(regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::Passthrough || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumZ); + const bool is_enabled = IsDepthClampEnabled(regs, device.IsExtDepthClipEnableSupported()); scheduler.Record( [is_enabled](vk::CommandBuffer cmdbuf) { cmdbuf.SetDepthClampEnableEXT(is_enabled); }); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 206bf1043a..5849eff833 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -17,6 +17,7 @@ #include "video_core/rasterizer_interface.h" #include "video_core/renderer_vulkan/blit_image.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_fence_manager.h" #include "video_core/renderer_vulkan/vk_pipeline_cache.h" @@ -179,7 +180,6 @@ private: void UpdateRasterizerDiscardEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateConservativeRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs& regs); - void UpdateLineStipple(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateDepthBiasEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLogicOpEnable(Tegra::Engines::Maxwell3D::Regs& regs); @@ -207,6 +207,7 @@ private: DescriptorPool descriptor_pool; GuestDescriptorQueue guest_descriptor_queue; ComputePassDescriptorQueue compute_pass_descriptor_queue; + DescriptorBufferRing descriptor_buffer_ring; BlitImageHelper blit_image; RenderPassCache render_pass_cache; diff --git a/src/video_core/renderer_vulkan/vk_resource_pool.cpp b/src/video_core/renderer_vulkan/vk_resource_pool.cpp index 6572f82ba9..9a922e6c85 100644 --- a/src/video_core/renderer_vulkan/vk_resource_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_resource_pool.cpp @@ -12,10 +12,7 @@ ResourcePool::ResourcePool(MasterSemaphore& master_semaphore_, size_t grow_step_ : master_semaphore{&master_semaphore_}, grow_step{grow_step_} {} size_t ResourcePool::CommitResource() { - // Refresh semaphore to query updated results - master_semaphore->Refresh(); - const u64 gpu_tick = master_semaphore->KnownGpuTick(); - const auto search = [this, gpu_tick](size_t begin, size_t end) -> std::optional { + const auto search = [this](size_t begin, size_t end, u64 gpu_tick) -> std::optional { for (size_t iterator = begin; iterator < end; ++iterator) { if (gpu_tick >= ticks[iterator]) { ticks[iterator] = master_semaphore->CurrentTick(); @@ -24,11 +21,17 @@ size_t ResourcePool::CommitResource() { } return std::nullopt; }; - // Try to find a free resource from the hinted position to the end. - std::optional found = search(hint_iterator, ticks.size()); + const auto find_free = [&](u64 gpu_tick) -> std::optional { + std::optional result = search(hint_iterator, ticks.size(), gpu_tick); + if (!result) { + result = search(0, hint_iterator, gpu_tick); + } + return result; + }; + std::optional found = find_free(master_semaphore->KnownGpuTick()); if (!found) { - // Search from beginning to the hinted position. - found = search(0, hint_iterator); + master_semaphore->Refresh(); + found = find_free(master_semaphore->KnownGpuTick()); if (!found) { // Both searches failed, the pool is full; handle it. const size_t free_resource = ManageOverflow(); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 6234b41978..2c3f067a29 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -47,6 +47,7 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) master_semaphore{std::make_unique(device)}, command_pool{std::make_unique(*master_semaphore, device)} { + vk::SetDeletionTimeline(master_semaphore->CurrentTick()); AcquireNewChunk(); AllocateWorkerCommandBuffer(); worker_thread = std::jthread([this](std::stop_token token) { WorkerThread(token); }); @@ -93,13 +94,87 @@ void Scheduler::DispatchWork() { } } +void Scheduler::BeginDynamicRendering(const Framebuffer* framebuffer, const DeferredClear* clear) { + const VkExtent2D render_area = framebuffer->RenderArea(); + std::array attachment_views{}; + const auto& color_views = framebuffer->ColorAttachments(); + for (size_t index = 0; index < color_views.size(); ++index) { + attachment_views[index] = color_views[index]; + } + attachment_views[8] = framebuffer->DepthAttachment(); + state.renderpass = VkRenderPass{}; + state.framebuffer = VkFramebuffer{}; + state.attachment_views = attachment_views; + state.color_resolve_views = framebuffer->ColorResolveAttachments(); + state.color_resolve_modes = framebuffer->ColorResolveModes(); + state.depth_resolve_view = framebuffer->DepthResolveAttachment(); + state.depth_resolve_mode = framebuffer->DepthResolveMode(); + state.stencil_resolve_mode = framebuffer->StencilResolveMode(); + state.discards_msaa_color = framebuffer->DiscardsMsaaColor(); + state.discards_msaa_depth = framebuffer->DiscardsMsaaDepth(); + state.render_area = render_area; + state.num_color = framebuffer->NumColorAttachments(); + state.has_depth = framebuffer->HasAspectDepthBit(); + state.has_stencil = framebuffer->HasAspectStencilBit(); + state.layer_count = framebuffer->NumLayers(); + state.rendering = true; + + if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { + const std::string render_pass_info = + fmt::format("renderArea={}x{}, numImages={}", render_area.width, render_area.height, + framebuffer->NumImages()); + GPU::Logging::GPULogger::GetInstance().LogRenderPassBegin(render_pass_info); + } + + pending_begin = true; + has_pending_begin_clear = clear != nullptr; + pending_begin_clear = clear != nullptr ? *clear : DeferredClear{}; + num_renderpass_images = framebuffer->NumImages(); + renderpass_images = framebuffer->Images(); + renderpass_image_ranges = framebuffer->ImageRanges(); +} + +void Scheduler::FlushPendingRenderPass() { + if (!pending_begin) { + return; + } + + pending_begin = false; + const bool had_clear = has_pending_begin_clear; + has_pending_begin_clear = false; + RecordDynamicBegin(had_clear ? &pending_begin_clear : nullptr); +} + +bool Scheduler::RetractUnrecordedRenderPass() { + if (!pending_begin) { + return false; + } + pending_begin = false; + if (has_pending_begin_clear) { + deferred_clear = pending_begin_clear; + has_pending_begin_clear = false; + } + state.renderpass = VkRenderPass{}; + state.framebuffer = VkFramebuffer{}; + state.attachment_views = {}; + state.rendering = false; + state.uses_transform_feedback = false; + num_renderpass_images = 0; + return true; +} + void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, const VkClearValue* clear_values, u32 clear_value_count) { - const VkFramebuffer framebuffer_handle = framebuffer->Handle(); + if (device.IsKhrDynamicRenderingSupported()) { + BeginDynamicRendering(framebuffer, nullptr); + return; + } const VkExtent2D render_area = framebuffer->RenderArea(); + const VkFramebuffer framebuffer_handle = framebuffer->Handle(); state.renderpass = renderpass; state.framebuffer = framebuffer_handle; state.render_area = render_area; + state.rendering = true; if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { const std::string render_pass_info = @@ -141,6 +216,12 @@ void Scheduler::RealizeDeferredClear() { const DeferredClear dc = deferred_clear; deferred_clear = {}; + if (device.IsKhrDynamicRenderingSupported()) { + EndRenderPass(); + BeginDynamicRendering(dc.framebuffer, &dc); + return; + } + std::array clear_values{}; u32 count = 0; const RenderPassKey& base = dc.framebuffer->RenderPassKeyBase(); @@ -195,9 +276,25 @@ void Scheduler::RequestRenderpass(const Framebuffer* framebuffer) { RealizeDeferredClear(); return; } + const VkExtent2D render_area = framebuffer->RenderArea(); + if (device.IsKhrDynamicRenderingSupported()) { + std::array attachment_views{}; + const auto& color_views = framebuffer->ColorAttachments(); + for (size_t index = 0; index < color_views.size(); ++index) { + attachment_views[index] = color_views[index]; + } + attachment_views[8] = framebuffer->DepthAttachment(); + if (state.rendering && attachment_views == state.attachment_views && + render_area.width == state.render_area.width && + render_area.height == state.render_area.height) { + return; + } + EndRenderPass(); + BeginDynamicRendering(framebuffer, nullptr); + return; + } const VkRenderPass renderpass = framebuffer->RenderPass(); const VkFramebuffer framebuffer_handle = framebuffer->Handle(); - const VkExtent2D render_area = framebuffer->RenderArea(); if (renderpass == state.renderpass && framebuffer_handle == state.framebuffer && render_area.width == state.render_area.width && render_area.height == state.render_area.height) { @@ -322,6 +419,7 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se InvalidateState(); const u64 signal_value = master_semaphore->NextTick(); + vk::SetDeletionTimeline(master_semaphore->CurrentTick()); RecordWithUploadBuffer([signal_semaphore, wait_semaphore, signal_value, this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { static constexpr VkMemoryBarrier WRITE_BARRIER{ @@ -372,6 +470,155 @@ void Scheduler::InvalidateState() { state_tracker.InvalidateCommandBufferState(); } +namespace { +/// The part of a dynamic rendering begin that every pass needs. +struct DynamicRenderingBase { + std::array views; + VkExtent2D render_area; + u32 num_color; + u32 layers; + VkImageView ds_resolve_view; + VkResolveModeFlagBits depth_resolve_mode; + VkResolveModeFlagBits stencil_resolve_mode; + bool has_depth; + bool has_stencil; + bool ds_discard; +}; + +/// The part only a pass that actually clears something needs. +struct DynamicRenderingClears { + std::array color_values; + VkClearValue ds_value; + u32 color_clear_mask; + u32 color_discard_mask; + bool ds_clear; +}; + +/// Builds the rendering info and begins the instance. `resolve_views`/`resolve_modes` and `clears` +/// are null for passes with no colour resolve targets and no clears, so those tables never have to +/// be copied into the recorded command. +void IssueBeginRendering(vk::CommandBuffer cmdbuf, const DynamicRenderingBase& base, + const std::array* resolve_views, + const std::array* resolve_modes, + const DynamicRenderingClears* clears) { + std::array color_infos{}; + for (u32 index = 0; index < base.num_color; ++index) { + const bool clear_slot = + clears != nullptr && ((clears->color_clear_mask >> index) & 1u) != 0; + const VkImageView resolve_view = + resolve_views != nullptr ? (*resolve_views)[index] : VK_NULL_HANDLE; + const bool has_resolve = resolve_view != VK_NULL_HANDLE; + const bool discard_slot = + has_resolve && clears != nullptr && ((clears->color_discard_mask >> index) & 1u) != 0; + color_infos[index] = VkRenderingAttachmentInfo{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = base.views[index], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = has_resolve ? (*resolve_modes)[index] : VK_RESOLVE_MODE_NONE, + .resolveImageView = resolve_view, + .resolveImageLayout = + has_resolve ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = clear_slot ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = discard_slot ? VK_ATTACHMENT_STORE_OP_DONT_CARE + : VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = clear_slot ? clears->color_values[index] : VkClearValue{}, + }; + } + const bool ds_clear = clears != nullptr && clears->ds_clear; + const VkClearValue ds_clear_value = clears != nullptr ? clears->ds_value : VkClearValue{}; + const bool has_ds_resolve = base.ds_resolve_view != VK_NULL_HANDLE; + const VkAttachmentLoadOp ds_load_op = ds_clear ? VK_ATTACHMENT_LOAD_OP_CLEAR + : base.ds_discard ? VK_ATTACHMENT_LOAD_OP_DONT_CARE + : VK_ATTACHMENT_LOAD_OP_LOAD; + const VkAttachmentStoreOp ds_store_op = + base.ds_discard ? VK_ATTACHMENT_STORE_OP_DONT_CARE : VK_ATTACHMENT_STORE_OP_STORE; + const VkRenderingAttachmentInfo depth_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = base.views[8], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = has_ds_resolve ? base.depth_resolve_mode : VK_RESOLVE_MODE_NONE, + .resolveImageView = has_ds_resolve ? base.ds_resolve_view : VK_NULL_HANDLE, + .resolveImageLayout = has_ds_resolve ? VK_IMAGE_LAYOUT_GENERAL + : VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = ds_load_op, + .storeOp = ds_store_op, + .clearValue = ds_clear ? ds_clear_value : VkClearValue{}, + }; + // Stencil gets its own struct because its resolve mode may differ from depth's. + const VkRenderingAttachmentInfo stencil_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = base.views[8], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = has_ds_resolve ? base.stencil_resolve_mode : VK_RESOLVE_MODE_NONE, + .resolveImageView = has_ds_resolve ? base.ds_resolve_view : VK_NULL_HANDLE, + .resolveImageLayout = has_ds_resolve ? VK_IMAGE_LAYOUT_GENERAL + : VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = ds_load_op, + .storeOp = ds_store_op, + .clearValue = ds_clear ? ds_clear_value : VkClearValue{}, + }; + const VkRenderingInfo rendering_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, + .pNext = nullptr, + .flags = 0, + .renderArea = + { + .offset = {.x = 0, .y = 0}, + .extent = base.render_area, + }, + .layerCount = base.layers, + .viewMask = 0, + .colorAttachmentCount = base.num_color, + .pColorAttachments = color_infos.data(), + .pDepthAttachment = base.has_depth ? &depth_info : nullptr, + .pStencilAttachment = base.has_stencil ? &stencil_info : nullptr, + }; + cmdbuf.BeginRendering(rendering_info); +} +} // Anonymous namespace + +void Scheduler::RecordDynamicBegin(const DeferredClear* clear) { + const DynamicRenderingBase base{ + .views = state.attachment_views, + .render_area = state.render_area, + .num_color = state.num_color, + .layers = state.layer_count, + .ds_resolve_view = state.depth_resolve_view, + .depth_resolve_mode = state.depth_resolve_mode, + .stencil_resolve_mode = state.stencil_resolve_mode, + .has_depth = state.has_depth, + .has_stencil = state.has_stencil, + .ds_discard = state.discards_msaa_depth, + }; + bool has_color_resolve = false; + for (const VkImageView resolve_view : state.color_resolve_views) { + has_color_resolve = has_color_resolve || resolve_view != VK_NULL_HANDLE; + } + if (clear == nullptr && !has_color_resolve) { + // Common case. Every byte captured here is copied into the command chunk on every single + // pass begin, and the clear values plus the two resolve tables are two thirds of them. + Record([base](vk::CommandBuffer cmdbuf) { + IssueBeginRendering(cmdbuf, base, nullptr, nullptr, nullptr); + }); + return; + } + const DynamicRenderingClears clears{ + .color_values = clear != nullptr ? clear->color_values : std::array{}, + .ds_value = clear != nullptr ? clear->depth_stencil_value : VkClearValue{}, + .color_clear_mask = clear != nullptr ? clear->color_clear_mask : 0u, + .color_discard_mask = + clear != nullptr && state.discards_msaa_color ? clear->color_clear_mask : 0u, + .ds_clear = clear != nullptr && clear->depth_stencil, + }; + Record([base, resolve_views = state.color_resolve_views, + resolve_modes = state.color_resolve_modes, clears](vk::CommandBuffer cmdbuf) { + IssueBeginRendering(cmdbuf, base, &resolve_views, &resolve_modes, &clears); + }); +} + void Scheduler::EndPendingOperations() { query_cache->CounterReset(VideoCommon::QueryType::ZPassPixelCount64); EndRenderPass(); @@ -380,7 +627,10 @@ void Scheduler::EndPendingOperations() { void Scheduler::EndRenderPass() { RealizeDeferredClear(); - if (!state.renderpass) { + if (query_cache) { + query_cache->NotifySegment(false); + } + if (!state.rendering) { return; } @@ -393,14 +643,24 @@ void Scheduler::EndRenderPass() } query_cache->CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, false); - query_cache->NotifySegment(false); + + if (pending_begin) { + if (!has_pending_begin_clear) { + RetractUnrecordedRenderPass(); + return; + } + FlushPendingRenderPass(); + } Record([num_images = num_renderpass_images, images = renderpass_images, ranges = renderpass_image_ranges, - has_transform_feedback = device.IsExtTransformFeedbackSupported()]( + has_transform_feedback = state.uses_transform_feedback, + dynamic_rendering = device.IsKhrDynamicRenderingSupported()]( vk::CommandBuffer cmdbuf) { + const bool use_sync2 = cmdbuf.HasPipelineBarrier2(); std::array barriers; + std::array barriers2; for (size_t i = 0; i < num_images; ++i) { const VkImageSubresourceRange& range = ranges[i]; const bool is_color = (range.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) != 0; @@ -418,15 +678,57 @@ void Scheduler::EndRenderPass() src_access |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - barriers[i] = VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + if (!use_sync2) { + barriers[i] = VkImageMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT + | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT + | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT + | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = images[i], + .subresourceRange = range, + }; + continue; + } + + VkPipelineStageFlags2 src_stage = 0; + VkAccessFlags2 dst_access = + VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT; + if (is_color) { + src_stage |= VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT; + dst_access |= VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT; + } + if (is_depth_stencil) { + src_stage |= VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT + | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT; + dst_access |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT + | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + } + if (!is_color && !is_depth_stencil) { + src_stage = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT + | VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT + | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT; + dst_access |= VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT + | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT + | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + } + barriers2[i] = VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = src_access, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT - | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT - | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, + .srcStageMask = src_stage, + .srcAccessMask = static_cast(src_access), + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT + | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = dst_access, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -435,10 +737,19 @@ void Scheduler::EndRenderPass() .subresourceRange = range, }; } - cmdbuf.EndRenderPass(); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, nullptr, nullptr, vk::Span(barriers.data(), num_images)); + if (dynamic_rendering) { + cmdbuf.EndRendering(); + } else { + cmdbuf.EndRenderPass(); + } + if (use_sync2) { + cmdbuf.PipelineBarrier2(0, nullptr, nullptr, + vk::Span(barriers2.data(), num_images)); + } else { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, + 0, nullptr, nullptr, vk::Span(barriers.data(), num_images)); + } if (has_transform_feedback) { static constexpr VkMemoryBarrier XFB_OUTPUT_BARRIER{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, @@ -453,6 +764,10 @@ void Scheduler::EndRenderPass() }); state.renderpass = VkRenderPass{}; + state.framebuffer = VkFramebuffer{}; + state.attachment_views = {}; + state.rendering = false; + state.uses_transform_feedback = false; num_renderpass_images = 0; } diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 5c4af8c146..5a48b50a11 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -71,7 +71,12 @@ public: /// Returns true when a render pass is currently active in the scheduler state. bool IsRenderPassActive() const { - return state.renderpass != VK_NULL_HANDLE; + return state.rendering; + } + + /// Flags that transform feedback writes have been recorded since the last render pass end. + void MarkTransformFeedbackUsed() noexcept { + state.uses_transform_feedback = true; } /// Update the pipeline to the current execution context. @@ -107,12 +112,17 @@ public: template requires std::is_invocable_v void Record(T&& c) { + if (pending_begin) { + FlushPendingRenderPass(); + } this->RecordWithUploadBuffer( [command = std::move(c)](vk::CommandBuffer cmdbuf, vk::CommandBuffer) { command(cmdbuf); }); } + bool RetractUnrecordedRenderPass(); + /// Returns the current command buffer tick. [[nodiscard]] u64 CurrentTick() const noexcept { return master_semaphore->CurrentTick(); @@ -131,33 +141,16 @@ public: } master_semaphore->Wait(tick); } - if (Settings::values.use_speed_limit.GetValue() && target_fps > 0.0) { - auto now = std::chrono::steady_clock::now(); - if (last_target_fps != target_fps) { - frame_interval = std::chrono::duration_cast(std::chrono::duration(1.0 / target_fps)); - max_frame_count = static_cast(0.1 * target_fps); - last_target_fps = target_fps; - frame_counter = 0; - start_time = now; - } - frame_counter++; - auto target_time = start_time + frame_interval * frame_counter; - if (target_time >= now) { - auto sleep_time = target_time - now; - if (sleep_time > std::chrono::milliseconds(15)) { - std::this_thread::sleep_for(sleep_time - std::chrono::milliseconds(1)); - } - while (std::chrono::steady_clock::now() < target_time) { - std::this_thread::yield(); - } - } else if (frame_counter > max_frame_count) { - frame_counter = 0; - start_time = now; - } - } + ApplyFramePacing(target_fps); + } + + void WaitSubmitted(u64 tick, double target_fps = 0.0) { + if (tick > 0 && tick < master_semaphore->CurrentTick()) { + master_semaphore->Wait(tick); + } + ApplyFramePacing(target_fps); } - /// Returns the master timeline semaphore. [[nodiscard]] MasterSemaphore& GetMasterSemaphore() const noexcept { return *master_semaphore; } @@ -165,6 +158,35 @@ public: std::mutex submit_mutex; private: + void ApplyFramePacing(double target_fps) { + if (!Settings::values.use_speed_limit.GetValue() || target_fps <= 0.0) { + return; + } + auto now = std::chrono::steady_clock::now(); + if (last_target_fps != target_fps) { + frame_interval = std::chrono::duration_cast( + std::chrono::duration(1.0 / target_fps)); + max_frame_count = static_cast(0.1 * target_fps); + last_target_fps = target_fps; + frame_counter = 0; + start_time = now; + } + frame_counter++; + auto target_time = start_time + frame_interval * frame_counter; + if (target_time >= now) { + auto sleep_time = target_time - now; + if (sleep_time > std::chrono::milliseconds(15)) { + std::this_thread::sleep_for(sleep_time - std::chrono::milliseconds(1)); + } + while (std::chrono::steady_clock::now() < target_time) { + std::this_thread::yield(); + } + } else if (frame_counter > max_frame_count) { + frame_counter = 0; + start_time = now; + } + } + class Command { public: virtual ~Command() = default; @@ -244,17 +266,31 @@ private: size_t command_offset = 0; bool submit = false; - alignas(std::max_align_t) std::array data{}; + alignas(std::max_align_t) std::array data{}; }; struct State { VkRenderPass renderpass{}; VkFramebuffer framebuffer{}; + std::array attachment_views{}; + std::array color_resolve_views{}; + std::array color_resolve_modes{}; + VkImageView depth_resolve_view{}; + VkResolveModeFlagBits depth_resolve_mode = VK_RESOLVE_MODE_NONE; + VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_NONE; VkExtent2D render_area = {0, 0}; GraphicsPipeline* graphics_pipeline = nullptr; + bool rendering = false; + bool discards_msaa_color = false; + bool discards_msaa_depth = false; + u32 num_color = 0; + bool has_depth = false; + bool has_stencil = false; + u32 layer_count = 1; bool is_rescaling = false; bool rescaling_defined = false; bool needs_state_enable_refresh = false; + bool uses_transform_feedback = false; }; struct DeferredClear { @@ -269,6 +305,9 @@ private: void BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, const VkClearValue* clear_values, u32 clear_value_count); + /// Begins a dynamic rendering pass, optionally realizing a deferred clear via load ops. + void BeginDynamicRendering(const Framebuffer* framebuffer, const DeferredClear* clear); + /// If a deferred clear is pending. void RealizeDeferredClear(); @@ -282,6 +321,10 @@ private: void EndPendingOperations(); + void RecordDynamicBegin(const DeferredClear* clear); + + void FlushPendingRenderPass(); + void EndRenderPass(); void AcquireNewChunk(); @@ -299,6 +342,10 @@ private: DeferredClear deferred_clear; + bool pending_begin = false; + bool has_pending_begin_clear = false; + DeferredClear pending_begin_clear; + std::unique_ptr chunk; std::function on_submit; diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index b03449f9f7..dfa374fe48 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -84,10 +84,16 @@ StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& mem if (device.IsExtTransformFeedbackSupported()) { stream_ci.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + stream_ci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } stream_buffer = memory_allocator.CreateBuffer(stream_ci, MemoryUsage::Stream); if (device.HasDebuggingToolAttached()) { stream_buffer.SetObjectNameEXT("Stream Buffer"); } + if (device.IsBufferDeviceAddressSupported()) { + stream_buffer_address = device.GetLogical().GetBufferDeviceAddress(*stream_buffer); + } stream_pointer = stream_buffer.Mapped(); ASSERT_MSG(!stream_pointer.empty(), "Stream buffer must be host visible!"); } @@ -149,6 +155,7 @@ StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) { iterator = Common::AlignUp(iterator + size, MAX_ALIGNMENT); return StagingBufferRef{ .buffer = *stream_buffer, + .device_address = stream_buffer_address, .offset = static_cast(offset), .mapped_span = stream_pointer.subspan(offset, size), .usage{}, @@ -212,14 +219,22 @@ StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, MemoryUsage if (device.IsExtTransformFeedbackSupported()) { buffer_ci.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + buffer_ci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } vk::Buffer buffer = memory_allocator.CreateBuffer(buffer_ci, usage); if (device.HasDebuggingToolAttached()) { ++buffer_index; buffer.SetObjectNameEXT(fmt::format("Staging Buffer {}", buffer_index).c_str()); } const std::span mapped_span = buffer.Mapped(); + const VkDeviceAddress buffer_address = + device.IsBufferDeviceAddressSupported() + ? device.GetLogical().GetBufferDeviceAddress(*buffer) + : VkDeviceAddress{}; StagingBuffer& entry = GetCache(usage)[log2_size].entries.emplace_back(StagingBuffer{ .buffer = std::move(buffer), + .device_address = buffer_address, .mapped_span = mapped_span, .usage = usage, .log2_level = log2_size, @@ -252,25 +267,62 @@ void StagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, size_t log2) { constexpr size_t deletions_per_tick = 16; auto& staging = cache[log2]; auto& entries = staging.entries; - const size_t old_size = entries.size(); + if (entries.empty()) { + staging.delete_index = 0; + staging.iterate_index = 0; + return; + } const auto is_deletable = [this](const StagingBuffer& entry) { return scheduler.IsFree(entry.tick); }; - const size_t begin_offset = staging.delete_index; - const size_t end_offset = (std::min)(begin_offset + deletions_per_tick, old_size); + const size_t begin_offset = (std::min)(staging.delete_index, entries.size()); + const size_t end_offset = (std::min)(begin_offset + deletions_per_tick, entries.size()); const auto begin = entries.begin() + begin_offset; const auto end = entries.begin() + end_offset; - entries.erase(std::remove_if(begin, end, is_deletable), end); + const auto surviving_end = std::remove_if(begin, end, is_deletable); + const size_t removed = static_cast(std::distance(surviving_end, end)); + entries.erase(surviving_end, end); - const size_t new_size = entries.size(); - staging.delete_index += deletions_per_tick; - if (staging.delete_index >= new_size) { + staging.delete_index = end_offset - removed; + if (staging.delete_index >= entries.size()) { staging.delete_index = 0; } - if (staging.iterate_index > new_size) { + if (staging.iterate_index > entries.size()) { staging.iterate_index = 0; } } +u64 StagingBufferPool::ReclaimMemory(u64 target_bytes) { + u64 freed = 0; + const auto is_deletable = [this](const StagingBuffer& entry) { + return scheduler.IsFree(entry.tick); + }; + const auto reclaim_cache = [&](StagingBuffersCache& cache) { + for (size_t level = NUM_LEVELS; level-- > 0 && freed < target_bytes;) { + auto& staging = cache[level]; + auto& entries = staging.entries; + if (entries.empty()) { + continue; + } + const u64 entry_bytes = 1ULL << level; + auto it = entries.begin(); + while (it != entries.end() && freed < target_bytes) { + if (is_deletable(*it)) { + it = entries.erase(it); + freed += entry_bytes; + } else { + ++it; + } + } + staging.delete_index = 0; + staging.iterate_index = 0; + } + }; + reclaim_cache(device_local_cache); + reclaim_cache(upload_cache); + reclaim_cache(download_cache); + return freed; +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h index f63a203272..9d05d1cca6 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later @@ -18,6 +21,7 @@ class Scheduler; struct StagingBufferRef { VkBuffer buffer; + VkDeviceAddress device_address; VkDeviceSize offset; std::span mapped_span; MemoryUsage usage; @@ -42,6 +46,8 @@ public: void TickFrame(); + u64 ReclaimMemory(u64 target_bytes); + private: struct StreamBufferCommit { size_t upper_bound; @@ -50,6 +56,7 @@ private: struct StagingBuffer { vk::Buffer buffer; + VkDeviceAddress device_address; std::span mapped_span; MemoryUsage usage; u32 log2_level; @@ -60,6 +67,7 @@ private: StagingBufferRef Ref() const noexcept { return { .buffer = *buffer, + .device_address = device_address, .offset = 0, .mapped_span = mapped_span, .usage = usage, @@ -103,6 +111,7 @@ private: Scheduler& scheduler; vk::Buffer stream_buffer; + VkDeviceAddress stream_buffer_address{}; std::span stream_pointer; VkDeviceSize stream_buffer_size; VkDeviceSize region_size; diff --git a/src/video_core/renderer_vulkan/vk_swapchain.cpp b/src/video_core/renderer_vulkan/vk_swapchain.cpp index fdbad16721..507c62b8f0 100644 --- a/src/video_core/renderer_vulkan/vk_swapchain.cpp +++ b/src/video_core/renderer_vulkan/vk_swapchain.cpp @@ -9,10 +9,6 @@ #include #include -#ifdef __ANDROID__ -#include -#endif - #include "common/logging.h" #include "common/settings.h" #include "common/settings_enums.h" @@ -176,34 +172,26 @@ bool Swapchain::AcquireNextImage() { break; } - const auto wait_with_frame_pacing = [this] { +#ifdef __ANDROID__ + scheduler.WaitSubmitted(resource_ticks[image_index]); +#else switch (Settings::values.frame_pacing_mode.GetValue()) { case Settings::FramePacingMode::Target_Auto: - scheduler.Wait(resource_ticks[image_index]); + scheduler.WaitSubmitted(resource_ticks[image_index]); break; case Settings::FramePacingMode::Target_30: - scheduler.Wait(resource_ticks[image_index], 30.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 30.0); break; case Settings::FramePacingMode::Target_60: - scheduler.Wait(resource_ticks[image_index], 60.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 60.0); break; case Settings::FramePacingMode::Target_90: - scheduler.Wait(resource_ticks[image_index], 90.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 90.0); break; case Settings::FramePacingMode::Target_120: - scheduler.Wait(resource_ticks[image_index], 120.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 120.0); break; } - }; - -#ifdef __ANDROID__ - if (android_get_device_api_level() >= 30) { - scheduler.Wait(resource_ticks[image_index]); - } else { - wait_with_frame_pacing(); - } -#else - wait_with_frame_pacing(); #endif resource_ticks[image_index] = scheduler.CurrentTick(); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 44a4227a19..6bd99a0f2a 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -54,6 +54,8 @@ using VideoCore::Surface::SurfaceType; namespace { constexpr bool ENABLE_MSAA_RESOLVE_CONSUME = true; constexpr bool ENABLE_MSAA_COLOR_DISCARD = true; +constexpr bool ENABLE_MSAA_DEPTH_DISCARD = true; +constexpr bool ENABLE_MSAA_DEPTH_RESOLVE = false; constexpr VkBorderColor ConvertBorderColor(const std::array& color) { if (color == std::array{0, 0, 0, 0}) { @@ -144,11 +146,6 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { info.size.depth == 1; } -[[nodiscard]] bool WillUseWidenedAstcFormat(const Device& device, const ImageInfo& info) { - return WillUseAcceleratedAstcDecode(device, info) && - !VideoCore::Surface::IsPixelFormatSRGB(info.format); -} - [[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info, std::optional format_override = {}) { auto format_info = @@ -212,7 +209,11 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return device.IsFormatSupported(view_format, VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT, FormatType::Optimal); }); - if (has_storage_compatible_view) { + const bool storage_allowed_for_samples = + image_ci.samples == VK_SAMPLE_COUNT_1_BIT || + (device.GetStorageImageSampleCounts() & + static_cast(image_ci.samples)) != 0; + if (has_storage_compatible_view && storage_allowed_for_samples) { image_ci.usage |= VK_IMAGE_USAGE_STORAGE_BIT; } @@ -269,6 +270,10 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { } } +[[nodiscard]] bool IsLdrAstcFormat(VkFormat format) { + return format >= VK_FORMAT_ASTC_4x4_UNORM_BLOCK && format <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK; +} + [[nodiscard]] VkImageAspectFlags ImageViewAspectMask(const VideoCommon::ImageViewInfo& info) { if (info.IsRenderTarget()) { return ImageAspectMask(info.format); @@ -896,6 +901,15 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const 0, nullptr, nullptr, write_barriers); }); } + +[[nodiscard]] bool CanBlitNatively(const Device& device, PixelFormat format) { + static constexpr auto OPTIMAL_FORMAT = FormatType::Optimal; + static constexpr VkFormatFeatureFlags BLIT_USAGE = + VK_FORMAT_FEATURE_BLIT_SRC_BIT | VK_FORMAT_FEATURE_BLIT_DST_BIT; + const VkFormat vk_format = + MaxwellToVK::SurfaceFormat(device, OPTIMAL_FORMAT, false, format).format; + return device.IsFormatSupported(vk_format, BLIT_USAGE, OPTIMAL_FORMAT); +} } // Anonymous namespace TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& scheduler_, @@ -934,6 +948,10 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched bl3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue); } + bl2d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); + bl3db_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); } void TextureCacheRuntime::Finish() { @@ -996,13 +1014,21 @@ VkBuffer TextureCacheRuntime::GetTemporaryBuffer(size_t needed_size) { } VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, VkFormat format, - VkExtent2D extent, u32 layers) { + VkExtent2D extent, u32 layers, + VkImageAspectFlags aspect) { + const bool is_depth_stencil = + (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0; ResolveShadow& shadow = resolve_shadows[msaa_image]; if (shadow.image && shadow.format == format && shadow.extent.width == extent.width && shadow.extent.height == extent.height && shadow.layers == layers) { shadow.up_to_date = true; return *shadow.view; } + const VkImageUsageFlags shadow_usage = + static_cast(is_depth_stencil + ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT + : VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) | + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; shadow.image = memory_allocator.CreateImage(VkImageCreateInfo{ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .pNext = nullptr, @@ -1014,8 +1040,7 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk .arrayLayers = layers, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, - .usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | - VK_IMAGE_USAGE_TRANSFER_SRC_BIT, + .usage = shadow_usage, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -1030,7 +1055,7 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk .format = format, .components{}, .subresourceRange{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, @@ -1041,6 +1066,38 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk shadow.extent = extent; shadow.layers = layers; shadow.up_to_date = true; + if (device.IsKhrDynamicRenderingSupported()) { + const VkAccessFlags dst_access = + is_depth_stencil ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + : VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const VkPipelineStageFlags dst_stage = + is_depth_stencil ? VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT + : VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + scheduler.RecordWithUploadBuffer( + [image = *shadow.image, layers, aspect, dst_access, dst_stage]( + vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + const VkImageMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = dst_access, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = layers, + }, + }; + upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, dst_stage, 0, + barrier); + }); + } return *shadow.view; } @@ -1228,27 +1285,19 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst blit_image_helper.ResolveDepthStencil(dst_framebuffer, src, dst_region, src_region); return; } - if (aspect_mask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { - const auto format = src.format; - const auto can_blit_depth_stencil = [this, format] { - switch (format) { - case VideoCore::Surface::PixelFormat::D24_UNORM_S8_UINT: - case VideoCore::Surface::PixelFormat::S8_UINT_D24_UNORM: - return device.IsBlitDepth24Stencil8Supported(); - case VideoCore::Surface::PixelFormat::D32_FLOAT_S8_UINT: - return device.IsBlitDepth32Stencil8Supported(); - default: - UNREACHABLE(); - } - }(); - // Use shader-based depth/stencil blits if hardware doesn't support the format - // Note: MSAA resolves (MSAA->single) use vkCmdResolveImage which works fine - if (!can_blit_depth_stencil) { - UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa); - blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region, - filter, operation); + static constexpr VkImageAspectFlags DEPTH_STENCIL_ASPECTS = + VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT; + if ((aspect_mask & DEPTH_STENCIL_ASPECTS) != 0 && !CanBlitNatively(device, src.format)) { + if (aspect_mask != DEPTH_STENCIL_ASPECTS) { + UNIMPLEMENTED_MSG("Host cannot blit format {} and no helper path exists for aspect " + "mask 0x{:x}", + src.format, aspect_mask); return; } + UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa); + blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region, filter, + operation); + return; } ASSERT(!(is_dst_msaa && !is_src_msaa)); ASSERT(operation == Fermi2D::Operation::SrcCopy); @@ -1353,7 +1402,7 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst } void TextureCacheRuntime::ConvertImage(Framebuffer* dst, ImageView& dst_view, ImageView& src_view) { - if (!dst->RenderPass()) { + if (!dst->RenderPass() && !device.IsKhrDynamicRenderingSupported()) { return; } @@ -1641,11 +1690,6 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, std::span copies) { const bool msaa_to_non_msaa = src.info.num_samples > 1 && dst.info.num_samples == 1; const u32 num_samples = msaa_to_non_msaa ? src.info.num_samples : dst.info.num_samples; - if (dst.AspectMask() != VK_IMAGE_ASPECT_COLOR_BIT || - VideoCore::Surface::IsPixelFormatInteger(dst.info.format)) { - UNIMPLEMENTED_MSG("Copying images with different samples is not supported."); - return; - } if (ENABLE_MSAA_RESOLVE_CONSUME && msaa_to_non_msaa && copies.size() == 1 && src.info.format == dst.info.format) { const VideoCommon::ImageCopy& copy = copies.front(); @@ -1656,16 +1700,19 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, static_cast(copy.extent.height) <= shadow->extent.height) { const VkImage shadow_image = *shadow->image; const VkImage dst_image = dst.Handle(); + const VkImageAspectFlags aspect = dst.AspectMask(); + const bool is_depth_stencil = + (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0; const VkImageCopy region{ .srcSubresource{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect, .mipLevel = 0, .baseArrayLayer = static_cast(copy.src_subresource.base_layer), .layerCount = static_cast(copy.src_subresource.num_layers), }, .srcOffset = {0, 0, 0}, .dstSubresource{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect, .mipLevel = static_cast(copy.dst_subresource.base_level), .baseArrayLayer = static_cast(copy.dst_subresource.base_layer), .layerCount = static_cast(copy.dst_subresource.num_layers), @@ -1674,34 +1721,38 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, .extent = {copy.extent.width, copy.extent.height, 1}, }; scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([shadow_image, dst_image, region](vk::CommandBuffer cmdbuf) { + scheduler.Record([shadow_image, dst_image, region, aspect, + is_depth_stencil](vk::CommandBuffer cmdbuf) { + const VkAccessFlags attachment_write = + is_depth_stencil ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + : VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; const std::array pre_barriers{ VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .srcAccessMask = attachment_write, .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = shadow_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, + .srcAccessMask = + VK_ACCESS_SHADER_WRITE_BIT | attachment_write | + VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, }; @@ -1716,7 +1767,7 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = shadow_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, VkImageMemoryBarrier{ @@ -1726,17 +1777,20 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, }; cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, nullptr, nullptr, pre_barriers); @@ -1749,10 +1803,36 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, return; } } + if (dst.AspectMask() != VK_IMAGE_ASPECT_COLOR_BIT || + VideoCore::Surface::IsPixelFormatInteger(dst.info.format)) { + const u64 key{(static_cast(dst.AspectMask()) << 32) | + static_cast(dst.info.format)}; + if (unsupported_msaa_resolves.insert(key).second) { + LOG_WARNING(Render_Vulkan, + "MSAA resolve unsupported: format={}, aspect={:#x}, samples {}->{}", + dst.info.format, dst.AspectMask(), src.info.num_samples, + dst.info.num_samples); + } + return; + } blit_image_helper.CopyMSAA(render_pass_cache, dst.Handle(), dst.info.format, src.Handle(), src.info.format, num_samples, copies, msaa_to_non_msaa); } +u64 TextureCacheRuntime::CurrentSyncPoint() const noexcept { + return scheduler.CurrentTick(); +} + +u64 TextureCacheRuntime::CompletedSyncPoint() const { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + return master_semaphore.KnownGpuTick(); +} + +void TextureCacheRuntime::WaitSyncPoint(u64 sync_point) { + scheduler.Wait(sync_point); +} + u64 TextureCacheRuntime::GetDeviceLocalMemory() const { return device.GetDeviceLocalMemory(); } @@ -1761,6 +1841,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::GetDeviceAllocationUsage() const { + return device.GetMemoryBudgetInfo().allocation_bytes; +} + bool TextureCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -1770,6 +1854,7 @@ std::optional TextureCacheRuntime::GetSamplerHeapBudget() const { } void TextureCacheRuntime::TickFrame() { + device.TickAllocatorFrame(); std::erase_if(pending_msaa_images, [this](const auto& pending) { return scheduler.IsFree(pending.first); }); @@ -1780,12 +1865,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, runtime{&runtime_}, original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info, - WillUseWidenedAstcFormat(runtime_.device, info) - ? std::span{} - : runtime->ViewFormats(info.format), - WillUseWidenedAstcFormat(runtime_.device, info) - ? std::make_optional(VK_FORMAT_R32G32B32A32_SFLOAT) - : std::nullopt)), + runtime->ViewFormats(info.format))), aspect_mask(ImageAspectMask(info.format)) { if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { switch (Settings::values.accelerate_astc.GetValue()) { @@ -1800,7 +1880,11 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu default: break; } - flags |= VideoCommon::ImageFlagBits::Converted; + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + flags |= VideoCommon::ImageFlagBits::CostlyLoad; + } else if (runtime->bl3db_unswizzle_pass && + BlockLinearUnswizzle3DBufferPass::IsSupported(runtime->device, info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; flags |= VideoCommon::ImageFlagBits::CostlyLoad; } if (IsPixelFormatBCn(info.format) && !runtime->device.IsOptimalBcnSupported()) { @@ -1816,9 +1900,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Uncompressed) { const auto& device = runtime->device.GetLogical(); - const VkFormat storage_format = WillUseWidenedAstcFormat(runtime->device, info) - ? VK_FORMAT_R32G32B32A32_SFLOAT - : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; for (s32 level = 0; level < info.resources.levels; ++level) { storage_image_views[level] = MakeStorageView(device, level, *original_image, storage_format); @@ -1892,9 +1974,11 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, ScaleDown(true); } - const bool wants_msaa_upload = info.num_samples > 1 - && (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0 + const bool is_color_upload = (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0 && !VideoCore::Surface::IsPixelFormatInteger(info.format); + const bool is_depth_upload = (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0; + const bool wants_msaa_upload = + info.num_samples > 1 && (is_color_upload || is_depth_upload); if (wants_msaa_upload) { ImageInfo temp_info = info; @@ -1933,10 +2017,10 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, image_copies.push_back(image_copy); } + runtime->TransitionImageLayout(*this); runtime->blit_image_helper.CopyMSAA(runtime->render_pass_cache, Handle(), info.format, temp_vk_image, info.format, info.num_samples, image_copies, false); - initialized = true; runtime->pending_msaa_images.emplace_back(scheduler->CurrentTick(), std::move(temp_image)); if (is_rescaled) { @@ -1947,6 +2031,9 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, if (info.num_samples > 1) { LOG_WARNING(Render_Vulkan, "MSAA upload not implemented for format {}", info.format); + if (runtime != nullptr) { + runtime->TransitionImageLayout(*this); + } if (is_rescaled) { ScaleUp(); } @@ -2204,9 +2291,7 @@ VkImageView Image::StorageImageView(s32 level) noexcept { auto format_info = MaxwellToVK::SurfaceFormat(runtime->device, FormatType::Optimal, true, info.format); if (WillUseAcceleratedAstcDecode(runtime->device, info)) { - format_info.format = WillUseWidenedAstcFormat(runtime->device, info) - ? VK_FORMAT_R32G32B32A32_SFLOAT - : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + format_info.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; } view = MakeStorageView(runtime->device.GetLogical(), level, *(this->*current_image), format_info.format); @@ -2382,11 +2467,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI SanitizeDepthStencilSwizzle(swizzle, device->SupportsDepthStencilSwizzleOne()); } } - uses_widened_astc_format = WillUseWidenedAstcFormat(*device, image.info); auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); - if (uses_widened_astc_format) { - format_info.format = VK_FORMAT_R32G32B32A32_SFLOAT; - } if (device->ApiVersion() >= VK_API_VERSION_1_3) { const VkFormatProperties3 properties3 = device->GetPhysical().GetFormatProperties3(format_info.format); @@ -2404,9 +2485,18 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI .pNext = nullptr, .usage = clamped_view_usage, }; + const VkImageViewASTCDecodeModeEXT astc_decode_mode{ + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_ASTC_DECODE_MODE_EXT, + .pNext = &image_view_usage, + .decodeMode = VK_FORMAT_R8G8B8A8_UNORM, + }; + const void* view_next = &image_view_usage; + if (device->IsExtAstcDecodeModeSupported() && IsLdrAstcFormat(format_info.format)) { + view_next = &astc_decode_mode; + } const VkImageViewCreateInfo create_info{ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, - .pNext = &image_view_usage, + .pNext = view_next, .flags = 0, .image = image.Handle(), .viewType = VkImageViewType{}, @@ -2529,14 +2619,12 @@ VkImageView ImageView::StorageView(Shader::TextureType texture_type, Shader::ImageFormat image_format) { if (image_handle) { if (image_format == Shader::ImageFormat::Typeless) { - if (!typeless_storage_view) { + auto& view{typeless_storage_views[static_cast(texture_type)]}; + if (!view) { auto info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); - if (uses_widened_astc_format) { - info.format = VK_FORMAT_R32G32B32A32_SFLOAT; - } - typeless_storage_view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type); + view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type); } - return *typeless_storage_view; + return *view; } const bool is_signed = image_format == Shader::ImageFormat::R8_SINT || image_format == Shader::ImageFormat::R16_SINT; @@ -2683,7 +2771,7 @@ Framebuffer::Framebuffer(TextureCacheRuntime& runtime, std::spanSamples(); + color_attachments[index] = color_buffer->RenderTarget(); + color_attachment_formats[index] = + MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, + color_buffer->format) + .format; + num_color_attachments = static_cast(index + 1); ++num_images; } const size_t num_colors = attachments.size(); @@ -2741,6 +2835,11 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, const VkImageSubresourceRange subresource_range = MakeSubresourceRange(depth_buffer); image_ranges[num_images] = subresource_range; samples = depth_buffer->Samples(); + depth_attachment = depth_buffer->RenderTarget(); + depth_attachment_format = + MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, + depth_buffer->format) + .format; ++num_images; has_depth = (subresource_range.aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0; has_stencil = (subresource_range.aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) != 0; @@ -2752,10 +2851,23 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, samples != VK_SAMPLE_COUNT_1_BIT && num_colors > 0 && runtime.device.IsTiler(); renderpass_key.resolve_color = do_resolve_color; + const VkResolveModeFlagBits depth_mode = runtime.device.GetDepthResolveMode(); + const VkResolveModeFlagBits stencil_mode = runtime.device.GetStencilResolveMode(); + const bool resolve_modes_compatible = + depth_mode != VK_RESOLVE_MODE_NONE && + (!has_stencil || (stencil_mode != VK_RESOLVE_MODE_NONE && + (runtime.device.IsIndependentResolveSupported() || + depth_mode == stencil_mode))); + const bool msaa_depth = + samples != VK_SAMPLE_COUNT_1_BIT && has_depth && runtime.device.IsTiler(); + const bool do_resolve_depth = ENABLE_MSAA_DEPTH_RESOLVE && msaa_depth && + runtime.device.IsKhrDynamicRenderingSupported() && + resolve_modes_compatible; + discard_msaa_color = ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_COLOR_DISCARD && do_resolve_color; + discard_msaa_depth = ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_DEPTH_DISCARD && msaa_depth; - renderpass = runtime.render_pass_cache.Get(renderpass_key); render_pass_key = renderpass_key; render_pass_cache = &runtime.render_pass_cache; render_area.width = (std::min)(render_area.width, width); @@ -2770,10 +2882,15 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, } const VkFormat vk_format = MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, format).format; + color_resolve_modes[index] = VideoCore::Surface::IsPixelFormatInteger(format) + ? VK_RESOLVE_MODE_SAMPLE_ZERO_BIT + : VK_RESOLVE_MODE_AVERAGE_BIT; if (ENABLE_MSAA_RESOLVE_CONSUME) { const VkImage msaa_image = images[rt_map[index]]; - attachments.push_back(runtime.GetOrCreateResolveShadow(msaa_image, vk_format, - render_area, layers)); + const VkImageView shadow_view = runtime.GetOrCreateResolveShadow( + msaa_image, vk_format, render_area, layers, VK_IMAGE_ASPECT_COLOR_BIT); + color_resolve_attachments[index] = shadow_view; + attachments.push_back(shadow_view); continue; } VkImageCreateInfo resolve_ci{ @@ -2812,13 +2929,61 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, .layerCount = layers, }, }); + if (runtime.device.IsKhrDynamicRenderingSupported()) { + runtime.scheduler.RecordWithUploadBuffer( + [image = *resolve_image, layers](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + const VkImageMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = layers, + }, + }; + upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, barrier); + }); + } + color_resolve_attachments[index] = *resolve_view; attachments.push_back(*resolve_view); resolve_images.push_back(std::move(resolve_image)); resolve_image_views.push_back(std::move(resolve_view)); } } + if (do_resolve_depth && depth_buffer != nullptr) { + const u32 layers = static_cast((std::max)(num_layers, 1)); + VkImageAspectFlags aspect = 0; + if (has_depth) { + aspect |= VK_IMAGE_ASPECT_DEPTH_BIT; + } + if (has_stencil) { + aspect |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + depth_resolve_attachment = + runtime.GetOrCreateResolveShadow(depth_buffer->ImageHandle(), depth_attachment_format, + render_area, layers, aspect); + depth_resolve_mode = depth_mode; + stencil_resolve_mode = has_stencil ? stencil_mode : VK_RESOLVE_MODE_NONE; + } + num_color_buffers = static_cast(num_colors); + layer_count = static_cast((std::max)(num_layers, 1)); + if (runtime.device.IsKhrDynamicRenderingSupported()) { + return; + } + renderpass = runtime.render_pass_cache.Get(renderpass_key); framebuffer = runtime.device.GetLogical().CreateFramebuffer({ .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, .pNext = nullptr, @@ -2828,7 +2993,7 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, .pAttachments = attachments.data(), .width = render_area.width, .height = render_area.height, - .layers = static_cast((std::max)(num_layers, 1)), + .layers = layer_count, }); } @@ -2844,6 +3009,53 @@ VkRenderPass Framebuffer::RenderPassVariant(u32 color_clear_mask, bool depth_ste return render_pass_cache->Get(key); } +void Framebuffer::BeginRendering(vk::CommandBuffer cmdbuf) const { + std::array color_attachment_infos{}; + for (size_t index = 0; index < num_color_attachments; ++index) { + color_attachment_infos[index] = VkRenderingAttachmentInfo{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = color_attachments[index], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = VK_RESOLVE_MODE_NONE, + .resolveImageView = VK_NULL_HANDLE, + .resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = {}, + }; + } + const VkRenderingAttachmentInfo depth_attachment_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = depth_attachment, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = VK_RESOLVE_MODE_NONE, + .resolveImageView = VK_NULL_HANDLE, + .resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = {}, + }; + const VkRenderingInfo rendering_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, + .pNext = nullptr, + .flags = 0, + .renderArea = + { + .offset = {.x = 0, .y = 0}, + .extent = render_area, + }, + .layerCount = layer_count, + .viewMask = 0, + .colorAttachmentCount = num_color_attachments, + .pColorAttachments = color_attachment_infos.data(), + .pDepthAttachment = has_depth ? &depth_attachment_info : nullptr, + .pStencilAttachment = has_stencil ? &depth_attachment_info : nullptr, + }; + cmdbuf.BeginRendering(rendering_info); +} + void TextureCacheRuntime::AccelerateImageUpload( Image& image, const StagingBufferRef& map, std::span swizzles, @@ -2853,6 +3065,15 @@ void TextureCacheRuntime::AccelerateImageUpload( return astc_decoder_pass->Assemble(image, map, swizzles); } + if (bl2d_unswizzle_pass && BlockLinearUnswizzle2DPass::IsSupported(image.info)) { + return bl2d_unswizzle_pass->Unswizzle(image, map, swizzles); + } + + if (bl3db_unswizzle_pass && + BlockLinearUnswizzle3DBufferPass::IsSupported(device, image.info)) { + return bl3db_unswizzle_pass->Unswizzle(image, map, swizzles); + } + if (!Settings::values.gpu_unswizzle_enabled.GetValue() || !bl3d_unswizzle_pass) { if (IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D) { ASSERT(false && "GPU unswizzle is disabled for BCn 3D texture"); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 99f2978cca..fa7dc39313 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -60,10 +60,18 @@ public: void TickFrame(); + u64 CurrentSyncPoint() const noexcept; + + u64 CompletedSyncPoint() const; + + void WaitSyncPoint(u64 sync_point); + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const; + bool CanReportMemoryUsage() const; std::optional GetSamplerHeapBudget() const; @@ -121,7 +129,8 @@ public: }; [[nodiscard]] VkImageView GetOrCreateResolveShadow(VkImage msaa_image, VkFormat format, - VkExtent2D extent, u32 layers); + VkExtent2D extent, u32 layers, + VkImageAspectFlags aspect); [[nodiscard]] const ResolveShadow* GetValidResolveShadow(VkImage msaa_image) const; @@ -149,6 +158,8 @@ public: std::optional astc_decoder_pass; std::optional bl3d_unswizzle_pass; + std::optional bl2d_unswizzle_pass; + std::optional bl3db_unswizzle_pass; const Settings::ResolutionScalingInfo& resolution; std::array, VideoCore::Surface::MaxPixelFormat> view_formats; @@ -156,6 +167,7 @@ public: std::array buffers{}; std::vector> pending_msaa_images; ankerl::unordered_dense::map resolve_shadows; + ankerl::unordered_dense::set unsupported_msaa_resolves; }; class Framebuffer { @@ -178,6 +190,8 @@ public: std::span color_buffers, ImageView* depth_buffer, bool is_rescaled = false); + void BeginRendering(vk::CommandBuffer cmdbuf) const; + [[nodiscard]] VkFramebuffer Handle() const noexcept { return *framebuffer; } @@ -193,6 +207,51 @@ public: [[nodiscard]] VkRenderPass RenderPassVariant(u32 color_clear_mask, bool depth_stencil_clear, u32 color_discard_mask) const; + [[nodiscard]] u32 NumColorAttachments() const noexcept { + return num_color_attachments; + } + + [[nodiscard]] const std::array& ColorAttachments() const noexcept { + return color_attachments; + } + + [[nodiscard]] const std::array& ColorAttachmentFormats() const noexcept { + return color_attachment_formats; + } + + [[nodiscard]] const std::array& ColorResolveAttachments() const noexcept { + return color_resolve_attachments; + } + + [[nodiscard]] const std::array& ColorResolveModes() + const noexcept { + return color_resolve_modes; + } + + [[nodiscard]] VkImageView DepthAttachment() const noexcept { + return depth_attachment; + } + + [[nodiscard]] VkImageView DepthResolveAttachment() const noexcept { + return depth_resolve_attachment; + } + + [[nodiscard]] VkResolveModeFlagBits DepthResolveMode() const noexcept { + return depth_resolve_mode; + } + + [[nodiscard]] VkResolveModeFlagBits StencilResolveMode() const noexcept { + return stencil_resolve_mode; + } + + [[nodiscard]] VkFormat DepthAttachmentFormat() const noexcept { + return depth_attachment_format; + } + + [[nodiscard]] u32 NumLayers() const noexcept { + return layer_count; + } + [[nodiscard]] VkExtent2D RenderArea() const noexcept { return render_area; } @@ -245,6 +304,10 @@ public: return discard_msaa_color; } + [[nodiscard]] bool DiscardsMsaaDepth() const noexcept { + return discard_msaa_depth; + } + private: vk::Framebuffer framebuffer; VkRenderPass renderpass{}; @@ -255,6 +318,17 @@ private: std::array images{}; std::array image_ranges{}; std::array rt_map{}; + std::array color_attachments{}; + std::array color_attachment_formats{}; + std::array color_resolve_attachments{}; + std::array color_resolve_modes{}; + VkImageView depth_attachment{}; + VkImageView depth_resolve_attachment{}; + VkResolveModeFlagBits depth_resolve_mode = VK_RESOLVE_MODE_NONE; + VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_NONE; + VkFormat depth_attachment_format = VK_FORMAT_UNDEFINED; + u32 num_color_attachments = 0; + u32 layer_count = 1; bool has_depth{}; bool has_stencil{}; bool is_rescaled{}; @@ -263,6 +337,7 @@ private: RenderPassKey render_pass_key{}; RenderPassCache* render_pass_cache{nullptr}; bool discard_msaa_color{}; + bool discard_msaa_depth{}; }; class Image : public VideoCommon::ImageBase { @@ -426,7 +501,7 @@ private: std::array image_views; std::optional storage_views; - vk::ImageView typeless_storage_view; + std::array typeless_storage_views; vk::ImageView depth_view; vk::ImageView stencil_view; vk::ImageView color_view; @@ -436,7 +511,6 @@ private: VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT; u32 buffer_size = 0; - bool uses_widened_astc_format = false; bool supports_depth_comparison = false; }; @@ -487,6 +561,7 @@ struct TextureCacheParams { static constexpr bool HAS_EMULATED_COPIES = false; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true; + static constexpr bool HAS_TIMELINE_SYNC_POINTS = true; using Runtime = Vulkan::TextureCacheRuntime; using Image = Vulkan::Image; diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp index b0227ac908..742a8d9969 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp @@ -16,8 +16,11 @@ namespace Vulkan { -UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_) - : device{device_} +UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_, + bool supports_descriptor_buffer_) + : device{device_}, frame_payload_size{frame_payload_size_}, + supports_descriptor_buffer{supports_descriptor_buffer_}, + payload(frame_payload_size_ * FRAMES_IN_FLIGHT) { payload_start = payload.data(); payload_cursor = payload.data(); @@ -29,19 +32,21 @@ void UpdateDescriptorQueue::TickFrame() { if (++frame_index >= FRAMES_IN_FLIGHT) { frame_index = 0; } - payload_start = payload.data() + frame_index * FRAME_PAYLOAD_SIZE; + payload_start = payload.data() + frame_index * frame_payload_size; payload_cursor = payload_start; } -void UpdateDescriptorQueue::Acquire(Scheduler& scheduler, size_t required_entries) { +void UpdateDescriptorQueue::Acquire(Scheduler& scheduler, size_t required_entries, + bool use_descriptor_buffer_) { + use_descriptor_buffer = supports_descriptor_buffer && use_descriptor_buffer_; static constexpr size_t DEFAULT_REQUIRED_ENTRIES = 0x400; const size_t reserve = required_entries > 0 ? required_entries : DEFAULT_REQUIRED_ENTRIES; - ASSERT_MSG(reserve < FRAME_PAYLOAD_SIZE, "Descriptor reservation {} >= frame capacity {}", - reserve, FRAME_PAYLOAD_SIZE); + ASSERT_MSG(reserve < frame_payload_size, "Descriptor reservation {} >= frame capacity {}", + reserve, frame_payload_size); const size_t used = static_cast(std::distance(payload_start, payload_cursor)); - if (used + reserve >= FRAME_PAYLOAD_SIZE) { + if (used + reserve >= frame_payload_size) { LOG_WARNING(Render_Vulkan, "Payload overflow (used={}, reserve={}, capacity={})", - used, reserve, FRAME_PAYLOAD_SIZE); + used, reserve, frame_payload_size); scheduler.WaitWorker(); payload_cursor = payload_start; } diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 39b2b5a688..9f30e65dbd 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,8 +6,8 @@ #pragma once -#include #include +#include #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { @@ -15,30 +15,45 @@ namespace Vulkan { class Device; class Scheduler; +struct DescriptorAddress { + VkDeviceAddress address; + VkDeviceSize range; + VkFormat format; +}; + union DescriptorUpdateEntry { DescriptorUpdateEntry() = default; DescriptorUpdateEntry(VkDescriptorImageInfo image_) : image{image_} {} DescriptorUpdateEntry(VkDescriptorBufferInfo buffer_) : buffer{buffer_} {} DescriptorUpdateEntry(VkBufferView texel_buffer_) : texel_buffer{texel_buffer_} {} + DescriptorUpdateEntry(DescriptorAddress address_) : address{address_} {} std::monostate empty{}; VkDescriptorImageInfo image; VkDescriptorBufferInfo buffer; VkBufferView texel_buffer; + DescriptorAddress address; }; class UpdateDescriptorQueue final { // This should be plenty for the vast majority of cases. Most desktop platforms only // provide up to 3 swapchain images. static constexpr size_t FRAMES_IN_FLIGHT = 8; - static constexpr size_t FRAME_PAYLOAD_SIZE = 0x20000; - static constexpr size_t PAYLOAD_SIZE = FRAME_PAYLOAD_SIZE * FRAMES_IN_FLIGHT; public: - explicit UpdateDescriptorQueue(const Device& device_); + static constexpr size_t GUEST_FRAME_PAYLOAD_SIZE = 0x80000; + static constexpr size_t COMPUTE_FRAME_PAYLOAD_SIZE = 0x20000; + + explicit UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_, + bool supports_descriptor_buffer_ = false); ~UpdateDescriptorQueue(); + [[nodiscard]] bool UsesDescriptorBuffer() const noexcept { + return use_descriptor_buffer; + } + void TickFrame(); - void Acquire(Scheduler& scheduler, size_t required_entries = 0); + void Acquire(Scheduler& scheduler, size_t required_entries = 0, + bool use_descriptor_buffer_ = false); const DescriptorUpdateEntry* UpdateData() const noexcept { return upload_start; @@ -68,17 +83,46 @@ public: }; } + void AddBuffer(VkBuffer buffer, VkDeviceAddress base_address, VkDeviceSize offset, + VkDeviceSize size) { + if (!use_descriptor_buffer) { + AddBuffer(buffer, offset, size); + return; + } + *(payload_cursor++) = DescriptorAddress{ + .address = base_address == 0 ? 0 : base_address + offset, + .range = base_address == 0 ? VK_WHOLE_SIZE : size, + .format = VK_FORMAT_UNDEFINED, + }; + } + void AddTexelBuffer(VkBufferView texel_buffer) { *(payload_cursor++) = texel_buffer; } + void AddTexelBuffer(VkBufferView texel_buffer, VkDeviceAddress base_address, + VkDeviceSize offset, VkDeviceSize size, VkFormat format) { + if (!use_descriptor_buffer) { + AddTexelBuffer(texel_buffer); + return; + } + *(payload_cursor++) = DescriptorAddress{ + .address = base_address == 0 ? 0 : base_address + offset, + .range = base_address == 0 ? VK_WHOLE_SIZE : size, + .format = format, + }; + } + private: const Device& device; + const size_t frame_payload_size; + const bool supports_descriptor_buffer; + bool use_descriptor_buffer{false}; size_t frame_index{0}; DescriptorUpdateEntry* payload_cursor = nullptr; DescriptorUpdateEntry* payload_start = nullptr; const DescriptorUpdateEntry* upload_start = nullptr; - std::array payload; + std::vector payload; }; // TODO: should these be separate classes instead? diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 4d5db7e511..be3e711cb1 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -284,7 +284,24 @@ std::optional GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - ASSERT(handle.first <= tic_limit); + if (handle.first > tic_limit) { + LOG_CRITICAL(Shader, + "TIC index out of range: raw=0x{:08x} tic_index={} tsc_index={} tic_limit={} " + "tic_addr=0x{:x} via_header_index={} stage={} program_base=0x{:x} " + "start_address=0x{:x}", + raw, handle.first, handle.second, tic_limit, tic_addr, via_header_index, + static_cast(stage), program_base, start_address); + ASSERT(handle.first <= tic_limit); + Tegra::Texture::TICEntry fallback{}; + fallback.format.Assign(Tegra::Texture::TextureFormat::A8B8G8R8); + fallback.r_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.g_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.b_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.a_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.texture_type.Assign(Tegra::Texture::TextureType::Texture2D); + fallback.normalized_coords.Assign(1); + return fallback; + } const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 2cfdb94359..68da600094 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -58,23 +58,9 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag void(slot_samplers.insert(runtime, sampler_descriptor)); if constexpr (HAS_DEVICE_MEMORY_INFO) { - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - expected_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); - minimum_memory = static_cast((device_local_memory - mem_threshold) / 2); + memory_budget = runtime.GetDeviceLocalMemory(); } else { - expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB; - critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; - minimum_memory = 0; + memory_budget = FALLBACK_MEMORY_BUDGET; } const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); @@ -114,71 +100,162 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag } template -void TextureCache

::RunGarbageCollector() { - bool high_priority_mode = false; - bool aggressive_mode = false; - u64 ticks_to_destroy = 0; - size_t num_iterations = 0; - const auto Configure = [&](bool allow_aggressive) { - high_priority_mode = total_used_memory >= expected_memory; - aggressive_mode = allow_aggressive && total_used_memory >= critical_memory; - ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL; - num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); - }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { - if (num_iterations == 0) { - return true; - } - --num_iterations; - auto& image = slot_images[image_id]; - if (True(image.flags & ImageFlagBits::IsDecoding)) { - return false; - } - const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { - return false; - } - if (must_download) { - auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes); - const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); - image.DownloadMemory(map, copies); - runtime.Finish(); - SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer); - } - if (True(image.flags & ImageFlagBits::Tracked)) { - UntrackImage(image, image_id); - } - UnregisterImage(image_id); - DeleteImage(image_id, image.scale_tick > frame_tick + 5); - if (aggressive_mode && total_used_memory < critical_memory) { - num_iterations >>= 2; - aggressive_mode = false; - } else if (high_priority_mode && total_used_memory < expected_memory) { - num_iterations >>= 1; - high_priority_mode = false; - } - return false; - }; - Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); - if (total_used_memory >= critical_memory) { - Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); +void TextureCache

::QueueEvictionDownload(Image& image) { + auto copies = FullDownloadCopies(image.info); + auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true); + image.DownloadMemory(staging, FixSmallVectorADL(copies)); + pending_eviction_downloads.push_back(PendingEvictionDownload{ + .staging = staging, + .gpu_memory = gpu_memory, + .copies = std::move(copies), + .info = image.info, + .gpu_addr = image.gpu_addr, + .sync_point = runtime.CurrentSyncPoint(), + }); +} + +template +void TextureCache

::TickEvictionDownloads(u64 completed_sync_point) { + while (!pending_eviction_downloads.empty() && + pending_eviction_downloads.front().sync_point <= completed_sync_point) { + auto& entry = pending_eviction_downloads.front(); + SwizzleImage(*entry.gpu_memory, entry.gpu_addr, entry.info, FixSmallVectorADL(entry.copies), + entry.staging.mapped_span.subspan(entry.staging.offset), swizzle_data_buffer); + runtime.FreeDeferredStagingBuffer(entry.staging); + pending_eviction_downloads.pop_front(); } } +template +void TextureCache

::FlushEvictionDownloads() { + if (pending_eviction_downloads.empty()) { + return; + } + const u64 last_sync_point = pending_eviction_downloads.back().sync_point; + runtime.WaitSyncPoint(last_sync_point); + TickEvictionDownloads(last_sync_point); +} + +template +u64 TextureCache

::ImageSizeBytes(const ImageBase& image) { + u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); + if ((IsPixelFormatASTC(image.info.format) && + True(image.flags & ImageFlagBits::AcceleratedUpload)) || + True(image.flags & ImageFlagBits::Converted)) { + tentative_size = TranscodedAstcSize(tentative_size, image.info.format); + } + u64 size = Common::AlignUp(tentative_size, 1024); + if (image.HasScaled()) { + size += GetScaledImageSizeBytes(image); + } + return size; +} + +template +u64 TextureCache

::DeviceUsage(bool force_refresh) { + if (!runtime.CanReportMemoryUsage()) { + return total_used_memory; + } + if (force_refresh || usage_refresh_countdown == 0) { + cached_device_usage = runtime.GetDeviceAllocationUsage(); + usage_refresh_countdown = USAGE_REFRESH_INTERVAL; + } else { + --usage_refresh_countdown; + } + return cached_device_usage; +} + +template +u64 TextureCache

::ReclaimMemory(u64 target_bytes, bool allow_download) { + if (target_bytes == 0 || in_reclaim) { + return 0; + } + in_reclaim = true; + u64 freed = 0; + const auto evict = [&](ImageId image_id) { + if (freed >= target_bytes) { + return true; + } + auto& image = slot_images[image_id]; + if (True(image.flags & ImageFlagBits::IsDecoding)) { + return false; + } + const bool must_download = image.IsSafeDownload(); + if (must_download && True(image.flags & ImageFlagBits::BadOverlap)) { + LOG_WARNING(HW_GPU, + "Recovering bad overlap on eviction: gpu_addr=0x{:x} fmt={} {}x{}x{}", + image.gpu_addr, static_cast(image.info.format), image.info.size.width, + image.info.size.height, image.info.size.depth); + } + bool queued_download = false; + if (must_download) { + if constexpr (HAS_TIMELINE_SYNC_POINTS) { + if (!allow_download) { + return false; + } + QueueEvictionDownload(image); + queued_download = true; + } else { + return false; + } + } + const u64 image_bytes = ImageSizeBytes(image); + if (True(image.flags & ImageFlagBits::Tracked)) { + UntrackImage(image, image_id); + } + UnregisterImage(image_id); + DeleteImage(image_id, !queued_download && image.scale_tick > frame_tick + 5); + freed += image_bytes; + return false; + }; + const u64 cold_tick = + frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0; + lru_cache.ForEachItemBelow(cold_tick, evict); + if (freed == 0) { + const u64 urgent_tick = frame_tick > RECLAIM_URGENT_GUARD_FRAMES + ? frame_tick - RECLAIM_URGENT_GUARD_FRAMES + : 0; + lru_cache.ForEachItemBelow(urgent_tick, evict); + } + in_reclaim = false; + usage_refresh_countdown = 0; + reclaim_stalled = freed == 0; + if (freed > 0) { + reclaim_wait_sync_point = runtime.CurrentSyncPoint(); + } + return freed; +} + +template +void TextureCache

::EnsureHeadroom(bool allow_download) { + if (reclaim_stalled) { + return; + } + if (runtime.CompletedSyncPoint() < reclaim_wait_sync_point) { + return; + } + const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0; + const u64 usage = DeviceUsage(false); + if (usage <= limit) { + return; + } + const u64 target = (limit / 100) * RECLAIM_TARGET_PERCENT; + const u64 excess = usage - target; + const u64 usage_mib = (std::max)(usage >> 20, u64{1}); + const u64 share = (((excess >> 20) * (total_used_memory >> 20)) / usage_mib) << 20; + ReclaimMemory((std::min)(share, total_used_memory), allow_download); +} + template void TextureCache

::TickFrame() { - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); - } - if (total_used_memory > minimum_memory) { - RunGarbageCollector(); - } - sentenced_images.Tick(); - sentenced_framebuffers.Tick(); - sentenced_image_view.Tick(); + usage_refresh_countdown = 0; + reclaim_stalled = false; + EnsureHeadroom(true); + const u64 completed_sync_point = runtime.CompletedSyncPoint(); + TickEvictionDownloads(completed_sync_point); + sentenced_images.Reclaim(completed_sync_point); + sentenced_framebuffers.Reclaim(completed_sync_point); + sentenced_image_view.Reclaim(completed_sync_point); TickAsyncDecode(); TickAsyncUnswizzle(); @@ -512,10 +589,24 @@ void TextureCache

::UpdateRenderTargets(bool is_clear) { up_scale = Settings::values.resolution_info.up_scale; down_shift = Settings::values.resolution_info.down_shift; } - render_targets.size = Extent2D{ - (maxwell3d->regs.surface_clip.width * up_scale) >> down_shift, - (maxwell3d->regs.surface_clip.height * up_scale) >> down_shift, + u32 clip_width = (maxwell3d->regs.surface_clip.width * up_scale) >> down_shift; + u32 clip_height = (maxwell3d->regs.surface_clip.height * up_scale) >> down_shift; + const auto& resolution = Settings::values.resolution_info; + const auto clamp_to_attachment = [&](ImageViewId view_id) { + if (!view_id) { + return; + } + const ImageViewBase& view = slot_image_views[view_id]; + clip_width = (std::min)(clip_width, is_rescaling ? resolution.ScaleUp(view.size.width) + : view.size.width); + clip_height = (std::min)(clip_height, is_rescaling ? resolution.ScaleUp(view.size.height) + : view.size.height); }; + for (size_t index = 0; index < NUM_RT; ++index) { + clamp_to_attachment(render_targets.color_buffer_ids[index]); + } + clamp_to_attachment(depth_buffer_id); + render_targets.size = Extent2D{clip_width, clip_height}; render_targets.is_rescaled = is_rescaling; if (render_targets != previous_render_targets) { @@ -596,6 +687,7 @@ void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { template void TextureCache

::DownloadMemory(DAddr cpu_addr, size_t size) { + FlushEvictionDownloads(); boost::container::small_vector images; ForEachImageInRegion(cpu_addr, size, [&images](ImageId image_id, ImageBase& image) { if (!image.IsSafeDownload()) { @@ -894,6 +986,7 @@ void TextureCache

::CommitAsyncFlushes() { template void TextureCache

::PopAsyncFlushes() { + TickEvictionDownloads(runtime.CompletedSyncPoint()); if (committed_downloads.empty()) { return; } @@ -1148,6 +1241,8 @@ void TextureCache

::UploadImageContents(Image& image, StagingBuffer& staging) return; } + gpu_memory->FlushRegion(gpu_addr, image.guest_size_bytes, + VideoCommon::CacheType::NoTextureCache); Tegra::Memory::GpuGuestMemory swizzle_data( *gpu_memory, gpu_addr, image.guest_size_bytes, &swizzle_data_buffer); if (True(image.flags & ImageFlagBits::Converted)) { @@ -1294,8 +1389,9 @@ void TextureCache

::InvalidateScale(Image& image) { } RemoveImageViewReferences(image_view_ids); RemoveFramebuffers(image_view_ids); + const u64 sync_point = runtime.CurrentSyncPoint(); for (const ImageViewId image_view_id : image_view_ids) { - sentenced_image_view.Push(std::move(slot_image_views[image_view_id])); + sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point); slot_image_views.erase(image_view_id); } image.image_view_ids.clear(); @@ -1331,6 +1427,7 @@ void TextureCache

::QueueAsyncDecode(Image& image, ImageId image_id) { LOG_INFO(HW_GPU, "Queuing async texture decode"); image.flags |= ImageFlagBits::IsDecoding; + runtime.TransitionImageLayout(image); auto decode = std::make_unique(); auto* decode_ptr = decode.get(); decode->image_id = image_id; @@ -1363,6 +1460,7 @@ void TextureCache

::QueueAsyncUnswizzle(Image& image, ImageId image_id) { } image.flags |= ImageFlagBits::IsDecoding; + runtime.TransitionImageLayout(image); unswizzle_queue.push_back({ .image_id = image_id, @@ -1523,6 +1621,7 @@ ImageId TextureCache

::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, template ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr) { + EnsureHeadroom(false); ImageInfo new_info = info; const size_t size_bytes = CalculateGuestSizeInBytes(new_info); const bool broken_views = runtime.HasBrokenTextureViewFormats(); @@ -1631,7 +1730,35 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA for (const ImageId overlap_id : join_ignore_textures) { Image& overlap = slot_images[overlap_id]; if (True(overlap.flags & ImageFlagBits::GpuModified)) { - UNIMPLEMENTED(); + if (new_image.TryFindBase(overlap.gpu_addr) && + (!can_rescale || ImageCanRescale(overlap))) { + if (can_rescale) { + ScaleUp(overlap); + } else { + ScaleDown(overlap); + } + join_copies_to_do.emplace_back(JoinCopy{false, overlap_id}); + continue; + } + if (overlap.IsSafeDownload() && + gpu_memory->GpuToCpuAddress(overlap.gpu_addr).has_value()) { + if (True(overlap.flags & ImageFlagBits::BadOverlap)) { + LOG_WARNING(HW_GPU, + "Recovering bad overlap on join: gpu_addr=0x{:x} fmt={} {}x{}x{}", + overlap.gpu_addr, static_cast(overlap.info.format), + overlap.info.size.width, overlap.info.size.height, + overlap.info.size.depth); + } + QueueEvictionDownload(overlap); + } else { + LOG_WARNING(HW_GPU, + "Dropping GPU modified overlap, contents are not recoverable: " + "gpu_addr=0x{:x} format={} size={}x{}x{} levels={} layers={}", + overlap.gpu_addr, static_cast(overlap.info.format), + overlap.info.size.width, overlap.info.size.height, + overlap.info.size.depth, overlap.info.resources.levels, + overlap.info.resources.layers); + } } if (True(overlap.flags & ImageFlagBits::Tracked)) { UntrackImage(overlap, overlap_id); @@ -1678,6 +1805,8 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA ImageBase& aliased = slot_images[aliased_id]; aliased.overlapping_images.push_back(new_image_id); new_image.overlapping_images.push_back(aliased_id); + const bool aliased_was_bad = True(aliased.flags & ImageFlagBits::BadOverlap); + const bool new_was_bad = True(new_image.flags & ImageFlagBits::BadOverlap); if (aliased.info.resources.levels == 1 && aliased.info.block.depth == 0 && aliased.overlapping_images.size() > 1) { aliased.flags |= ImageFlagBits::BadOverlap; @@ -1686,6 +1815,21 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA new_image.overlapping_images.size() > 1) { new_image.flags |= ImageFlagBits::BadOverlap; } + const bool aliased_is_bad = True(aliased.flags & ImageFlagBits::BadOverlap); + const bool new_is_bad = True(new_image.flags & ImageFlagBits::BadOverlap); + if ((!aliased_was_bad && aliased_is_bad) || (!new_was_bad && new_is_bad)) { + LOG_WARNING(HW_GPU, + "Bad overlap: existing gpu_addr={:#x} {}x{}x{} fmt={} type={} rt={} | " + "incoming gpu_addr={:#x} {}x{}x{} fmt={} type={} rt={}", + aliased.gpu_addr, aliased.info.size.width, aliased.info.size.height, + aliased.info.size.depth, static_cast(aliased.info.format), + static_cast(aliased.info.type), + True(aliased.flags & ImageFlagBits::GpuModified), + new_image.gpu_addr, new_image.info.size.width, new_image.info.size.height, + new_image.info.size.depth, static_cast(new_image.info.format), + static_cast(new_image.info.type), + True(new_image.flags & ImageFlagBits::GpuModified)); + } } for (const auto& copy_object : join_copies_to_do) { @@ -2185,13 +2329,7 @@ void TextureCache

::RegisterImage(ImageId image_id) { ASSERT_MSG(False(image.flags & ImageFlagBits::Registered), "Trying to register an already registered image"); image.flags |= ImageFlagBits::Registered; - u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - if ((IsPixelFormatASTC(image.info.format) && - True(image.flags & ImageFlagBits::AcceleratedUpload)) || - True(image.flags & ImageFlagBits::Converted)) { - tentative_size = TranscodedAstcSize(tentative_size, image.info.format); - } - total_used_memory += Common::AlignUp(tentative_size, 1024); + total_used_memory += ImageSizeBytes(image); image.lru_index = lru_cache.Insert(image_id, frame_tick); ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) { @@ -2308,6 +2446,7 @@ void TextureCache

::TrackImage(ImageBase& image, ImageId image_id) { if (False(image.flags & ImageFlagBits::Sparse)) { if (image.cpu_addr < ~(1ULL << 40)) { device_memory.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1); + device_memory.UpdateTexturePagesCount(image.cpu_addr, image.guest_size_bytes, 1); } return; } @@ -2320,12 +2459,14 @@ void TextureCache

::TrackImage(ImageBase& image, ImageId image_id) { const DAddr cpu_addr = map.cpu_addr; const std::size_t size = map.size; device_memory.UpdatePagesCachedCount(cpu_addr, size, 1); + device_memory.UpdateTexturePagesCount(cpu_addr, size, 1); } return; } ForEachSparseSegment(image, [this]([[maybe_unused]] GPUVAddr gpu_addr, DAddr cpu_addr, size_t size) { device_memory.UpdatePagesCachedCount(cpu_addr, size, 1); + device_memory.UpdateTexturePagesCount(cpu_addr, size, 1); }); } @@ -2336,6 +2477,7 @@ void TextureCache

::UntrackImage(ImageBase& image, ImageId image_id) { if (False(image.flags & ImageFlagBits::Sparse)) { if (image.cpu_addr < ~(1ULL << 40)) { device_memory.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1); + device_memory.UpdateTexturePagesCount(image.cpu_addr, image.guest_size_bytes, -1); } return; } @@ -2348,22 +2490,14 @@ void TextureCache

::UntrackImage(ImageBase& image, ImageId image_id) { const DAddr cpu_addr = map.cpu_addr; const std::size_t size = map.size; device_memory.UpdatePagesCachedCount(cpu_addr, size, -1); + device_memory.UpdateTexturePagesCount(cpu_addr, size, -1); } } template void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ImageBase& image = slot_images[image_id]; - if (image.HasScaled()) { - total_used_memory -= GetScaledImageSizeBytes(image); - } - u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - if ((IsPixelFormatASTC(image.info.format) && - True(image.flags & ImageFlagBits::AcceleratedUpload)) || - True(image.flags & ImageFlagBits::Converted)) { - tentative_size = TranscodedAstcSize(tentative_size, image.info.format); - } - total_used_memory -= Common::AlignUp(tentative_size, 1024); + total_used_memory -= std::min(total_used_memory, ImageSizeBytes(image)); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2417,14 +2551,15 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ASSERT_MSG(num_removed_overlaps == 1, "Invalid number of removed overlapps: {}", num_removed_overlaps); } + const u64 sync_point = runtime.CurrentSyncPoint(); for (const ImageViewId image_view_id : image_view_ids) { if (!immediate_delete) { - sentenced_image_view.Push(std::move(slot_image_views[image_view_id])); + sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point); } slot_image_views.erase(image_view_id); } if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_images.Push(std::move(slot_images[image_id]), sync_point); } slot_images.erase(image_id); @@ -2470,7 +2605,8 @@ void TextureCache

::RemoveFramebuffers(std::span removed_vi last_framebuffer_id = {}; last_framebuffer_serial = 0; } - sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]), + runtime.CurrentSyncPoint()); it = framebuffers.erase(it); } else { ++it; diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index fbc2bb4cf7..23d7cc6cae 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -30,7 +30,7 @@ #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/engines/fermi_2d.h" #include "video_core/surface.h" #include "video_core/texture_cache/descriptor_table.h" @@ -108,18 +108,21 @@ class TextureCache : public VideoCommon::ChannelSetupCaches::max)()}; #ifdef YUZU_LEGACY - static constexpr s64 TARGET_THRESHOLD = 3_GiB; + static constexpr u64 RECLAIM_HEADROOM = 192_MiB; #else - static constexpr s64 TARGET_THRESHOLD = 4_GiB; + static constexpr u64 RECLAIM_HEADROOM = 256_MiB; #endif - static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; - static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; - static constexpr size_t GC_EMERGENCY_COUNTS = 2; + static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB; + static constexpr u32 USAGE_REFRESH_INTERVAL = 16; + static constexpr u64 RECLAIM_GUARD_FRAMES = 16; + static constexpr u64 RECLAIM_URGENT_GUARD_FRAMES = 4; + static constexpr u64 RECLAIM_TARGET_PERCENT = 90; using Runtime = typename P::Runtime; using Image = typename P::Image; @@ -154,6 +157,8 @@ public: /// Notify the cache that a new frame has been queued void TickFrame(); + u64 ReclaimMemory(u64 target_bytes, bool allow_download); + /// Return a constant reference to the given image view id [[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept; @@ -257,7 +262,7 @@ public: /// Prepare an image to be used void PrepareImage(ImageId image_id, bool is_modification, bool invalidate); - std::recursive_mutex mutex; + std::mutex mutex; private: /// Iterate over all page indices in a range @@ -293,8 +298,17 @@ private: void OnGPUASRegister(size_t map_id) final override; - /// Runs the Garbage Collector. - void RunGarbageCollector(); + u64 ImageSizeBytes(const ImageBase& image); + + u64 DeviceUsage(bool force_refresh); + + void EnsureHeadroom(bool allow_download); + + void QueueEvictionDownload(Image& image); + + void TickEvictionDownloads(u64 completed_sync_point); + + void FlushEvictionDownloads(); /// Find or create an image view in the guest descriptor table ImageViewId VisitImageView(u32 index, bool compute); @@ -451,9 +465,12 @@ private: bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; - u64 minimum_memory; - u64 expected_memory; - u64 critical_memory; + u64 memory_budget = 0; + u64 cached_device_usage = 0; + u64 reclaim_wait_sync_point = 0; + u32 usage_refresh_countdown = 0; + bool in_reclaim = false; + bool reclaim_stalled = false; size_t gpu_unswizzle_maxsize = 0; size_t swizzle_chunk_size = 0; u32 swizzle_slices_per_batch = 0; @@ -491,14 +508,19 @@ private: }; Common::LeastRecentlyUsedCache lru_cache; - #ifdef YUZU_LEGACY - static constexpr size_t TICKS_TO_DESTROY = 6; - #else - static constexpr size_t TICKS_TO_DESTROY = 8; -#endif - DelayedDestructionRing sentenced_images; - DelayedDestructionRing sentenced_image_view; - DelayedDestructionRing sentenced_framebuffers; + DeferredDestructionQueue sentenced_images; + DeferredDestructionQueue sentenced_image_view; + DeferredDestructionQueue sentenced_framebuffers; + + struct PendingEvictionDownload { + AsyncBuffer staging; + Tegra::MemoryManager* gpu_memory; + boost::container::small_vector copies; + VideoCommon::ImageInfo info; + GPUVAddr gpu_addr; + u64 sync_point; + }; + std::deque pending_eviction_downloads; ankerl::unordered_dense::map image_allocs_table; @@ -509,7 +531,8 @@ private: u64 frame_tick = 0; u64 last_sampler_gc_frame = (std::numeric_limits::max)(); - Common::ThreadWorker texture_decode_worker{1, "TextureDecoder"}; + Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, + Common::ThreadPlacement::Background}; std::vector> async_decodes; std::deque unswizzle_queue; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index e55d0752ec..e5afcd1a47 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project @@ -747,7 +747,7 @@ boost::container::small_vector MakeShrinkImageCopies(const ImageI const bool is_dst_3d = dst.type == ImageType::e3D; if (is_dst_3d) { - ASSERT(src.type == ImageType::e3D); + ASSERT(src.type == ImageType::e3D || src.resources.layers == 1); ASSERT(src.resources.levels == 1); } const bool both_2d{src.type == ImageType::e2D && dst.type == ImageType::e2D}; diff --git a/src/video_core/textures/workers.cpp b/src/video_core/textures/workers.cpp index 780e0c4da9..058cc0ab6f 100644 --- a/src/video_core/textures/workers.cpp +++ b/src/video_core/textures/workers.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project @@ -10,7 +10,8 @@ namespace Tegra::Texture { Common::ThreadWorker& GetThreadWorkers() { static Common::ThreadWorker workers{(std::max)(std::thread::hardware_concurrency(), 2U) / 2, - "ImageTranscode"}; + "ImageTranscode", {}, + Common::ThreadPlacement::Background}; return workers; } diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 8fd9b6e69f..5b0b247375 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -5,8 +5,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include +#include +#include #include #include #include @@ -16,6 +19,9 @@ #include #include "common/assert.h" +#include "common/fs/fs.h" +#include "common/fs/path_util.h" +#include "common/host_memory.h" #include "common/literals.h" #include #include "common/settings.h" @@ -393,6 +399,17 @@ std::vector ExtensionListForVulkan( return output; } +constexpr std::array STATIC_CACHE_MAGIC_NUMBER{'e', 'd', 'e', 'n', 's', 't', 'p', 'c'}; +constexpr u32 STATIC_CACHE_VERSION = 1; + +std::filesystem::path StaticPipelineCacheFilename() { + const auto shader_dir = Common::FS::GetEdenPath(Common::FS::EdenPath::ShaderDir); + if (!Common::FS::CreateDir(shader_dir)) { + return {}; + } + return shader_dir / "vulkan_static_pipelines.bin"; +} + } // Anonymous namespace void Device::RemoveExtension(bool& extension, const std::string& extension_name) { @@ -523,10 +540,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR features.shader_atomic_int64.shaderBufferInt64Atomics = false; features.shader_atomic_int64.shaderSharedInt64Atomics = false; features.features.shaderInt64 = false; - LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken workgroup memory explicit layout."); - RemoveExtensionFeature(extensions.workgroup_memory_explicit_layout, - features.workgroup_memory_explicit_layout, - VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME); #if defined(__ANDROID__) && defined(ARCHITECTURE_arm64) // BCn patching only safe on Android 9+ (API 28+). Older versions crash on driver load. @@ -713,6 +726,17 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR RemoveExtensionFeature(extensions.vertex_input_dynamic_state, features.vertex_input_dynamic_state, VK_EXT_VERTEX_INPUT_DYNAMIC_STATE_EXTENSION_NAME); } + // VK_EXT_descriptor_buffer requires VK_KHR_buffer_device_address + if (extensions.descriptor_buffer && !features.buffer_device_address.bufferDeviceAddress) { + LOG_WARNING(Render_Vulkan, "Descriptor buffer needs buffer device address, disabling."); + RemoveExtensionFeature(extensions.descriptor_buffer, features.descriptor_buffer, + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME); + } + if (!extensions.descriptor_buffer) { + RemoveExtensionFeature(extensions.buffer_device_address, features.buffer_device_address, + VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME); + } + logical = vk::Device::Create(physical, queue_cis, ExtensionListForVulkan(loaded_extensions), first_next, dld); graphics_queue = logical.GetQueue(graphics_family); @@ -726,13 +750,16 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR if (extensions.memory_budget) { flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; } + if (extensions.buffer_device_address) { + flags |= VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT; + } const VmaAllocatorCreateInfo allocator_info{ .flags = flags, .physicalDevice = physical, .device = *logical, .preferredLargeHeapBlockSize = is_integrated ? (64u * 1024u * 1024u) - : (256u * 1024u * 1024u), + : (128u * 1024u * 1024u), .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = nullptr, @@ -744,15 +771,124 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR vk::Check(vmaCreateAllocator(&allocator_info, &allocator)); + { + const auto& limits = properties.properties.limits; + LOG_INFO(Render_Vulkan, "MSAA sample count support:"); + LOG_INFO(Render_Vulkan, " framebufferColorSampleCounts: {:#x}", + limits.framebufferColorSampleCounts); + LOG_INFO(Render_Vulkan, " framebufferDepthSampleCounts: {:#x}", + limits.framebufferDepthSampleCounts); + LOG_INFO(Render_Vulkan, " framebufferStencilSampleCounts: {:#x}", + limits.framebufferStencilSampleCounts); + LOG_INFO(Render_Vulkan, " sampledImageColorSampleCounts: {:#x}", + limits.sampledImageColorSampleCounts); + LOG_INFO(Render_Vulkan, " sampledImageDepthSampleCounts: {:#x}", + limits.sampledImageDepthSampleCounts); + LOG_INFO(Render_Vulkan, " sampledImageIntegerSampleCounts:{:#x}", + limits.sampledImageIntegerSampleCounts); + LOG_INFO(Render_Vulkan, " storageImageSampleCounts: {:#x}", + limits.storageImageSampleCounts); + } + + owns_static_pipeline_cache = surface != VkSurfaceKHR{}; + LoadStaticPipelineCache(); + // Initialize GPU logging if enabled InitializeGPULogging(); } Device::~Device() { + SaveStaticPipelineCache(); ShutdownGPULogging(); + vk::FlushDeletionQueue(); vmaDestroyAllocator(allocator); } +void Device::LoadStaticPipelineCache() { + const auto create = [this](size_t size, const void* data) { + static_pipeline_cache = logical.CreatePipelineCache({ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .initialDataSize = size, + .pInitialData = data, + }); + }; + if (!owns_static_pipeline_cache) { + create(0, nullptr); + return; + } + const auto filename = StaticPipelineCacheFilename(); + if (filename.empty()) { + create(0, nullptr); + return; + } + std::vector data; + try { + std::ifstream file(filename, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + create(0, nullptr); + return; + } + file.exceptions(std::ifstream::failbit | std::ifstream::badbit); + const size_t total = static_cast(file.tellg()); + file.seekg(0, std::ios::beg); + std::array magic{}; + u32 version{}; + if (total < magic.size() + sizeof(version)) { + create(0, nullptr); + return; + } + file.read(magic.data(), magic.size()) + .read(reinterpret_cast(&version), sizeof(version)); + if (magic != STATIC_CACHE_MAGIC_NUMBER || version != STATIC_CACHE_VERSION) { + LOG_INFO(Render_Vulkan, "Discarding stale static pipeline cache"); + create(0, nullptr); + return; + } + data.resize(total - magic.size() - sizeof(version)); + file.read(data.data(), static_cast(data.size())); + } catch (const std::ios_base::failure& e) { + LOG_WARNING(Render_Vulkan, "Failed to read static pipeline cache: {}", e.what()); + create(0, nullptr); + return; + } + create(data.size(), data.empty() ? nullptr : data.data()); + LOG_INFO(Render_Vulkan, "Loaded {} bytes of static pipeline cache", data.size()); +} + +void Device::SaveStaticPipelineCache() const { + if (!owns_static_pipeline_cache || !static_pipeline_cache) { + return; + } + const auto filename = StaticPipelineCacheFilename(); + if (filename.empty()) { + return; + } + size_t size = 0; + std::vector data; + static_pipeline_cache.Read(&size, nullptr); + if (size == 0) { + return; + } + data.resize(size); + static_pipeline_cache.Read(&size, data.data()); + try { + std::ofstream file(filename, std::ios::binary | std::ios::trunc); + file.exceptions(std::ofstream::failbit); + if (!file.is_open()) { + return; + } + file.write(STATIC_CACHE_MAGIC_NUMBER.data(), STATIC_CACHE_MAGIC_NUMBER.size()) + .write(reinterpret_cast(&STATIC_CACHE_VERSION), + sizeof(STATIC_CACHE_VERSION)) + .write(data.data(), static_cast(size)); + } catch (const std::ios_base::failure& e) { + LOG_WARNING(Render_Vulkan, "Failed to write static pipeline cache: {}", e.what()); + Common::FS::RemoveFile(filename); + } +} + VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags wanted_usage, FormatType format_type) const { if (IsFormatSupported(wanted_format, wanted_usage, format_type)) { @@ -939,6 +1075,7 @@ bool Device::GetSuitability(bool requires_swapchain) { FOR_EACH_VK_FEATURE_EXT(FEATURE_EXTENSION); FOR_EACH_VK_EXTENSION(EXTENSION); + FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION); if (supported_extensions.contains(VK_KHR_ROBUSTNESS_2_EXTENSION_NAME)) { loaded_extensions.erase(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); @@ -951,6 +1088,17 @@ bool Device::GetSuitability(bool requires_swapchain) { extensions.robustness_2 = false; } +#ifdef __ANDROID__ + if (extensions.external_memory_ahb && !extensions.queue_family_foreign) { + LOG_INFO(Render_Vulkan, + "Not loading {} because its dependency {} is unavailable", + VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME, + VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME); + loaded_extensions.erase(VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME); + extensions.external_memory_ahb = false; + } +#endif + #undef FEATURE_EXTENSION #undef EXTENSION @@ -1083,6 +1231,11 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR; SetNext(next, properties.push_descriptor); } + if (extensions.descriptor_buffer) { + properties.descriptor_buffer.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_PROPERTIES_EXT; + SetNext(next, properties.descriptor_buffer); + } if (extensions.subgroup_size_control || features.subgroup_size_control.subgroupSizeControl) { properties.subgroup_size_control.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES; @@ -1098,6 +1251,21 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR; SetNext(next, properties.maintenance5); } + if (extensions.external_memory_host) { + properties.external_memory_host.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT; + SetNext(next, properties.external_memory_host); + } + if (extensions.maintenance4 || features.maintenance4.maintenance4) { + properties.maintenance4.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES; + SetNext(next, properties.maintenance4); + } + if (instance_version >= VK_API_VERSION_1_2) { + properties.depth_stencil_resolve.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES; + SetNext(next, properties.depth_stencil_resolve); + } // Perform the property fetch. physical.GetProperties2(properties2); @@ -1209,10 +1377,20 @@ void Device::RemoveUnsuitableExtensions() { RemoveExtensionFeatureIfUnsuitable(extensions.depth_bias_control, features.depth_bias_control, VK_EXT_DEPTH_BIAS_CONTROL_EXTENSION_NAME); + // VK_EXT_depth_clamp_zero_one + extensions.depth_clamp_zero_one = features.depth_clamp_zero_one.depthClampZeroOne; + RemoveExtensionFeatureIfUnsuitable(extensions.depth_clamp_zero_one, + features.depth_clamp_zero_one, + VK_EXT_DEPTH_CLAMP_ZERO_ONE_EXTENSION_NAME); + // VK_EXT_depth_clip_control extensions.depth_clip_control = features.depth_clip_control.depthClipControl; RemoveExtensionFeatureIfUnsuitable(extensions.depth_clip_control, features.depth_clip_control, VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME); + // VK_EXT_depth_clip_enable + extensions.depth_clip_enable = features.depth_clip_enable.depthClipEnable; + RemoveExtensionFeatureIfUnsuitable(extensions.depth_clip_enable, features.depth_clip_enable, + VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME); // VK_EXT_extended_dynamic_state extensions.extended_dynamic_state = features.extended_dynamic_state.extendedDynamicState; @@ -1326,6 +1504,12 @@ void Device::RemoveUnsuitableExtensions() { features.subgroup_size_control, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + // VK_EXT_texture_compression_astc_hdr + extensions.texture_compression_astc_hdr = false; + RemoveExtensionFeatureIfUnsuitable(extensions.texture_compression_astc_hdr, + features.texture_compression_astc_hdr, + VK_EXT_TEXTURE_COMPRESSION_ASTC_HDR_EXTENSION_NAME); + // VK_EXT_transform_feedback extensions.transform_feedback = features.transform_feedback.transformFeedback && @@ -1353,7 +1537,18 @@ void Device::RemoveUnsuitableExtensions() { VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME); } + // VK_KHR_shader_quad_control + extensions.shader_quad_control = features.shader_quad_control.shaderQuadControl; + RemoveExtensionFeatureIfUnsuitable(extensions.shader_quad_control, features.shader_quad_control, + VK_KHR_SHADER_QUAD_CONTROL_EXTENSION_NAME); + // VK_KHR_workgroup_memory_explicit_layout +#ifdef __ANDROID__ + if (!Settings::values.workgroup_memory_explicit_layout.GetValue()) { + LOG_INFO(Render_Vulkan, "Workgroup memory explicit layout disabled by user setting"); + features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout = false; + } +#endif extensions.workgroup_memory_explicit_layout = features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout && features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayoutScalarBlockLayout; @@ -1373,6 +1568,15 @@ void Device::RemoveUnsuitableExtensions() { extensions.maintenance3 = loaded_extensions.contains(VK_KHR_MAINTENANCE_3_EXTENSION_NAME); RemoveExtensionIfUnsuitable(extensions.maintenance3, VK_KHR_MAINTENANCE_3_EXTENSION_NAME); + // VK_KHR_dynamic_rendering + if (!Settings::values.dynamic_rendering.GetValue()) { + LOG_INFO(Render_Vulkan, "Dynamic rendering disabled by user setting"); + features.dynamic_rendering.dynamicRendering = false; + } + extensions.dynamic_rendering = features.dynamic_rendering.dynamicRendering; + RemoveExtensionFeatureIfUnsuitable(extensions.dynamic_rendering, features.dynamic_rendering, + VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME); + // VK_KHR_maintenance4 extensions.maintenance4 = features.maintenance4.maintenance4; RemoveExtensionFeatureIfUnsuitable(extensions.maintenance4, features.maintenance4, @@ -1444,8 +1648,28 @@ std::optional Device::GetSamplerHeapBudget() const { return sampler_heap_budget; } +Device::MemoryBudgetInfo Device::GetMemoryBudgetInfo() const { + std::array budgets{}; + vmaGetHeapBudgets(allocator, budgets.data()); + MemoryBudgetInfo info{}; + for (const size_t heap : valid_heap_memory) { + info.usage += budgets[heap].usage; + info.budget += budgets[heap].budget; + info.block_bytes += budgets[heap].statistics.blockBytes; + info.allocation_bytes += budgets[heap].statistics.allocationBytes; + } + return info; +} + +void Device::TickAllocatorFrame() const { + vmaSetCurrentFrameIndex(allocator, ++allocator_frame_index); +} + u64 Device::GetDeviceMemoryUsage() const { - VkPhysicalDeviceMemoryBudgetPropertiesEXT budget; + if (!extensions.memory_budget) { + return GetMemoryBudgetInfo().allocation_bytes; + } + VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; budget.pNext = nullptr; physical.GetMemoryProperties(&budget); @@ -1467,12 +1691,27 @@ void Device::CollectPhysicalMemoryInfo() { device_access_memory = 0; u64 device_initial_usage = 0; u64 local_memory = 0; + const auto heap_has_usable_type = [&mem_properties](size_t heap) { + for (u32 index = 0; index < mem_properties.memoryTypeCount; ++index) { + if (mem_properties.memoryTypes[index].heapIndex != heap) { + continue; + } + if ((mem_properties.memoryTypes[index].propertyFlags & + VK_MEMORY_PROPERTY_PROTECTED_BIT) == 0) { + return true; + } + } + return false; + }; for (size_t element = 0; element < num_properties; ++element) { const bool is_heap_local = (mem_properties.memoryHeaps[element].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0; if (!is_integrated && !is_heap_local) { continue; } + if (!heap_has_usable_type(element)) { + continue; + } valid_heap_memory.push_back(element); if (is_heap_local) { local_memory += mem_properties.memoryHeaps[element].size; @@ -1484,18 +1723,35 @@ void Device::CollectPhysicalMemoryInfo() { } device_access_memory += mem_properties.memoryHeaps[element].size; } + const u64 committed_backing = Common::GetCommittedBackingSize(); + if (committed_backing != 0) { + LOG_INFO(Render_Vulkan, "Discounting {} MiB of guest memory committed by the host", + committed_backing >> 20); + local_memory -= std::min(local_memory, committed_backing); + device_access_memory -= std::min(device_access_memory, committed_backing); + } if (is_integrated) { - const s64 available_memory = static_cast(device_access_memory - device_initial_usage); - const u64 memory_size = Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive ? 6_GiB : 4_GiB; - device_access_memory = static_cast(std::max(std::min(available_memory - 8_GiB, memory_size), std::min(local_memory, memory_size))); + const bool aggressive = + Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive; + const s64 available_memory = + static_cast(device_access_memory) - static_cast(device_initial_usage); + const u64 reserve = std::min(local_memory / 8, 1_GiB); + const u64 usable = available_memory > static_cast(reserve) + ? static_cast(available_memory) - reserve + : 0; + const u64 cap = (local_memory / 8) * (aggressive ? 6 : 5); + device_access_memory = std::max(std::min(usable, cap), 1_GiB); } else { const u64 reserve_memory = std::min(device_access_memory / 8, 1_GiB); device_access_memory -= reserve_memory; if (Settings::values.vram_usage_mode.GetValue() != Settings::VramUsageMode::Aggressive) { // Account for resolution scaling in memory limits - const size_t normal_memory = 6_GiB; - const size_t scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1); - device_access_memory = std::min(device_access_memory, normal_memory + scaler_memory); + const u64 normal_memory = 6_GiB; + const u64 scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1); + const u64 baseline = normal_memory + scaler_memory; + const u64 proportional = (device_access_memory / 4) * 3; + device_access_memory = + std::min(device_access_memory, std::max(baseline, proportional)); } } } diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index c449f60324..89238a4015 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -36,6 +36,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, DescriptorIndexing, DESCRIPTOR_INDEXING, descriptor_indexing) \ FEATURE(EXT, HostQueryReset, HOST_QUERY_RESET, host_query_reset) \ FEATURE(KHR, 8BitStorage, 8BIT_STORAGE, bit8_storage) \ + FEATURE(KHR, BufferDeviceAddress, BUFFER_DEVICE_ADDRESS, buffer_device_address) \ FEATURE(KHR, TimelineSemaphore, TIMELINE_SEMAPHORE, timeline_semaphore) #define FOR_EACH_VK_FEATURE_1_3(FEATURE) \ @@ -43,6 +44,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, ShaderDemoteToHelperInvocation, SHADER_DEMOTE_TO_HELPER_INVOCATION, \ shader_demote_to_helper_invocation) \ FEATURE(EXT, SubgroupSizeControl, SUBGROUP_SIZE_CONTROL, subgroup_size_control) \ + FEATURE(KHR, DynamicRendering, DYNAMIC_RENDERING, dynamic_rendering) \ FEATURE(KHR, Maintenance4, MAINTENANCE_4, maintenance4) \ FEATURE(KHR, Synchronization2, SYNCHRONIZATION_2, synchronization2) @@ -54,7 +56,10 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, ColorWriteEnable, COLOR_WRITE_ENABLE, color_write_enable) \ FEATURE(EXT, CustomBorderColor, CUSTOM_BORDER_COLOR, custom_border_color) \ FEATURE(EXT, DepthBiasControl, DEPTH_BIAS_CONTROL, depth_bias_control) \ + FEATURE(EXT, DepthClampZeroOne, DEPTH_CLAMP_ZERO_ONE, depth_clamp_zero_one) \ FEATURE(EXT, DepthClipControl, DEPTH_CLIP_CONTROL, depth_clip_control) \ + FEATURE(EXT, DepthClipEnable, DEPTH_CLIP_ENABLE, depth_clip_enable) \ + FEATURE(EXT, DescriptorBuffer, DESCRIPTOR_BUFFER, descriptor_buffer) \ FEATURE(EXT, ExtendedDynamicState, EXTENDED_DYNAMIC_STATE, extended_dynamic_state) \ FEATURE(EXT, ExtendedDynamicState2, EXTENDED_DYNAMIC_STATE_2, extended_dynamic_state2) \ FEATURE(EXT, ExtendedDynamicState3, EXTENDED_DYNAMIC_STATE_3, extended_dynamic_state3) \ @@ -71,6 +76,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(KHR, Maintenance6, MAINTENANCE_6, maintenance6) \ FEATURE(KHR, PipelineExecutableProperties, PIPELINE_EXECUTABLE_PROPERTIES, \ pipeline_executable_properties) \ + FEATURE(KHR, ShaderQuadControl, SHADER_QUAD_CONTROL, shader_quad_control) \ FEATURE(KHR, WorkgroupMemoryExplicitLayout, WORKGROUP_MEMORY_EXPLICIT_LAYOUT, \ workgroup_memory_explicit_layout) \ FEATURE(EXT, TextureCompressionASTCHDR, TEXTURE_COMPRESSION_ASTC_HDR, \ @@ -79,10 +85,13 @@ VK_DEFINE_HANDLE(VmaAllocator) // Define miscellaneous extensions which may be used by the implementation here. #define FOR_EACH_VK_EXTENSION(EXTENSION) \ + EXTENSION(EXT, ASTC_DECODE_MODE, astc_decode_mode) \ EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering) \ EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization) \ EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted) \ + EXTENSION(EXT, EXTERNAL_MEMORY_HOST, external_memory_host) \ EXTENSION(EXT, MEMORY_BUDGET, memory_budget) \ + EXTENSION(EXT, PIPELINE_CREATION_FEEDBACK, pipeline_creation_feedback) \ EXTENSION(EXT, ROBUSTNESS_2, robustness_2) \ EXTENSION(EXT, SAMPLER_FILTER_MINMAX, sampler_filter_minmax) \ EXTENSION(EXT, SHADER_STENCIL_EXPORT, shader_stencil_export) \ @@ -111,6 +120,14 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION(IMG, FILTER_CUBIC, filter_cubic_img) \ EXTENSION(QCOM, FILTER_CUBIC_WEIGHTS, filter_cubic_weights) +#ifdef __ANDROID__ +#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) \ + EXTENSION(EXT, QUEUE_FAMILY_FOREIGN, queue_family_foreign) \ + EXTENSION(ANDROID, EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER, external_memory_ahb) +#else +#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) +#endif + // Define extensions which must be supported. #define FOR_EACH_VK_MANDATORY_EXTENSION(EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME) \ @@ -255,6 +272,17 @@ public: return allocator; } + struct MemoryBudgetInfo { + u64 usage; + u64 budget; + u64 block_bytes; + u64 allocation_bytes; + }; + + MemoryBudgetInfo GetMemoryBudgetInfo() const; + + void TickAllocatorFrame() const; + /// Returns the logical device. const vk::Device& GetLogical() const { return logical; @@ -265,6 +293,10 @@ public: return physical; } + VkPipelineCache StaticPipelineCache() const noexcept { + return *static_pipeline_cache; + } + /// Returns the main graphics queue. vk::Queue GetGraphicsQueue() const { return graphics_queue; @@ -369,8 +401,7 @@ FN_MAX_LIMIT_LIST } bool IsOptimalAstcSupported() const { - return features.features.textureCompressionASTC_LDR && - features.texture_compression_astc_hdr.textureCompressionASTC_HDR; + return is_optimal_astc_supported; } /// Returns true if BCn is natively supported. @@ -468,6 +499,29 @@ FN_MAX_LIMIT_LIST return properties.push_descriptor.maxPushDescriptors; } + /// Returns true if the device supports descriptor buffers. + bool IsExtPipelineCreationFeedbackSupported() const { + return extensions.pipeline_creation_feedback; + } + + bool IsRobustBufferAccessEnabled() const { + return features.features.robustBufferAccess == VK_TRUE; + } + + bool IsExtDescriptorBufferSupported() const { + return extensions.descriptor_buffer; + } + + /// Returns the descriptor buffer properties of the device. + const VkPhysicalDeviceDescriptorBufferPropertiesEXT& DescriptorBufferProperties() const { + return properties.descriptor_buffer; + } + + /// Returns true if the device supports buffer device address. + bool IsBufferDeviceAddressSupported() const { + return extensions.buffer_device_address; + } + /// Returns true if formatless image load is supported. bool IsFormatlessImageLoadSupported() const { return features.features.shaderStorageImageReadWithoutFormat; @@ -554,6 +608,11 @@ FN_MAX_LIMIT_LIST features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout16BitAccess && features.features.shaderInt16; } + + /// Returns true if the device supports VK_KHR_shader_quad_control. + bool IsKhrShaderQuadControlSupported() const { + return extensions.shader_quad_control && features.shader_quad_control.shaderQuadControl; + } /// Returns true if the device supports VK_KHR_image_format_list. bool IsKhrImageFormatListSupported() const { @@ -586,7 +645,6 @@ FN_MAX_LIMIT_LIST } /// Returns true if depth/stencil operations can be performed efficiently. - /// Either through shader export or hardware blits. bool CanPerformDepthStencilOperations() const { return extensions.shader_stencil_export || is_blit_depth24_stencil8_supported || is_blit_depth32_stencil8_supported; @@ -602,6 +660,16 @@ FN_MAX_LIMIT_LIST return extensions.depth_clip_control; } + /// Returns true if the device supports VK_EXT_depth_clamp_zero_one. + bool IsExtDepthClampZeroOneSupported() const { + return extensions.depth_clamp_zero_one; + } + + /// Returns true if the device supports VK_EXT_depth_clip_enable. + bool IsExtDepthClipEnableSupported() const { + return extensions.depth_clip_enable; + } + /// Returns true if the device supports VK_EXT_depth_bias_control. bool IsExtDepthBiasControlSupported() const { return extensions.depth_bias_control; @@ -734,6 +802,38 @@ FN_MAX_LIMIT_LIST return features.line_rasterization.stippledRectangularLines != VK_FALSE; } + VkLineRasterizationModeEXT GetLineRasterizationMode(bool wants_smooth) const { + if (wants_smooth && SupportsSmoothLines()) { + return VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT; + } + if (SupportsRectangularLines()) { + return VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT; + } + return VK_LINE_RASTERIZATION_MODE_DEFAULT_EXT; + } + + bool SupportsStippleForMode(VkLineRasterizationModeEXT mode) const { + switch (mode) { + case VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT: + return features.line_rasterization.stippledSmoothLines != VK_FALSE; + case VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT: + return features.line_rasterization.stippledBresenhamLines != VK_FALSE; + default: + return features.line_rasterization.stippledRectangularLines != VK_FALSE; + } + } + + float ClampLineWidth(float width) const { + if (!features.features.wideLines) { + return 1.0f; + } + const auto& range = properties.properties.limits.lineWidthRange; + if (!(width >= range[0])) { + return range[0]; + } + return width > range[1] ? range[1] : width; + } + bool SupportsAlphaToOne() const { return features.features.alphaToOne != VK_FALSE; } @@ -812,10 +912,41 @@ FN_MAX_LIMIT_LIST features.shader_atomic_int64.shaderSharedInt64Atomics; } + /// Returns true if the device supports VK_EXT_conditional_rendering. bool IsExtConditionalRendering() const { return extensions.conditional_rendering; } + /// Returns true if the device supports VK_EXT_astc_decode_mode. + bool IsExtAstcDecodeModeSupported() const { + return extensions.astc_decode_mode; + } + + /// Returns true if the device supports VK_EXT_external_memory_host. + bool IsExtExternalMemoryHostSupported() const { + return extensions.external_memory_host; + } + + bool IsExtExternalMemoryAhbSupported() const { +#ifdef __ANDROID__ + return extensions.external_memory_ahb && extensions.queue_family_foreign; +#else + return false; +#endif + } + + bool IsDescriptorBindingPartiallyBoundSupported() const { + return features.descriptor_indexing.descriptorBindingPartiallyBound; + } + + u64 GetMinImportedHostPointerAlignment() const { + return properties.external_memory_host.minImportedHostPointerAlignment; + } + + u64 GetMaxBufferSize() const { + return properties.maintenance4.maxBufferSize; + } + bool HasTimelineSemaphore() const; /// Returns true if the device supports VK_KHR_synchronization2. @@ -876,6 +1007,10 @@ FN_MAX_LIMIT_LIST u64 GetDeviceMemoryUsage() const; + VkSampleCountFlags GetStorageImageSampleCounts() const { + return properties.properties.limits.storageImageSampleCounts; + } + u32 GetSetsPerPool() const { return sets_per_pool; } @@ -941,6 +1076,33 @@ FN_MAX_LIMIT_LIST return extensions.maintenance3; } + static constexpr bool ENABLE_DYNAMIC_RENDERING = true; + + /// Returns true if the device supports VK_KHR_dynamic_rendering. + bool IsKhrDynamicRenderingSupported() const { + return ENABLE_DYNAMIC_RENDERING && extensions.dynamic_rendering; + } + + /// Returns the resolve mode to use for multisampled depth. + VkResolveModeFlagBits GetDepthResolveMode() const { + return PickResolveMode(properties.depth_stencil_resolve.supportedDepthResolveModes); + } + + /// Same as GetDepthResolveMode, for the stencil aspect. + VkResolveModeFlagBits GetStencilResolveMode() const { + return PickResolveMode(properties.depth_stencil_resolve.supportedStencilResolveModes); + } + + /// Returns true if depth and stencil may resolve with independent modes. + bool IsIndependentResolveSupported() const { + return properties.depth_stencil_resolve.independentResolve == VK_TRUE; + } + + /// Returns true if one aspect may resolve while the other does not. + bool IsIndependentResolveNoneSupported() const { + return properties.depth_stencil_resolve.independentResolveNone == VK_TRUE; + } + /// Returns true if the device supports VK_KHR_maintenance4. bool IsKhrMaintenance4Supported() const { return extensions.maintenance4; @@ -1022,6 +1184,22 @@ FN_MAX_LIMIT_LIST void ShutdownGPULogging(); private: + /// Picks a usable resolve mode out of a supported-modes mask. SAMPLE_ZERO is preferred because + /// it is the only mode the spec guarantees whenever any depth/stencil resolve is supported, + /// and averaging depth samples is not meaningful for depth testing anyway. + static VkResolveModeFlagBits PickResolveMode(VkResolveModeFlags supported) { + if ((supported & VK_RESOLVE_MODE_SAMPLE_ZERO_BIT) != 0) { + return VK_RESOLVE_MODE_SAMPLE_ZERO_BIT; + } + if ((supported & VK_RESOLVE_MODE_MIN_BIT) != 0) { + return VK_RESOLVE_MODE_MIN_BIT; + } + if ((supported & VK_RESOLVE_MODE_MAX_BIT) != 0) { + return VK_RESOLVE_MODE_MAX_BIT; + } + return VK_RESOLVE_MODE_NONE; + } + /// Checks if the physical device is suitable and configures the object state /// with all necessary info about its properties. bool GetSuitability(bool requires_swapchain); @@ -1057,14 +1235,20 @@ private: /// Returns true if the device natively supports blitting depth stencil images. bool TestDepthStencilBlits(VkFormat format) const; + void LoadStaticPipelineCache(); + void SaveStaticPipelineCache() const; + private: VkInstance instance; ///< Vulkan instance. VmaAllocator allocator; ///< VMA allocator. + mutable u32 allocator_frame_index{}; vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. vk::Queue graphics_queue; ///< Main graphics queue. vk::Queue present_queue; ///< Main present queue. + vk::PipelineCache static_pipeline_cache; + bool owns_static_pipeline_cache{}; u32 instance_version{}; ///< Vulkan instance version. u32 graphics_family{}; ///< Main graphics queue family index. u32 present_family{}; ///< Main present queue family index. @@ -1079,6 +1263,7 @@ private: FOR_EACH_VK_FEATURE_1_4(FEATURE); FOR_EACH_VK_FEATURE_EXT(FEATURE); FOR_EACH_VK_EXTENSION(EXTENSION); + FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION); #undef EXTENSION #undef FEATURE @@ -1107,9 +1292,13 @@ private: VkPhysicalDeviceSubgroupProperties subgroup_properties{}; VkPhysicalDeviceFloatControlsProperties float_controls{}; VkPhysicalDevicePushDescriptorPropertiesKHR push_descriptor{}; + VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{}; VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{}; VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{}; + VkPhysicalDeviceMaintenance4Properties maintenance4{}; VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{}; + VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{}; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host{}; VkPhysicalDeviceProperties properties{}; }; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index e57864ede8..0ad90ccafd 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -25,31 +25,36 @@ #include "video_core/gpu_logging/gpu_logging.h" #include "common/settings.h" +#ifdef __ANDROID__ +#include +#endif + namespace Vulkan { namespace { -// Helpers translating MemoryUsage to flags/usage - - [[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) { - switch (usage) { - case MemoryUsage::DeviceLocal: - return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - case MemoryUsage::Upload: - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - case MemoryUsage::Download: - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; - case MemoryUsage::Stream: - return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + [[nodiscard]] std::optional FindImportMemoryType( + const VkPhysicalDeviceMemoryProperties &props, u32 type_mask) { + const auto find = [&](VkMemoryPropertyFlags wanted) -> std::optional { + for (u32 i = 0; i < props.memoryTypeCount; ++i) { + if (((type_mask >> i) & 1u) != 0 && + (props.memoryTypes[i].propertyFlags & wanted) == wanted) { + return i; + } + } + return std::nullopt; + }; + auto type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + if (!type_index) { + type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } - ASSERT_MSG(false, "Invalid memory usage={}", usage); - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + return type_index; } +// Helpers translating MemoryUsage to flags/usage + [[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferredVmaFlags(MemoryUsage usage) { if (usage == MemoryUsage::Download) { return VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; @@ -86,125 +91,254 @@ namespace Vulkan { } -// This avoids calling vkGetBufferMemoryRequirements* directly. - template - static VkBuffer GetVkHandleFromBuffer(const T &buf) { - if constexpr (requires { static_cast(buf); }) { - return static_cast(buf); - } else if constexpr (requires {{ buf.GetHandle() } -> std::convertible_to; }) { - return buf.GetHandle(); - } else if constexpr (requires {{ buf.Handle() } -> std::convertible_to; }) { - return buf.Handle(); - } else if constexpr (requires {{ buf.vk_handle() } -> std::convertible_to; }) { - return buf.vk_handle(); - } else { - static_assert(sizeof(T) == 0, "Cannot extract VkBuffer handle from vk::Buffer"); - return VK_NULL_HANDLE; - } - } - } // namespace -//MemoryCommit is now VMA-backed - MemoryCommit::MemoryCommit(VmaAllocator alloc, VmaAllocation a, - const VmaAllocationInfo &info) noexcept - : allocator{alloc}, allocation{a}, memory{info.deviceMemory}, - offset{info.offset}, size{info.size}, mapped_ptr{info.pMappedData} { - // Log GPU memory allocation - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_memory_tracking.GetValue()) { - GPU::Logging::GPULogger::GetInstance().LogMemoryAllocation( - reinterpret_cast(memory), - static_cast(size), - 0 // Memory property flags (not easily available from VMA) - ); + HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window) + : device{device_} { + if (ImportHostPointer(base, size)) { + return; + } + ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, size); + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path"); } } - MemoryCommit::~MemoryCommit() { Release(); } - - MemoryCommit::MemoryCommit(MemoryCommit &&rhs) noexcept - : allocator{std::exchange(rhs.allocator, nullptr)}, - allocation{std::exchange(rhs.allocation, nullptr)}, - memory{std::exchange(rhs.memory, VK_NULL_HANDLE)}, - offset{std::exchange(rhs.offset, 0)}, - size{std::exchange(rhs.size, 0)}, - mapped_ptr{std::exchange(rhs.mapped_ptr, nullptr)} {} - - MemoryCommit &MemoryCommit::operator=(MemoryCommit &&rhs) noexcept { - if (this != &rhs) { - Release(); - allocator = std::exchange(rhs.allocator, nullptr); - allocation = std::exchange(rhs.allocation, nullptr); - memory = std::exchange(rhs.memory, VK_NULL_HANDLE); - offset = std::exchange(rhs.offset, 0); - size = std::exchange(rhs.size, 0); - mapped_ptr = std::exchange(rhs.mapped_ptr, nullptr); + bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { + if (!device.IsExtExternalMemoryHostSupported()) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, VK_EXT_external_memory_host is not supported"); + return false; } - return *this; - } - - std::span MemoryCommit::Map() - { - if (!allocation) return {}; - if (!mapped_ptr) { - if (vmaMapMemory(allocator, allocation, &mapped_ptr) != VK_SUCCESS) return {}; + const u64 alignment = device.GetMinImportedHostPointerAlignment(); + if (alignment == 0 || !Common::IsAligned(reinterpret_cast(base), alignment) || + !Common::IsAligned(size, alignment)) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, host allocation does not satisfy alignment {}", + alignment); + return false; } - const size_t n = static_cast(std::min(size, - (std::numeric_limits::max)())); - return std::span{static_cast(mapped_ptr), n}; - } - - std::span MemoryCommit::Map() const - { - if (!allocation) return {}; - if (!mapped_ptr) { - void *p = nullptr; - if (vmaMapMemory(allocator, allocation, &p) != VK_SUCCESS) return {}; - const_cast(this)->mapped_ptr = p; + using namespace Common::Literals; + VkDeviceSize candidate_window = 1_GiB; + const u64 max_buffer_size = device.GetMaxBufferSize(); + if (max_buffer_size != 0 && max_buffer_size < candidate_window) { + candidate_window = Common::AlignDown(max_buffer_size, alignment); } - const size_t n = static_cast(std::min(size, - (std::numeric_limits::max)())); - return std::span{static_cast(mapped_ptr), n}; - } - - void MemoryCommit::Unmap() - { - if (allocation && mapped_ptr) { - vmaUnmapMemory(allocator, allocation); - mapped_ptr = nullptr; + if (candidate_window == 0) { + return false; } - } + window_size = candidate_window; - void MemoryCommit::Release() { - if (allocation && allocator) { - // Log GPU memory deallocation - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_memory_tracking.GetValue() && - memory != VK_NULL_HANDLE) { - GPU::Logging::GPULogger::GetInstance().LogMemoryDeallocation( - reinterpret_cast(memory) - ); + const auto &logical = device.GetLogical(); + const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; + + for (size_t offset = 0; offset < size; offset += window_size) { + u8 *const window_base = static_cast(base) + offset; + const VkDeviceSize window_len = + (std::min)(static_cast(size - offset), window_size); + VkMemoryHostPointerPropertiesEXT host_props{ + .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, + .pNext = nullptr, + .memoryTypeBits = 0, + }; + if (logical.GetMemoryHostPointerPropertiesEXT( + VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, window_base, + &host_props) != VK_SUCCESS || + host_props.memoryTypeBits == 0) { + break; } - - if (mapped_ptr) { - vmaUnmapMemory(allocator, allocation); - mapped_ptr = nullptr; + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & host_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > window_len) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const u32 heap_index = memory_props.memoryTypes[*type_index].heapIndex; + const VkDeviceSize heap_size = memory_props.memoryHeaps[heap_index].size; + if (imported_size + window_len > heap_size / 2) { + LOG_INFO(Render_Vulkan, + "Stopping guest memory import at {} MiB to leave room on heap {} of {} MiB", + imported_size >> 20, heap_index, heap_size >> 20); + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportMemoryHostPointerInfoEXT import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, + .pNext = nullptr, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, + .pHostPointer = window_base, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &import_info, + .allocationSize = window_len, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + }); + imported_size += static_cast(window_len); + } + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Host pointer import failed"); + return false; + } + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory for unified memory access in {} windows", + imported_size >> 20, windows.size()); + return true; + } + + void HostMemoryImport::ImportHardwareBuffers( + [[maybe_unused]] std::span hardware_buffers, + [[maybe_unused]] size_t hardware_buffer_window, [[maybe_unused]] size_t size) { +#ifdef __ANDROID__ + if (hardware_buffers.empty() || hardware_buffer_window == 0 || + !device.IsExtExternalMemoryAhbSupported()) { + return; + } + const auto &logical = device.GetLogical(); + const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; + window_size = hardware_buffer_window; + for (size_t i = 0; i < hardware_buffers.size(); ++i) { + const size_t offset = i * hardware_buffer_window; + if (offset >= size) { + break; + } + const VkDeviceSize window_len = (std::min)( + static_cast(size - offset), + static_cast(hardware_buffer_window)); + VkAndroidHardwareBufferPropertiesANDROID ahb_props{ + .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID, + .pNext = nullptr, + .allocationSize = 0, + .memoryTypeBits = 0, + }; + if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i], + &ahb_props) != VK_SUCCESS || + ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) { + break; + } + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > ahb_props.allocationSize) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportAndroidHardwareBufferInfoANDROID import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, + .pNext = nullptr, + .buffer = hardware_buffers[i], + }; + const VkMemoryDedicatedAllocateInfo dedicated_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, + .pNext = &import_info, + .image = VK_NULL_HANDLE, + .buffer = new_buffer, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &dedicated_info, + .allocationSize = ahb_props.allocationSize, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + }); + imported_size += static_cast(window_len); + } + if (!windows.empty()) { + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory via hardware buffers in {} windows", + imported_size >> 20, windows.size()); + } +#endif + } + + HostMemoryImport::~HostMemoryImport() { + for (Window &window : windows) { + if (window.buffer != VK_NULL_HANDLE) { + device.GetLogical().DestroyBufferRaw(window.buffer); } - vmaFreeMemory(allocator, allocation); } - allocation = nullptr; - allocator = nullptr; - memory = VK_NULL_HANDLE; - offset = 0; - size = 0; } MemoryAllocator::MemoryAllocator(const Device &device_) : device{device_}, allocator{device.GetAllocator()}, - properties{device_.GetPhysical().GetMemoryProperties().memoryProperties}, - buffer_image_granularity{ - device_.GetPhysical().GetProperties().limits.bufferImageGranularity} { + properties{device_.GetPhysical().GetMemoryProperties().memoryProperties} { // Preserve the previous "RenderDoc small heap" trimming behavior that we had in original vma minus the heap bug if (device.HasDebuggingToolAttached()) @@ -224,10 +358,25 @@ namespace Vulkan { MemoryAllocator::~MemoryAllocator() = default; + void MemoryAllocator::SetReclaimCallback(ReclaimCallback callback) { + reclaim_callback = std::move(callback); + vk::SetAllocatorOwnerThread(); + } + + bool MemoryAllocator::ReclaimAtLeast(u64 hint_bytes) const { + if (!reclaim_callback || in_reclaim) { + return false; + } + in_reclaim = true; + const u64 freed = reclaim_callback(hint_bytes); + in_reclaim = false; + return freed > 0; + } + vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const { const VmaAllocationCreateInfo alloc_ci = { - .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + .flags = device.CanReportMemoryUsage() ? VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT : 0u, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, .requiredFlags = 0, .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, @@ -240,7 +389,26 @@ namespace Vulkan { VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + DEBUG_ASSERT(vk::OnAllocatorOwnerThread()); + + VkResult res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && ReclaimAtLeast(IMAGE_RECLAIM_HINT)) { + res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + } + + if (res != VK_SUCCESS) { + auto relaxed_ci = alloc_ci; + relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS) { + relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + } + } + + vk::Check(res); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -262,7 +430,8 @@ namespace Vulkan { && device.GetDriverID() == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA) ? VK_MEMORY_PROPERTY_HOST_CACHED_BIT : 0; const VmaAllocationCreateInfo alloc_ci = { - .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage), + .flags = (device.CanReportMemoryUsage() ? VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT : 0u) | + MemoryUsageVmaFlags(usage), .usage = MemoryUsageVma(usage), .requiredFlags = 0, .preferredFlags = MemoryUsagePreferredVmaFlags(usage) | anv_flags, @@ -277,7 +446,28 @@ namespace Vulkan { VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + DEBUG_ASSERT(vk::OnAllocatorOwnerThread()); + + VkResult res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && ReclaimAtLeast(ci.size)) { + res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + } + + if (res != VK_SUCCESS) { + auto relaxed_ci = alloc_ci; + relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && + (relaxed_ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, + &alloc_info); + } + } + + vk::Check(res); vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers @@ -299,77 +489,4 @@ namespace Vulkan { device.GetDispatchLoader()); } - MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements &reqs, MemoryUsage usage) - { - const auto vma_usage = MemoryUsageVma(usage); - VmaAllocationCreateInfo ci{}; - ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage); - ci.usage = vma_usage; - ci.memoryTypeBits = reqs.memoryTypeBits & valid_memory_types; - ci.requiredFlags = 0; - ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage); - - VmaAllocation a{}; - VmaAllocationInfo info{}; - - VkResult res = vmaAllocateMemory(allocator, &reqs, &ci, &a, &info); - - if (res != VK_SUCCESS) { - // Relax 1: drop budget constraint - auto ci2 = ci; - ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; - res = vmaAllocateMemory(allocator, &reqs, &ci2, &a, &info); - - // Relax 2: if we preferred DEVICE_LOCAL, drop that preference - if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { - auto ci3 = ci2; - ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - res = vmaAllocateMemory(allocator, &reqs, &ci3, &a, &info); - } - } - - vk::Check(res); - return MemoryCommit(allocator, a, info); - } - - MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage) { - // Allocate memory appropriate for this buffer automatically - const auto vma_usage = MemoryUsageVma(usage); - - VmaAllocationCreateInfo ci{}; - ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage); - ci.usage = vma_usage; - ci.requiredFlags = 0; - ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage); - ci.pool = VK_NULL_HANDLE; - ci.pUserData = nullptr; - ci.priority = 0.0f; - - const VkBuffer raw = *buffer; - - VmaAllocation a{}; - VmaAllocationInfo info{}; - - // Let VMA infer memory requirements from the buffer - VkResult res = vmaAllocateMemoryForBuffer(allocator, raw, &ci, &a, &info); - - if (res != VK_SUCCESS) { - auto ci2 = ci; - ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; - res = vmaAllocateMemoryForBuffer(allocator, raw, &ci2, &a, &info); - - if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { - auto ci3 = ci2; - ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - res = vmaAllocateMemoryForBuffer(allocator, raw, &ci3, &a, &info); - } - } - - vk::Check(res); - vk::Check(vmaBindBufferMemory2(allocator, a, 0, raw, nullptr)); - return MemoryCommit(allocator, a, info); - } - - - } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 581f2e66d2..411836fe85 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -15,6 +16,8 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" #include "video_core/vulkan_common/vma.h" +struct AHardwareBuffer; + namespace Vulkan { class Device; @@ -39,49 +42,53 @@ namespace Vulkan { } } -/// Ownership handle of a memory commitment (real VMA allocation). - class MemoryCommit { + class HostMemoryImport { public: - MemoryCommit() noexcept = default; + explicit HostMemoryImport(const Device &device_, void *base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window); - MemoryCommit(VmaAllocator allocator, VmaAllocation allocation, - const VmaAllocationInfo &info) noexcept; + ~HostMemoryImport(); - ~MemoryCommit(); + HostMemoryImport(const HostMemoryImport &) = delete; - MemoryCommit(const MemoryCommit &) = delete; + HostMemoryImport &operator=(const HostMemoryImport &) = delete; - MemoryCommit &operator=(const MemoryCommit &) = delete; + [[nodiscard]] bool IsValid() const noexcept { + return !windows.empty(); + } - MemoryCommit(MemoryCommit &&) noexcept; + [[nodiscard]] size_t GetSize() const noexcept { + return imported_size; + } - MemoryCommit &operator=(MemoryCommit &&) noexcept; + [[nodiscard]] VkDeviceSize GetWindowSize() const noexcept { + return window_size; + } - [[nodiscard]] std::span Map(); + [[nodiscard]] VkBuffer GetWindowBuffer(size_t index) const noexcept { + return windows[index].buffer; + } - [[nodiscard]] std::span Map() const; - - void Unmap(); - - explicit operator bool() const noexcept { return allocation != nullptr; } - - VkDeviceMemory Memory() const noexcept { return memory; } - - VkDeviceSize Offset() const noexcept { return offset; } - - VkDeviceSize Size() const noexcept { return size; } - - VmaAllocation Allocation() const noexcept { return allocation; } + [[nodiscard]] size_t GetWindowCount() const noexcept { + return windows.size(); + } private: - void Release(); + struct Window { + vk::DeviceMemory memory; + VkBuffer buffer{}; + }; - VmaAllocator allocator{}; ///< VMA allocator - VmaAllocation allocation{}; ///< VMA allocation handle - VkDeviceMemory memory{}; ///< Underlying VkDeviceMemory chosen by VMA - VkDeviceSize offset{}; ///< Offset of this allocation inside VkDeviceMemory - VkDeviceSize size{}; ///< Size of the allocation - void *mapped_ptr{}; ///< Optional persistent mapped pointer + bool ImportHostPointer(void *base, size_t size); + + void ImportHardwareBuffers(std::span hardware_buffers, + size_t hardware_buffer_window, size_t size); + + const Device &device; + std::vector windows; + VkDeviceSize window_size{}; + size_t imported_size{}; }; /// Memory allocator container. @@ -107,36 +114,21 @@ namespace Vulkan { vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; - /** - * Commits a memory with the specified requirements. - * - * @param requirements Requirements returned from a Vulkan call. - * @param usage Indicates how the memory will be used. - * - * @returns A memory commit. - */ - MemoryCommit Commit(const VkMemoryRequirements &requirements, MemoryUsage usage); + using ReclaimCallback = std::function; - /// Commits memory required by the buffer and binds it (for buffers created outside VMA). - MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage); + void SetReclaimCallback(ReclaimCallback callback); private: - static bool IsAutoUsage(VmaMemoryUsage u) noexcept { - switch (u) { - case VMA_MEMORY_USAGE_AUTO: - case VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE: - case VMA_MEMORY_USAGE_AUTO_PREFER_HOST: - return true; - default: - return false; - } - } + bool ReclaimAtLeast(u64 hint_bytes) const; + + static constexpr u64 IMAGE_RECLAIM_HINT = 64ULL * 1024 * 1024; const Device &device; ///< Device handle. VmaAllocator allocator; ///< VMA allocator. const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. - VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + ReclaimCallback reclaim_callback; + mutable bool in_reclaim{false}; }; } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 4943705859..024aca07d4 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -5,11 +5,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include #include +#include #include +#include #include #include +#include "common/assert.h" #include "common/common_types.h" #include "common/logging.h" #include "video_core/vulkan_common/vk_enum_string_helper.h" @@ -20,6 +25,60 @@ namespace Vulkan::vk { namespace { +std::thread::id allocator_owner_thread; + +template +struct PendingRelease { + VmaAllocator allocator; + HandleType handle; + VmaAllocation allocation; + u64 timeline; +}; + +std::mutex deletion_mutex; +std::atomic deletion_timeline{1}; +std::vector> pending_images; +std::vector> pending_buffers; + +template +void PushPendingRelease(std::vector>& pending, VmaAllocator allocator, + HandleType handle, VmaAllocation allocation) noexcept { + std::scoped_lock lock{deletion_mutex}; + pending.push_back(PendingRelease{ + .allocator = allocator, + .handle = handle, + .allocation = allocation, + .timeline = deletion_timeline.load(std::memory_order_acquire), + }); +} + +template +void ExtractReleased(std::vector>& pending, + std::vector>& released, u64 completed_value) { + const auto split = std::partition(pending.begin(), pending.end(), + [completed_value](const PendingRelease& entry) { + return entry.timeline > completed_value; + }); + released.assign(split, pending.end()); + pending.erase(split, pending.end()); +} + +void DrainDeletionQueue(u64 completed_value) noexcept { + std::vector> images; + std::vector> buffers; + { + std::scoped_lock lock{deletion_mutex}; + ExtractReleased(pending_images, images, completed_value); + ExtractReleased(pending_buffers, buffers, completed_value); + } + for (const auto& entry : images) { + vmaDestroyImage(entry.allocator, entry.handle, entry.allocation); + } + for (const auto& entry : buffers) { + vmaDestroyBuffer(entry.allocator, entry.handle, entry.allocation); + } +} + template void SortPhysicalDevices(std::vector& devices, const InstanceDispatch& dld, Func&& func) { @@ -92,6 +151,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdBeginConditionalRenderingEXT); X(vkCmdBeginQuery); X(vkCmdBeginRenderPass); + X(vkCmdBeginRendering); X(vkCmdBeginTransformFeedbackEXT); X(vkCmdBeginDebugUtilsLabelEXT); X(vkCmdBindDescriptorSets); @@ -119,6 +179,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdEndConditionalRenderingEXT); X(vkCmdEndQuery); X(vkCmdEndRenderPass); + X(vkCmdEndRendering); X(vkCmdEndTransformFeedbackEXT); X(vkCmdEndDebugUtilsLabelEXT); X(vkCmdFillBuffer); @@ -216,12 +277,16 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkGetBufferMemoryRequirements2); X(vkGetDeviceQueue); X(vkGetEventStatus); + X(vkGetMemoryHostPointerPropertiesEXT); X(vkGetFenceStatus); X(vkGetImageMemoryRequirements); X(vkGetPipelineCacheData); X(vkGetMemoryFdKHR); #ifdef _WIN32 X(vkGetMemoryWin32HandleKHR); +#endif +#ifdef __ANDROID__ + X(vkGetAndroidHardwareBufferPropertiesANDROID); #endif X(vkGetQueryPoolResults); X(vkGetPipelineExecutablePropertiesKHR); @@ -230,6 +295,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkMapMemory); X(vkQueueSubmit); X(vkQueueSubmit2); + X(vkResetCommandPool); X(vkResetFences); X(vkResetQueryPool); X(vkSetDebugUtilsObjectNameEXT); @@ -237,6 +303,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkUnmapMemory); X(vkUpdateDescriptorSetWithTemplate); X(vkUpdateDescriptorSets); + X(vkGetBufferDeviceAddress); + X(vkGetDescriptorSetLayoutSizeEXT); + X(vkGetDescriptorSetLayoutBindingOffsetEXT); + X(vkGetDescriptorEXT); + X(vkCmdBindDescriptorBuffersEXT); + X(vkCmdSetDescriptorBufferOffsetsEXT); X(vkWaitForFences); X(vkWaitSemaphores); @@ -257,6 +329,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { Proc(dld.vkCmdDrawIndexedIndirectCount, dld, "vkCmdDrawIndexedIndirectCountKHR", device); } + // Support for dynamic rendering is optional until Vulkan 1.3 + if (!dld.vkCmdBeginRendering) { + Proc(dld.vkCmdBeginRendering, dld, "vkCmdBeginRenderingKHR", device); + Proc(dld.vkCmdEndRendering, dld, "vkCmdEndRenderingKHR", device); + } + // Synchronization2 is core in Vulkan 1.3, otherwise requires VK_KHR_synchronization2 if (!dld.vkCmdPipelineBarrier2) { Proc(dld.vkCmdPipelineBarrier2, dld, "vkCmdPipelineBarrier2KHR", device); @@ -502,13 +580,35 @@ DebugReportCallback Instance::CreateDebugReportCallback( return DebugReportCallback(object, handle, *dld); } +void SetAllocatorOwnerThread() { + allocator_owner_thread = std::this_thread::get_id(); +} + +bool OnAllocatorOwnerThread() noexcept { + return allocator_owner_thread == std::thread::id{} || + allocator_owner_thread == std::this_thread::get_id(); +} + +void SetDeletionTimeline(u64 value) noexcept { + deletion_timeline.store(value, std::memory_order_release); +} + +void TickDeletionQueue(u64 completed_value) noexcept { + DEBUG_ASSERT(OnAllocatorOwnerThread()); + DrainDeletionQueue(completed_value); +} + +void FlushDeletionQueue() noexcept { + DrainDeletionQueue((std::numeric_limits::max)()); +} + void Image::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE, name); } void Image::Release() const noexcept { if (handle) { - vmaDestroyImage(allocator, handle, allocation); + PushPendingRelease(pending_images, allocator, handle, allocation); } } @@ -530,7 +630,7 @@ void Buffer::SetObjectNameEXT(const char* name) const { void Buffer::Release() const noexcept { if (handle) { - vmaDestroyBuffer(allocator, handle, allocation); + PushPendingRelease(pending_buffers, allocator, handle, allocation); } } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index ef975e2bda..77a4bda15c 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -131,6 +131,16 @@ private: VkResult result; }; +void SetAllocatorOwnerThread(); + +[[nodiscard]] bool OnAllocatorOwnerThread() noexcept; + +void SetDeletionTimeline(u64 value) noexcept; + +void TickDeletionQueue(u64 completed_value) noexcept; + +void FlushDeletionQueue() noexcept; + /// Throws a Vulkan exception if result is not success. inline void Check(VkResult result) { if (result != VK_SUCCESS) { @@ -208,6 +218,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdBeginDebugUtilsLabelEXT vkCmdBeginDebugUtilsLabelEXT{}; PFN_vkCmdBeginQuery vkCmdBeginQuery{}; PFN_vkCmdBeginRenderPass vkCmdBeginRenderPass{}; + PFN_vkCmdBeginRendering vkCmdBeginRendering{}; PFN_vkCmdBeginTransformFeedbackEXT vkCmdBeginTransformFeedbackEXT{}; PFN_vkCmdBindDescriptorSets vkCmdBindDescriptorSets{}; PFN_vkCmdBindIndexBuffer vkCmdBindIndexBuffer{}; @@ -236,6 +247,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdEndDebugUtilsLabelEXT vkCmdEndDebugUtilsLabelEXT{}; PFN_vkCmdEndQuery vkCmdEndQuery{}; PFN_vkCmdEndRenderPass vkCmdEndRenderPass{}; + PFN_vkCmdEndRendering vkCmdEndRendering{}; PFN_vkCmdEndTransformFeedbackEXT vkCmdEndTransformFeedbackEXT{}; PFN_vkCmdFillBuffer vkCmdFillBuffer{}; PFN_vkCmdPipelineBarrier vkCmdPipelineBarrier{}; @@ -332,12 +344,16 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{}; PFN_vkGetDeviceQueue vkGetDeviceQueue{}; PFN_vkGetEventStatus vkGetEventStatus{}; + PFN_vkGetMemoryHostPointerPropertiesEXT vkGetMemoryHostPointerPropertiesEXT{}; PFN_vkGetFenceStatus vkGetFenceStatus{}; PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{}; PFN_vkGetPipelineCacheData vkGetPipelineCacheData{}; PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{}; #ifdef _WIN32 PFN_vkGetMemoryWin32HandleKHR vkGetMemoryWin32HandleKHR{}; +#endif +#ifdef __ANDROID__ + PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID{}; #endif PFN_vkGetPipelineExecutablePropertiesKHR vkGetPipelineExecutablePropertiesKHR{}; PFN_vkGetPipelineExecutableStatisticsKHR vkGetPipelineExecutableStatisticsKHR{}; @@ -346,12 +362,19 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkMapMemory vkMapMemory{}; PFN_vkQueueSubmit vkQueueSubmit{}; PFN_vkQueueSubmit2 vkQueueSubmit2{}; + PFN_vkResetCommandPool vkResetCommandPool{}; PFN_vkResetFences vkResetFences{}; PFN_vkResetQueryPool vkResetQueryPool{}; PFN_vkSetDebugUtilsObjectNameEXT vkSetDebugUtilsObjectNameEXT{}; PFN_vkSetDebugUtilsObjectTagEXT vkSetDebugUtilsObjectTagEXT{}; PFN_vkUnmapMemory vkUnmapMemory{}; PFN_vkUpdateDescriptorSetWithTemplate vkUpdateDescriptorSetWithTemplate{}; + PFN_vkGetBufferDeviceAddress vkGetBufferDeviceAddress{}; + PFN_vkGetDescriptorSetLayoutSizeEXT vkGetDescriptorSetLayoutSizeEXT{}; + PFN_vkGetDescriptorSetLayoutBindingOffsetEXT vkGetDescriptorSetLayoutBindingOffsetEXT{}; + PFN_vkGetDescriptorEXT vkGetDescriptorEXT{}; + PFN_vkCmdBindDescriptorBuffersEXT vkCmdBindDescriptorBuffersEXT{}; + PFN_vkCmdSetDescriptorBufferOffsetsEXT vkCmdSetDescriptorBufferOffsetsEXT{}; PFN_vkUpdateDescriptorSets vkUpdateDescriptorSets{}; PFN_vkWaitForFences vkWaitForFences{}; PFN_vkWaitSemaphores vkWaitSemaphores{}; @@ -925,6 +948,10 @@ public: CommandBuffers Allocate(std::size_t num_buffers, VkCommandBufferLevel level = VK_COMMAND_BUFFER_LEVEL_PRIMARY) const; + void Reset(VkCommandPoolResetFlags flags = 0) const { + Check(dld->vkResetCommandPool(owner, handle, flags)); + } + /// Set object name. void SetObjectNameEXT(const char* name) const; }; @@ -1071,6 +1098,34 @@ public: VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer, void* pnext = nullptr) const noexcept; + VkResult GetMemoryHostPointerPropertiesEXT( + VkExternalMemoryHandleTypeFlagBits handle_type, const void* host_pointer, + VkMemoryHostPointerPropertiesEXT* out_properties) const noexcept { + return dld->vkGetMemoryHostPointerPropertiesEXT(handle, handle_type, host_pointer, + out_properties); + } + +#ifdef __ANDROID__ + VkResult GetAndroidHardwareBufferPropertiesANDROID( + const struct AHardwareBuffer* buffer, + VkAndroidHardwareBufferPropertiesANDROID* out_properties) const noexcept { + return dld->vkGetAndroidHardwareBufferPropertiesANDROID(handle, buffer, out_properties); + } +#endif + + VkResult CreateBufferRaw(const VkBufferCreateInfo& ci, VkBuffer* out_buffer) const noexcept { + return dld->vkCreateBuffer(handle, &ci, nullptr, out_buffer); + } + + void DestroyBufferRaw(VkBuffer buffer) const noexcept { + dld->vkDestroyBuffer(handle, buffer, nullptr); + } + + VkResult BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory, + VkDeviceSize offset) const noexcept { + return dld->vkBindBufferMemory(handle, buffer, memory, offset); + } + VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept; std::vector GetPipelineExecutablePropertiesKHR( @@ -1087,6 +1142,34 @@ public: dld->vkUpdateDescriptorSetWithTemplate(handle, set, update_template, data); } + [[nodiscard]] VkDeviceAddress GetBufferDeviceAddress(VkBuffer buffer) const noexcept { + const VkBufferDeviceAddressInfo info{ + .sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, + .pNext = nullptr, + .buffer = buffer, + }; + return dld->vkGetBufferDeviceAddress(handle, &info); + } + + [[nodiscard]] VkDeviceSize GetDescriptorSetLayoutSizeEXT( + VkDescriptorSetLayout layout) const noexcept { + VkDeviceSize size{}; + dld->vkGetDescriptorSetLayoutSizeEXT(handle, layout, &size); + return size; + } + + [[nodiscard]] VkDeviceSize GetDescriptorSetLayoutBindingOffsetEXT( + VkDescriptorSetLayout layout, u32 binding) const noexcept { + VkDeviceSize offset{}; + dld->vkGetDescriptorSetLayoutBindingOffsetEXT(handle, layout, binding, &offset); + return offset; + } + + void GetDescriptorEXT(const VkDescriptorGetInfoEXT& info, size_t size, + void* descriptor) const noexcept { + dld->vkGetDescriptorEXT(handle, &info, size, descriptor); + } + VkResult AcquireNextImageKHR(VkSwapchainKHR swapchain, u64 timeout, VkSemaphore semaphore, VkFence fence, u32* image_index) const noexcept { return dld->vkAcquireNextImageKHR(handle, swapchain, timeout, semaphore, fence, @@ -1186,6 +1269,14 @@ public: dld->vkCmdEndRenderPass(handle); } + void BeginRendering(const VkRenderingInfo& rendering_info) const noexcept { + dld->vkCmdBeginRendering(handle, &rendering_info); + } + + void EndRendering() const noexcept { + dld->vkCmdEndRendering(handle); + } + void BeginQuery(VkQueryPool query_pool, u32 query, VkQueryControlFlags flags) const noexcept { dld->vkCmdBeginQuery(handle, query_pool, query, flags); } @@ -1400,6 +1491,40 @@ public: PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, {}, {}, image_barrier); } + void BindDescriptorBuffersEXT(Span bindings) const noexcept { + dld->vkCmdBindDescriptorBuffersEXT(handle, bindings.size(), bindings.data()); + } + + void SetDescriptorBufferOffsetsEXT(VkPipelineBindPoint bind_point, VkPipelineLayout layout, + u32 first_set, Span buffer_indices, + Span offsets) const noexcept { + dld->vkCmdSetDescriptorBufferOffsetsEXT(handle, bind_point, layout, first_set, + buffer_indices.size(), buffer_indices.data(), + offsets.data()); + } + + [[nodiscard]] bool HasPipelineBarrier2() const noexcept { + return dld->vkCmdPipelineBarrier2 != nullptr; + } + + void PipelineBarrier2(VkDependencyFlags dependency_flags, + Span memory_barriers, + Span buffer_barriers, + Span image_barriers) const noexcept { + const VkDependencyInfo dependency_info{ + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .pNext = nullptr, + .dependencyFlags = dependency_flags, + .memoryBarrierCount = memory_barriers.size(), + .pMemoryBarriers = memory_barriers.data(), + .bufferMemoryBarrierCount = buffer_barriers.size(), + .pBufferMemoryBarriers = buffer_barriers.data(), + .imageMemoryBarrierCount = image_barriers.size(), + .pImageMemoryBarriers = image_barriers.data(), + }; + dld->vkCmdPipelineBarrier2(handle, &dependency_info); + } + void CopyBufferToImage(VkBuffer src_buffer, VkImage dst_image, VkImageLayout dst_image_layout, Span regions) const noexcept { dld->vkCmdCopyBufferToImage(handle, src_buffer, dst_image, dst_image_layout, regions.size(),