diff --git a/src/android/app/build.gradle.kts b/src/android/app/build.gradle.kts index a4076978a1..fd7c40e415 100644 --- a/src/android/app/build.gradle.kts +++ b/src/android/app/build.gradle.kts @@ -65,7 +65,7 @@ android { defaultConfig { applicationId = "dev.eden.eden_emulator" - minSdk = 24 + minSdk = 33 targetSdk = 36 versionName = getGitVersion() versionCode = autoVersion diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt index a8d6eaf6e1..797ac803eb 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/NativeLibrary.kt @@ -218,6 +218,8 @@ object NativeLibrary { external fun logSettings() + external fun refreshThreadPolicies() + external fun getDebugKnobAt(index: Int): Boolean /** diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt index 7f6e793199..16e74553cc 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt @@ -27,6 +27,7 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"), RENDERER_ASYNC_PRESENTATION("async_presentation"), RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"), + RENDERER_UNIFIED_MEMORY("use_unified_memory"), RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"), ENABLE_BUFFER_HISTORY("enable_buffer_history"), USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"), @@ -36,6 +37,8 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_DEBUG("debug"), RENDERER_PATCH_OLD_QCOM_DRIVERS("patch_old_qcom_drivers"), RENDERER_VERTEX_INPUT_DYNAMIC_STATE("vertex_input_dynamic_state"), + RENDERER_DYNAMIC_RENDERING("dynamic_rendering"), + RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT("workgroup_memory_explicit_layout"), RENDERER_SAMPLE_SHADING("sample_shading"), GPU_UNSWIZZLE_ENABLED("gpu_unswizzle_enabled"), PICTURE_IN_PICTURE("picture_in_picture"), diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt index 7c6063988f..12051e356e 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt @@ -155,6 +155,20 @@ abstract class SettingsItem( descriptionId = R.string.vertex_input_dynamic_state_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_DYNAMIC_RENDERING, + titleId = R.string.dynamic_rendering, + descriptionId = R.string.dynamic_rendering_description + ) + ) + put( + SwitchSetting( + BooleanSetting.RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT, + titleId = R.string.workgroup_memory_explicit_layout, + descriptionId = R.string.workgroup_memory_explicit_layout_description + ) + ) put( SliderSetting( IntSetting.RENDERER_SAMPLE_SHADING, @@ -594,7 +608,7 @@ abstract class SettingsItem( IntSetting.ANDROID_PIPELINE_WORKERS, titleId = R.string.pipeline_worker_cores, descriptionId = R.string.pipeline_worker_cores_description, - min = 4, + min = 2, max = 8, units = "cores" ) @@ -685,6 +699,13 @@ abstract class SettingsItem( descriptionId = R.string.renderer_asynchronous_shaders_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_UNIFIED_MEMORY, + titleId = R.string.renderer_unified_memory, + descriptionId = R.string.renderer_unified_memory_description + ) + ) put( SingleChoiceSetting( IntSetting.FAST_GPU_TIME, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt index 81702d7110..4f11fdb96d 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt @@ -304,6 +304,7 @@ class SettingsFragmentPresenter( add(BooleanSetting.EMULATE_BGR565.key) add(BooleanSetting.RESCALE_HACK.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key) + add(BooleanSetting.RENDERER_UNIFIED_MEMORY.key) add(IntSetting.ANDROID_PIPELINE_WORKERS.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key) add(BooleanSetting.RENDERER_ASYNC_PRESENTATION.key) @@ -313,6 +314,8 @@ class SettingsFragmentPresenter( add(IntSetting.RENDERER_DYNA_STATE.key) add(BooleanSetting.RENDERER_VERTEX_INPUT_DYNAMIC_STATE.key) + add(BooleanSetting.RENDERER_DYNAMIC_RENDERING.key) + add(BooleanSetting.RENDERER_WORKGROUP_MEMORY_EXPLICIT_LAYOUT.key) add(IntSetting.RENDERER_SAMPLE_SHADING.key) add(HeaderSetting(R.string.display)) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt index 0069e169b3..5b93c88161 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/fragments/EmulationFragment.kt @@ -1451,6 +1451,7 @@ class EmulationFragment : Fragment(), SurfaceHolder.Callback { override fun onResume() { super.onResume() + NativeLibrary.refreshThreadPolicies() val b = _binding ?: return updateStatsPosition(IntSetting.PERF_OVERLAY_POSITION.getInt()) updateSocPosition(IntSetting.SOC_OVERLAY_POSITION.getInt()) diff --git a/src/android/app/src/main/jni/android_settings.h b/src/android/app/src/main/jni/android_settings.h index 304ff62f07..37da651519 100644 --- a/src/android/app/src/main/jni/android_settings.h +++ b/src/android/app/src/main/jni/android_settings.h @@ -147,13 +147,6 @@ namespace AndroidSettings { &show_performance_overlay}; - Settings::Setting pipeline_worker_count{linkage, 4, "pipeline_worker_count", - Settings::Category::Android, - Settings::Specialization::Default, - true, - true}; - - Settings::Setting show_input_overlay{linkage, true, "show_input_overlay", Settings::Category::Overlay}; Settings::Setting overlay_snap_to_grid{linkage, false, "overlay_snap_to_grid", diff --git a/src/android/app/src/main/jni/native.cpp b/src/android/app/src/main/jni/native.cpp index 41ba1b2019..c367a06dfd 100644 --- a/src/android/app/src/main/jni/native.cpp +++ b/src/android/app/src/main/jni/native.cpp @@ -50,6 +50,7 @@ extern "C" { #include "common/scope_exit.h" #include "common/settings.h" #include "common/string_util.h" +#include "common/thread.h" #include "frontend_common/play_time_manager.h" #include "core/constants.h" #include "core/core.h" @@ -1182,6 +1183,10 @@ void Java_org_yuzu_yuzu_1emu_NativeLibrary_logSettings(JNIEnv* env, jobject jobj Settings::LogSettings(); } +void Java_org_yuzu_yuzu_1emu_NativeLibrary_refreshThreadPolicies(JNIEnv* env, jobject jobj) { + Common::RefreshThreadPolicies(); +} + jboolean Java_org_yuzu_yuzu_1emu_NativeLibrary_getDebugKnobAt(JNIEnv* env, jobject jobj, jint index) { return static_cast(Settings::getDebugKnobAt(static_cast(index))); } diff --git a/src/android/app/src/main/res/values/strings.xml b/src/android/app/src/main/res/values/strings.xml index cd5d0c41e7..b57c610e76 100644 --- a/src/android/app/src/main/res/values/strings.xml +++ b/src/android/app/src/main/res/values/strings.xml @@ -524,6 +524,8 @@ Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path Use asynchronous shaders Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches. + Unified memory access (UMA) + Allows GPU write buffer readbacks directly into guest memory, skipping the CPU staging copy. GPU Unswizzle Settings Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality. Enable GPU Unswizzle @@ -544,6 +546,10 @@ Disabled Vertex Input Dynamic State Enabling this feature allows for more flexible vertex input handling, potentially reducing pipeline compilation time in vertex/buffer. + Dynamic Rendering + Render without render pass and framebuffer objects. Results vary by driver: some gain performance, others lose it. + Workgroup Memory Explicit Layout + Let shaders declare explicit layouts for workgroup memory. Disabled by default: some Qualcomm drivers are unstable with it. Sample Shading Allows the fragment shader to execute per sample in a multi-sampled fragment instead once per fragment. Improves graphics quality at the cost of some performance. diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 445d34c444..cb7ec1ffa7 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -51,14 +51,45 @@ #endif // ^^^ POSIX ^^^ +#include #include #include +#include #include "common/alignment.h" #include "common/assert.h" #include "common/free_region_manager.h" #include "common/host_memory.h" #include "common/logging.h" +#include "common/memory_detect.h" +#include "common/settings.h" + +#ifdef __ANDROID__ +#include +#include + +namespace { + +struct NativeHandle { + int version; + int numFds; + int numInts; + int data[1]; +}; + +using PFN_AHardwareBuffer_getNativeHandle = const NativeHandle* (*)(const AHardwareBuffer*); + +PFN_AHardwareBuffer_getNativeHandle ResolveGetNativeHandle() { + void* const lib = dlopen("libnativewindow.so", RTLD_NOW); + if (lib == nullptr) { + return nullptr; + } + return reinterpret_cast( + dlsym(lib, "AHardwareBuffer_getNativeHandle")); +} + +} // namespace +#endif #if defined(__ANDROID__) && __ANDROID_API__ < 30 #include @@ -75,6 +106,12 @@ namespace Common { [[maybe_unused]] constexpr size_t PageAlignment = 0x1000; [[maybe_unused]] constexpr size_t HugePageSize = 0x200000; +static std::atomic committed_backing_size{}; + +u64 GetCommittedBackingSize() noexcept { + return committed_backing_size.load(std::memory_order_relaxed); +} + #ifdef _WIN32 // Manually imported for MinGW compatibility @@ -123,7 +160,7 @@ static void GetFuncAddress(Common::DynamicLibrary& dll, const char* name, T& pfn class HostMemory::Impl { public: - explicit Impl(size_t backing_size_, size_t virtual_size_) + explicit Impl(size_t backing_size_, size_t virtual_size_, size_t) : backing_size{backing_size_} , virtual_size{virtual_size_} , process{GetCurrentProcess()} @@ -229,6 +266,10 @@ public: UNREACHABLE(); } + bool IsBackingShared() const noexcept { + return true; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes @@ -501,9 +542,10 @@ static int shm_open_anon(int flags, mode_t mode) { class HostMemory::Impl { public: - explicit Impl(size_t backing_size_, size_t virtual_size_) + explicit Impl(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_) : backing_size{backing_size_} , virtual_size{virtual_size_} + , preferred_offset{preferred_offset_} {} bool Init() { @@ -543,10 +585,15 @@ public: LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones"); backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); if (fd > 0) { - fd = -1; close(fd); } + fd = -1; } else { +#ifdef __ANDROID__ + if (InitAhbBacking()) { + return InitVirtual(); + } +#endif backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); } if (backing_base == MAP_FAILED) { @@ -554,7 +601,10 @@ public: return false; } - // Virtual memory initialization + return InitVirtual(); + } + + bool InitVirtual() { virtual_base = virtual_map_base = static_cast(ChooseVirtualBase(virtual_size)); if (virtual_base == MAP_FAILED) { LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); @@ -567,6 +617,248 @@ public: return true; } +#ifdef __ANDROID__ + static AHardwareBuffer_Desc MakeBlobDesc(size_t len) { + return AHardwareBuffer_Desc{ + .width = static_cast(len), + .height = 1, + .layers = 1, + .format = AHARDWAREBUFFER_FORMAT_BLOB, + .usage = AHARDWAREBUFFER_USAGE_CPU_READ_OFTEN | + AHARDWAREBUFFER_USAGE_CPU_WRITE_OFTEN | + AHARDWAREBUFFER_USAGE_GPU_DATA_BUFFER, + .stride = 0, + .rfu0 = 0, + .rfu1 = 0, + }; + } + + static bool ProbeAhbBacking(PFN_AHardwareBuffer_getNativeHandle get_native_handle) { + const AHardwareBuffer_Desc desc = MakeBlobDesc(PageAlignment * 2); + AHardwareBuffer* buffer{}; + if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { + LOG_WARNING(HW_Memory, "Hardware buffer probe allocation failed"); + return false; + } + const NativeHandle* const handle = get_native_handle(buffer); + if (handle == nullptr || handle->numFds < 1) { + LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor"); + AHardwareBuffer_release(buffer); + return false; + } + const int probe_fd = handle->data[0]; + bool ok = true; + const auto try_map = [&](int prot, off_t offset, const char* what) { + if (!ok) { + return; + } + void* const ptr = mmap(nullptr, PageAlignment, prot, MAP_SHARED, probe_fd, offset); + if (ptr == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Hardware buffer backing rejects {}: {}", what, + strerror(errno)); + ok = false; + return; + } + munmap(ptr, PageAlignment); + }; + try_map(PROT_READ | PROT_WRITE, 0, "shared mappings"); + try_map(PROT_READ | PROT_WRITE, static_cast(PageAlignment), "mappings at an offset"); +#ifdef ARCHITECTURE_arm64 + try_map(PROT_READ | PROT_EXEC, 0, "executable mappings"); +#endif + AHardwareBuffer_release(buffer); + return ok; + } + + size_t ComputeAhbBudget(size_t window_size) const { + const u64 total_physical = Common::GetMemInfo().TotalPhysicalMemory; + if (total_physical == 0) { + LOG_WARNING(HW_Memory, "Host memory size is unknown, not committing hardware buffers"); + return 0; + } + constexpr u64 MinimumTotalPhysical = 7ULL << 30; + if (total_physical < MinimumTotalPhysical) { + LOG_INFO(HW_Memory, + "Skipping hardware buffer backing, {} MiB of RAM is below the {} MiB minimum", + total_physical >> 20, MinimumTotalPhysical >> 20); + return 0; + } + const u64 max_map_count = Common::GetMaxMapCount(); + constexpr u64 ReservedMaps = 24576; + if (max_map_count == 0 || max_map_count <= ReservedMaps) { + LOG_WARNING(HW_Memory, + "Skipping hardware buffer backing, vm.max_map_count is unknown or too low"); + return 0; + } + u64 budget = total_physical / 6; + budget = (std::min)(budget, (max_map_count - ReservedMaps) * PageAlignment); + const u64 available = Common::GetAvailablePhysicalMemory(); + if (available != 0) { + constexpr u64 Headroom = 2ULL << 30; + budget = (std::min)(budget, available > Headroom ? available - Headroom : 0); + } + budget = (std::min)(budget, static_cast(backing_size)); + budget = Common::AlignDown(budget, window_size); + constexpr u64 MinimumBudget = 256ULL << 20; + if (budget < MinimumBudget) { + LOG_INFO(HW_Memory, + "Skipping hardware buffer backing, only {} MiB could be committed on a {} MiB " + "system with {} MiB available and vm.max_map_count {}", + budget >> 20, total_physical >> 20, available >> 20, max_map_count); + return 0; + } + return static_cast(budget); + } + + bool InitAhbBacking() { + if (!Settings::values.use_unified_memory.GetValue()) { + return false; + } + static const PFN_AHardwareBuffer_getNativeHandle get_native_handle = + ResolveGetNativeHandle(); + if (get_native_handle == nullptr) { + LOG_WARNING(HW_Memory, "AHardwareBuffer_getNativeHandle is not available"); + return false; + } + constexpr size_t window_size = 64ULL << 20; + const AHardwareBuffer_Desc window_desc = MakeBlobDesc(window_size); + if (AHardwareBuffer_isSupported(&window_desc) == 0) { + LOG_WARNING(HW_Memory, "Allocator rejects {} MiB hardware buffer windows", + window_size >> 20); + return false; + } + const size_t budget = ComputeAhbBudget(window_size); + if (budget == 0) { + return false; + } + if (!ProbeAhbBacking(get_native_handle)) { + return false; + } + const size_t aligned_backing = Common::AlignDown(backing_size, window_size); + const size_t region_size = (std::min)(budget, aligned_backing); + const size_t region_base = Common::AlignDown( + (std::min)(preferred_offset, aligned_backing - region_size), window_size); + const size_t num_windows = region_size / window_size; + + std::vector buffers; + std::vector buffer_fds; + const auto cleanup = [&] { + for (AHardwareBuffer* buffer : buffers) { + AHardwareBuffer_release(buffer); + } + buffers.clear(); + buffer_fds.clear(); + }; + for (size_t i = 0; i < num_windows; ++i) { + const AHardwareBuffer_Desc desc = MakeBlobDesc(window_size); + AHardwareBuffer* buffer{}; + if (AHardwareBuffer_allocate(&desc, &buffer) != 0 || buffer == nullptr) { + LOG_WARNING(HW_Memory, "Hardware buffer allocation failed for window {} of {}", i, + num_windows); + cleanup(); + return false; + } + buffers.push_back(buffer); + const NativeHandle* const handle = get_native_handle(buffer); + if (handle == nullptr || handle->numFds < 1) { + LOG_WARNING(HW_Memory, "Hardware buffer has no mappable file descriptor"); + cleanup(); + return false; + } + const int buffer_fd = handle->data[0]; + const off_t buffer_len = lseek(buffer_fd, 0, SEEK_END); + if (buffer_len < static_cast(window_size)) { + LOG_WARNING(HW_Memory, "Hardware buffer descriptor smaller than requested"); + cleanup(); + return false; + } + buffer_fds.push_back(buffer_fd); + } + u8* const base = static_cast(mmap(nullptr, backing_size, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0)); + if (base == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Failed to reserve backing address space: {}", strerror(errno)); + cleanup(); + return false; + } + const auto map_over_reservation = [&](size_t offset, size_t len, int map_fd, + off_t map_offset) { + if (len == 0) { + return true; + } + if (mmap(base + offset, len, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, map_fd, + map_offset) == MAP_FAILED) { + LOG_WARNING(HW_Memory, "Backing mmap failed: {}", strerror(errno)); + munmap(base, backing_size); + cleanup(); + return false; + } + return true; + }; + if (!map_over_reservation(0, region_base, fd, 0)) { + return false; + } + for (size_t i = 0; i < num_windows; ++i) { + if (!map_over_reservation(region_base + i * window_size, window_size, buffer_fds[i], + 0)) { + return false; + } + } + const size_t tail_offset = region_base + region_size; + if (!map_over_reservation(tail_offset, backing_size - tail_offset, fd, + static_cast(tail_offset))) { + return false; + } + backing_base = base; + ahb_windows = std::move(buffers); + ahb_fds = std::move(buffer_fds); + ahb_window_size = window_size; + ahb_base = region_base; + ahb_bytes = region_size; + committed_backing_size.store(region_size, std::memory_order_relaxed); + LOG_INFO(HW_Memory, + "Guest memory {:#x}-{:#x} backed by {} hardware buffer windows, {} MiB committed", + region_base, region_base + region_size, ahb_windows.size(), region_size >> 20); + return true; + } + + void MapBackingRange(size_t virtual_offset, size_t host_offset, size_t length, int prot_flags) { + while (length > 0) { + int map_fd = fd; + off_t map_offset = static_cast(host_offset); + size_t chunk = length; + if (host_offset < ahb_base) { + chunk = (std::min)(chunk, ahb_base - host_offset); + } else if (host_offset < ahb_base + ahb_bytes) { + const size_t relative = host_offset - ahb_base; + const size_t window = relative / ahb_window_size; + const size_t local = relative % ahb_window_size; + map_fd = ahb_fds[window]; + map_offset = static_cast(local); + chunk = (std::min)(chunk, ahb_window_size - local); + } + void* const ret = mmap(virtual_base + virtual_offset, chunk, prot_flags, + MAP_SHARED | MAP_FIXED, map_fd, map_offset); + ASSERT_MSG(ret != MAP_FAILED, "mmap: {}", strerror(errno)); + virtual_offset += chunk; + host_offset += chunk; + length -= chunk; + } + } + + std::span AhbWindows() const noexcept { + return ahb_windows; + } + + size_t AhbWindowSize() const noexcept { + return ahb_bytes != 0 ? ahb_window_size : 0; + } + + size_t AhbBase() const noexcept { + return ahb_base; + } +#endif + ~Impl() { Release(); } @@ -587,6 +879,12 @@ public: #ifdef ARCHITECTURE_arm64 if (True(perms & MemoryPermission::Execute)) prot_flags |= PROT_EXEC; +#endif +#ifdef __ANDROID__ + if (ahb_bytes != 0) { + MapBackingRange(virtual_offset, host_offset, length, prot_flags); + return; + } #endif int flags = (fd >= 0 ? MAP_SHARED : MAP_PRIVATE) | MAP_FIXED; void* ret = mmap(virtual_base + virtual_offset, length, prot_flags, flags, fd, host_offset); @@ -632,8 +930,18 @@ public: virtual_base = nullptr; } + bool IsBackingShared() const noexcept { +#ifdef __ANDROID__ + if (ahb_bytes != 0) { + return true; + } +#endif + return fd >= 0; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes + const size_t preferred_offset; u8* backing_base{reinterpret_cast(MAP_FAILED)}; u8* virtual_base{reinterpret_cast(MAP_FAILED)}; @@ -656,6 +964,18 @@ private: int ret = close(fd); ASSERT_MSG(ret == 0, "close failed: {}", strerror(errno)); } + +#ifdef __ANDROID__ + for (AHardwareBuffer* buffer : ahb_windows) { + AHardwareBuffer_release(buffer); + } + ahb_windows.clear(); + ahb_fds.clear(); + if (ahb_bytes != 0) { + committed_backing_size.store(0, std::memory_order_relaxed); + ahb_bytes = 0; + } +#endif } void AdjustMap(size_t* virtual_offset, size_t* length) { @@ -681,11 +1001,19 @@ private: int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create FreeRegionManager free_manager{}; + +#ifdef __ANDROID__ + std::vector ahb_windows; + std::vector ahb_fds; + size_t ahb_window_size{}; + size_t ahb_base{}; + size_t ahb_bytes{}; +#endif }; #endif // ^^^ POSIX ^^^ -HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) +HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_) : backing_size(backing_size_) , virtual_size(virtual_size_) { @@ -697,7 +1025,7 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) #else // Try to allocate a fastmem arena. // The implementation will fail with std::bad_alloc on errors. - impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize); + impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize, preferred_offset_); if (impl->Init()) { backing_base = impl->backing_base; virtual_base = impl->virtual_base; @@ -767,6 +1095,38 @@ void HostMemory::ClearBackingRegion(size_t physical_offset, size_t length, u32 f std::memset(backing_base + physical_offset, fill_value, length); } +std::span HostMemory::BackingHardwareBuffers() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbWindows() : std::span{}; +#else + return {}; +#endif +} + +size_t HostMemory::BackingHardwareBufferWindowSize() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbWindowSize() : 0; +#else + return 0; +#endif +} + +bool HostMemory::IsBackingShared() const noexcept { +#if defined(__OPENORBIS__) || defined(__managarm__) + return false; +#else + return impl && impl->IsBackingShared(); +#endif +} + +size_t HostMemory::BackingHardwareBufferBase() const noexcept { +#ifdef __ANDROID__ + return impl ? impl->AhbBase() : 0; +#else + return 0; +#endif +} + void HostMemory::EnableDirectMappedAddress() { #if !(defined(__OPENORBIS__) || defined(__managarm__)) if (impl) { diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 24f4670732..13d4830028 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -8,12 +8,17 @@ #include #include +#include #include "common/common_funcs.h" #include "common/common_types.h" #include "common/virtual_buffer.h" +struct AHardwareBuffer; + namespace Common { +[[nodiscard]] u64 GetCommittedBackingSize() noexcept; + enum class MemoryPermission : u32 { Read = 1 << 0, Write = 1 << 1, @@ -28,7 +33,7 @@ DECLARE_ENUM_FLAG_OPERATORS(MemoryPermission) */ class HostMemory { public: - explicit HostMemory(size_t backing_size_, size_t virtual_size_); + explicit HostMemory(size_t backing_size_, size_t virtual_size_, size_t preferred_offset_ = 0); ~HostMemory(); /** @@ -62,6 +67,18 @@ public: return backing_base; } + [[nodiscard]] size_t BackingSize() const noexcept { + return backing_size; + } + + [[nodiscard]] std::span BackingHardwareBuffers() const noexcept; + + [[nodiscard]] size_t BackingHardwareBufferWindowSize() const noexcept; + + [[nodiscard]] size_t BackingHardwareBufferBase() const noexcept; + + [[nodiscard]] bool IsBackingShared() const noexcept; + [[nodiscard]] u8* VirtualBasePointer() noexcept { return virtual_base; } diff --git a/src/common/memory_detect.cpp b/src/common/memory_detect.cpp index 86a3abcc6d..d0f9518bf5 100644 --- a/src/common/memory_detect.cpp +++ b/src/common/memory_detect.cpp @@ -17,6 +17,10 @@ #endif #endif +#include +#include +#include + #include "common/memory_detect.h" namespace Common { @@ -69,4 +73,55 @@ const MemoryInfo& GetMemInfo() { return mem_info; } +u64 GetAvailablePhysicalMemory() { +#ifdef _WIN32 + MEMORYSTATUSEX memorystatus; + memorystatus.dwLength = sizeof(memorystatus); + if (GlobalMemoryStatusEx(&memorystatus)) { + return memorystatus.ullAvailPhys; + } + return 0; +#elif defined(__linux__) + if (std::FILE* const file = std::fopen("/proc/meminfo", "re")) { + char line[256]; + u64 available = 0; + while (std::fgets(line, sizeof(line), file) != nullptr) { + if (std::strncmp(line, "MemAvailable:", 13) == 0) { + available = std::strtoull(line + 13, nullptr, 10) * 1024ULL; + break; + } + } + std::fclose(file); + if (available != 0) { + return available; + } + } + struct sysinfo info; + if (sysinfo(&info) == 0) { + const u64 unit = info.mem_unit != 0 ? info.mem_unit : 1ULL; + return (static_cast(info.freeram) + static_cast(info.bufferram)) * unit; + } + return 0; +#else + return 0; +#endif +} + +u64 GetMaxMapCount() { +#ifdef __linux__ + if (std::FILE* const file = std::fopen("/proc/sys/vm/max_map_count", "re")) { + char line[32]; + u64 count = 0; + if (std::fgets(line, sizeof(line), file) != nullptr) { + count = std::strtoull(line, nullptr, 10); + } + std::fclose(file); + return count; + } + return 0; +#else + return 0; +#endif +} + } // namespace Common diff --git a/src/common/memory_detect.h b/src/common/memory_detect.h index c8f239aed3..3bf013b79a 100644 --- a/src/common/memory_detect.h +++ b/src/common/memory_detect.h @@ -18,4 +18,8 @@ struct MemoryInfo { */ [[nodiscard]] const MemoryInfo& GetMemInfo(); +[[nodiscard]] u64 GetAvailablePhysicalMemory(); + +[[nodiscard]] u64 GetMaxMapCount(); + } // namespace Common diff --git a/src/common/settings.h b/src/common/settings.h index 00ba7dddc9..0b74601a6e 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -587,6 +587,9 @@ struct Values { SwitchableSetting use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders", Category::RendererHacks}; + SwitchableSetting use_unified_memory{linkage, false, "use_unified_memory", + Category::RendererHacks}; + SwitchableSetting gpu_unswizzle_texture_size{linkage, GpuUnswizzleSize::Large, "gpu_unswizzle_texture_size", @@ -635,6 +638,16 @@ struct Values { #endif "vertex_input_dynamic_state", Category::RendererExtensions}; + SwitchableSetting dynamic_rendering{linkage, true, "dynamic_rendering", + Category::RendererExtensions}; + + SwitchableSetting workgroup_memory_explicit_layout{ + linkage, false, "workgroup_memory_explicit_layout", Category::RendererExtensions}; + + SwitchableSetting pipeline_worker_count{ + linkage, 2, 2, 8, "pipeline_worker_count", Category::RendererAdvanced, + Specialization::Scalar}; + Setting renderer_debug{linkage, false, "debug", Category::RendererDebug}; Setting renderer_shader_feedback{linkage, false, "shader_feedback", Category::RendererDebug}; diff --git a/src/common/thread.cpp b/src/common/thread.cpp index c947c88373..fd5ba319cc 100644 --- a/src/common/thread.cpp +++ b/src/common/thread.cpp @@ -1,5 +1,6 @@ // SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: 2013 Dolphin Emulator Project // SPDX-FileCopyrightText: 2014 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -39,6 +40,258 @@ #include #endif +#ifdef __ANDROID__ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +constexpr int ANDROID_THREAD_PRIORITY_AUDIO = -16; +constexpr int ANDROID_THREAD_PRIORITY_URGENT_DISPLAY = -8; +constexpr int ANDROID_THREAD_PRIORITY_DISPLAY = -4; +constexpr int ANDROID_THREAD_PRIORITY_DEFAULT = 0; +constexpr int ANDROID_THREAD_PRIORITY_BACKGROUND = 10; + +constexpr size_t ANDROID_MINIMUM_PERFORMANCE_CORES = 4; + +enum class CoreGroup { + Unrestricted, + Performance, + Efficiency, +}; + +struct CoreTopology { + cpu_set_t allowed; + cpu_set_t performance; + cpu_set_t efficiency; + bool separated; + bool initialized; +}; + +struct ThreadPolicy { + pid_t tid; + CoreGroup group; + int nice_value; + bool has_nice; +}; + +std::mutex g_topology_mutex; +CoreTopology g_topology{}; + +std::mutex g_policy_mutex; + +std::vector& Policies() { + static auto* const policies = new std::vector(); + return *policies; +} + +struct PolicyRegistration { + ~PolicyRegistration() { + const pid_t tid = gettid(); + std::scoped_lock lock{g_policy_mutex}; + std::erase_if(Policies(), [tid](const ThreadPolicy& policy) { return policy.tid == tid; }); + } +}; + +thread_local PolicyRegistration t_policy_registration; + +int PossibleCpuCount() { + std::ifstream file("/sys/devices/system/cpu/possible"); + std::string list; + if (file && std::getline(file, list) && !list.empty()) { + int highest = -1; + const char* cursor = list.c_str(); + while (*cursor != '\0') { + char* end = nullptr; + const long value = std::strtol(cursor, &end, 10); + if (end == cursor) { + break; + } + highest = (std::max)(highest, static_cast(value)); + cursor = end; + while (*cursor == '-' || *cursor == ',') { + ++cursor; + } + } + if (highest >= 0) { + return (std::min)(highest + 1, CPU_SETSIZE); + } + } + const long configured = sysconf(_SC_NPROCESSORS_CONF); + if (configured > 0) { + return static_cast((std::min)(configured, CPU_SETSIZE)); + } + return static_cast((std::min)(std::thread::hardware_concurrency(), CPU_SETSIZE)); +} + +long ReadCpuScalar(int cpu, const char* node) { + long value = 0; + std::ifstream file("/sys/devices/system/cpu/cpu" + std::to_string(cpu) + "/" + node); + if (!file || !(file >> value) || value <= 0) { + return 0; + } + return value; +} + +std::vector> CollectCoreWeights(const cpu_set_t& allowed, int total, + const char* node, bool require_all) { + std::vector> cores; + for (int cpu = 0; cpu < total; ++cpu) { + if (!CPU_ISSET(cpu, &allowed)) { + continue; + } + const long weight = ReadCpuScalar(cpu, node); + if (weight <= 0) { + if (require_all) { + return {}; + } + LOG_WARNING(Common, "Could not read {} for CPU {}, treating it as an efficiency core", + node, cpu); + continue; + } + cores.emplace_back(weight, cpu); + } + return cores; +} + +void ComputeTopologyLocked() { + g_topology.initialized = true; + g_topology.separated = false; + CPU_ZERO(&g_topology.allowed); + CPU_ZERO(&g_topology.performance); + CPU_ZERO(&g_topology.efficiency); + + if (sched_getaffinity(getpid(), sizeof(g_topology.allowed), &g_topology.allowed) != 0) { + LOG_WARNING(Common, "Could not query process CPU affinity: {}", + ::Common::GetLastErrorMsg()); + return; + } + + const int total = PossibleCpuCount(); + auto cores = CollectCoreWeights(g_topology.allowed, total, "cpu_capacity", true); + if (cores.empty()) { + cores = CollectCoreWeights(g_topology.allowed, total, "cpufreq/cpuinfo_max_freq", false); + } + if (cores.empty()) { + LOG_WARNING(Common, "Could not determine CPU topology, thread placement is disabled"); + return; + } + + std::sort(cores.begin(), cores.end(), + [](const auto& lhs, const auto& rhs) { return lhs.first > rhs.first; }); + + const size_t allowed_count = static_cast(CPU_COUNT(&g_topology.allowed)); + const size_t maximum = + allowed_count > 2 * ANDROID_MINIMUM_PERFORMANCE_CORES + ? allowed_count - ANDROID_MINIMUM_PERFORMANCE_CORES + : ANDROID_MINIMUM_PERFORMANCE_CORES; + + size_t taken = 0; + long cluster_weight = cores.front().first; + for (const auto& [weight, cpu] : cores) { + if (weight != cluster_weight) { + if (taken >= ANDROID_MINIMUM_PERFORMANCE_CORES) { + break; + } + cluster_weight = weight; + } + if (taken >= maximum) { + break; + } + CPU_SET(cpu, &g_topology.performance); + ++taken; + } + if (taken == 0) { + return; + } + + for (int cpu = 0; cpu < total; ++cpu) { + if (CPU_ISSET(cpu, &g_topology.allowed) && !CPU_ISSET(cpu, &g_topology.performance)) { + CPU_SET(cpu, &g_topology.efficiency); + } + } + + g_topology.separated = CPU_COUNT(&g_topology.efficiency) > 0; + LOG_INFO(Common, "CPU topology: {} performance cores, {} efficiency cores, separation {}", + CPU_COUNT(&g_topology.performance), CPU_COUNT(&g_topology.efficiency), + g_topology.separated ? "enabled" : "unavailable"); +} + +void EnsureTopologyLocked() { + if (!g_topology.initialized) { + ComputeTopologyLocked(); + } +} + +void RefreshTopologyLocked() { + if (!g_topology.initialized) { + ComputeTopologyLocked(); + return; + } + cpu_set_t current; + CPU_ZERO(¤t); + if (sched_getaffinity(getpid(), sizeof(current), ¤t) != 0) { + return; + } + if (std::memcmp(¤t, &g_topology.allowed, sizeof(current)) != 0) { + ComputeTopologyLocked(); + } +} + +bool ApplyCoreGroupLocked(pid_t tid, CoreGroup group) { + if (!g_topology.separated || group == CoreGroup::Unrestricted) { + return false; + } + const cpu_set_t& mask = + group == CoreGroup::Performance ? g_topology.performance : g_topology.efficiency; + if (CPU_COUNT(&mask) == 0) { + return false; + } + if (sched_setaffinity(tid, sizeof(mask), &mask) != 0) { + LOG_WARNING(Common, "Could not restrict thread {} to its core group: {}", tid, + ::Common::GetLastErrorMsg()); + return false; + } + return true; +} + +ThreadPolicy& AcquirePolicyLocked(pid_t tid) { + auto& policies = Policies(); + for (auto& policy : policies) { + if (policy.tid == tid) { + return policy; + } + } + return policies.emplace_back(ThreadPolicy{tid, CoreGroup::Unrestricted, 0, false}); +} + +void SetCurrentThreadCoreGroup(CoreGroup group) { + const pid_t tid = gettid(); + { + std::scoped_lock lock{g_topology_mutex}; + EnsureTopologyLocked(); + ApplyCoreGroupLocked(tid, group); + } + (void)&t_policy_registration; + std::scoped_lock lock{g_policy_mutex}; + AcquirePolicyLocked(tid).group = group; +} + +void RememberCurrentThreadNice(pid_t tid, int nice_value) { + (void)&t_policy_registration; + std::scoped_lock lock{g_policy_mutex}; + ThreadPolicy& policy = AcquirePolicyLocked(tid); + policy.nice_value = nice_value; + policy.has_nice = true; +} +} // Anonymous namespace +#endif + #include "common/cpu_features.h" #ifdef ARCHITECTURE_x86_64 #ifdef _MSC_VER @@ -48,7 +301,6 @@ #endif #include "common/x64/rdtsc.h" #endif -#include "core/core_timing.h" namespace Common { @@ -78,6 +330,24 @@ void SetCurrentThreadPriority(ThreadPriority new_priority) { } }(); set_thread_priority(find_thread(NULL), priority); +#elif defined(__ANDROID__) + const int nice_value = [&]() { + switch (new_priority) { + case ThreadPriority::Low: return ANDROID_THREAD_PRIORITY_BACKGROUND; + case ThreadPriority::Normal: return ANDROID_THREAD_PRIORITY_DEFAULT; + case ThreadPriority::High: return ANDROID_THREAD_PRIORITY_DISPLAY; + case ThreadPriority::VeryHigh: return ANDROID_THREAD_PRIORITY_URGENT_DISPLAY; + case ThreadPriority::Critical: return ANDROID_THREAD_PRIORITY_AUDIO; + default: return ANDROID_THREAD_PRIORITY_DEFAULT; + } + }(); + const pid_t tid = gettid(); + if (setpriority(PRIO_PROCESS, static_cast(tid), nice_value) != 0) { + LOG_WARNING(Common, "Could not set thread nice value to {}: {}", nice_value, + GetLastErrorMsg()); + return; + } + RememberCurrentThreadNice(tid, nice_value); #else pthread_t this_thread = pthread_self(); const auto scheduling_type = SCHED_OTHER; @@ -132,29 +402,31 @@ void SetCurrentThreadName(const char* name) { #endif } -void PinCurrentThreadToPerformanceCore(size_t core_id) { - ASSERT(core_id < 4); - // If we set a flag for a CPU that doesn't exist, the thread may not be allowed to - // run in ANY processor! - auto const total_cores = std::thread::hardware_concurrency(); - if (core_id < total_cores) { +void SetCurrentThreadToPerformanceCores() { #if defined(__ANDROID__) - cpu_set_t set; - CPU_ZERO(&set); - CPU_SET(core_id, &set); - sched_setaffinity(pthread_self(), sizeof(set), &set); -#elif defined(__linux__) || defined(__FreeBSD__) - cpu_set_t set; - CPU_ZERO(&set); - CPU_SET(core_id, &set); - pthread_setaffinity_np(pthread_self(), sizeof(set), &set); -#elif defined(_WIN32) - DWORD set = 1UL << core_id; - SetThreadAffinityMask(GetCurrentThread(), set); -#else - // No pin functionality implemented + SetCurrentThreadCoreGroup(CoreGroup::Performance); #endif +} + +void SetCurrentThreadToEfficiencyCores() { +#if defined(__ANDROID__) + SetCurrentThreadCoreGroup(CoreGroup::Efficiency); +#endif +} + +void RefreshThreadPolicies() { +#if defined(__ANDROID__) + std::scoped_lock topology_lock{g_topology_mutex}; + RefreshTopologyLocked(); + + std::scoped_lock policy_lock{g_policy_mutex}; + for (const auto& policy : Policies()) { + if (policy.has_nice) { + setpriority(PRIO_PROCESS, static_cast(policy.tid), policy.nice_value); + } + ApplyCoreGroupLocked(policy.tid, policy.group); } +#endif } #ifdef ARCHITECTURE_x86_64 diff --git a/src/common/thread.h b/src/common/thread.h index a75e342802..24709dfa35 100644 --- a/src/common/thread.h +++ b/src/common/thread.h @@ -99,8 +99,16 @@ enum class ThreadPriority : u32 { Critical = 4, }; +enum class ThreadPlacement : u32 { + Default = 0, + Background = 1, + Efficiency = 2, +}; + void SetCurrentThreadPriority(ThreadPriority new_priority); void SetCurrentThreadName(const char* name); -void PinCurrentThreadToPerformanceCore(size_t core_id); +void SetCurrentThreadToPerformanceCores(); +void SetCurrentThreadToEfficiencyCores(); +void RefreshThreadPolicies(); } // namespace Common diff --git a/src/common/thread_worker.h b/src/common/thread_worker.h index 6ec9d6a2bd..8ae010e6bf 100644 --- a/src/common/thread_worker.h +++ b/src/common/thread_worker.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project @@ -37,10 +37,17 @@ class StatefulThreadWorker { using StateMaker = std::conditional_t, DummyCallable>; public: - explicit StatefulThreadWorker(size_t num_workers, std::string name, StateMaker func = {}) + explicit StatefulThreadWorker(size_t num_workers, std::string name, StateMaker func = {}, + ThreadPlacement placement = ThreadPlacement::Default) : workers_queued{num_workers}, thread_name{std::move(name)} { - const auto lambda = [this, func](std::stop_token stop_token) { + const auto lambda = [this, func, placement](std::stop_token stop_token) { Common::SetCurrentThreadName(thread_name.c_str()); + if (placement != ThreadPlacement::Default) { + Common::SetCurrentThreadPriority(ThreadPriority::Low); + } + if (placement == ThreadPlacement::Efficiency) { + Common::SetCurrentThreadToEfficiencyCores(); + } { [[maybe_unused]] std::conditional_t state{func()}; while (!stop_token.stop_requested()) { diff --git a/src/core/arm/nce/arm_nce.cpp b/src/core/arm/nce/arm_nce.cpp index 63d5638fce..b181839db8 100644 --- a/src/core/arm/nce/arm_nce.cpp +++ b/src/core/arm/nce/arm_nce.cpp @@ -157,6 +157,8 @@ bool ArmNce::HandleGuestAlignmentFault(GuestContext* guest_ctx, void* raw_info, return HandleFailedGuestFault(guest_ctx, raw_info, raw_context); } +constexpr size_t NCE_WRITE_FAULT_CLUSTER_PAGES = 4; + bool ArmNce::HandleGuestAccessFault(GuestContext* guest_ctx, void* raw_info, void* raw_context) { auto* info = static_cast(raw_info); @@ -165,7 +167,7 @@ bool ArmNce::HandleGuestAccessFault(GuestContext* guest_ctx, void* raw_info, voi const Common::ProcessAddress addr = (reinterpret_cast(info->si_addr) & ~Memory::YUZU_PAGEMASK); auto& memory = guest_ctx->parent->m_running_thread->GetOwnerProcess()->GetMemory(); - if (memory.InvalidateNCE(addr, Memory::YUZU_PAGESIZE)) { + if (memory.InvalidateNCE(addr, Memory::YUZU_PAGESIZE * NCE_WRITE_FAULT_CLUSTER_PAGES)) { // We handled the access successfully and are returning to guest code. return true; } diff --git a/src/core/core.cpp b/src/core/core.cpp index 8b9133f963..f8383f01ac 100644 --- a/src/core/core.cpp +++ b/src/core/core.cpp @@ -118,6 +118,7 @@ struct System::Impl { is_multicore = Settings::values.use_multi_core.GetValue(); extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb; + unified_memory = Settings::values.use_unified_memory.GetValue(); core_timing.SetMulticore(is_multicore); core_timing.Initialize([&system]() { system.RegisterHostThread(); }); @@ -145,7 +146,8 @@ struct System::Impl { !device_memory.has_value() || is_multicore != Settings::values.use_multi_core.GetValue() || extended_memory_layout != (Settings::values.memory_layout_mode.GetValue() != - Settings::MemoryLayout::Memory_4Gb); + Settings::MemoryLayout::Memory_4Gb) || + unified_memory != Settings::values.use_unified_memory.GetValue(); if (!must_reinitialize) { return; @@ -156,6 +158,7 @@ struct System::Impl { is_multicore = Settings::values.use_multi_core.GetValue(); extended_memory_layout = Settings::values.memory_layout_mode.GetValue() != Settings::MemoryLayout::Memory_4Gb; + unified_memory = Settings::values.use_unified_memory.GetValue(); Initialize(system); } @@ -503,6 +506,7 @@ struct System::Impl { std::atomic_bool is_powered_on{}; bool is_multicore : 1 = false; bool extended_memory_layout : 1 = false; + bool unified_memory : 1 = false; bool exit_locked : 1 = false; bool exit_requested : 1 = false; bool nvdec_active : 1 = false; diff --git a/src/core/core_timing.cpp b/src/core/core_timing.cpp index 3e4e7bdc68..d18c8a72f1 100644 --- a/src/core/core_timing.cpp +++ b/src/core/core_timing.cpp @@ -58,7 +58,8 @@ void CoreTiming::Initialize(std::function&& on_thread_init_) { if (is_multicore) { timer_thread = std::jthread([this](std::stop_token stop_token) { Common::SetCurrentThreadName("HostTiming"); - Common::SetCurrentThreadPriority(Common::ThreadPriority::High); + Common::SetCurrentThreadPriority(Common::ThreadPriority::VeryHigh); + Common::SetCurrentThreadToPerformanceCores(); on_thread_init(); has_started = true; diff --git a/src/core/cpu_manager.cpp b/src/core/cpu_manager.cpp index 7194fa5022..23d5ac3dfd 100644 --- a/src/core/cpu_manager.cpp +++ b/src/core/cpu_manager.cpp @@ -174,12 +174,7 @@ void CpuManager::RunThread(std::stop_token token, std::size_t core) { std::string name = is_multicore ? ("CPUCore_" + std::to_string(core)) : std::string{"CPUThread"}; Common::SetCurrentThreadName(name.c_str()); Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical); -#ifdef __ANDROID__ - // Aimed specifically for Snapdragon 8 Elite devices - // This kills performance on desktop, but boosts perf for UMA devices - // like the S8E. Mediatek and Mali likely won't suffer. - Common::PinCurrentThreadToPerformanceCore(core); -#endif + Common::SetCurrentThreadToPerformanceCores(); auto& data = core_data[core]; data.host_context = Common::Fiber::ThreadToFiber(); diff --git a/src/core/device_memory.cpp b/src/core/device_memory.cpp index 1aea56a991..e36ed42e31 100644 --- a/src/core/device_memory.cpp +++ b/src/core/device_memory.cpp @@ -12,9 +12,18 @@ constexpr size_t VirtualReserveSize = 1ULL << 38; constexpr size_t VirtualReserveSize = 1ULL << 39; #endif +namespace { +size_t ApplicationPoolOffset() { + using Init = Kernel::Board::Nintendo::Nx::KSystemControl::Init; + const size_t dram_size = Init::GetIntendedMemorySize(); + const size_t application_pool_size = Init::GetApplicationPoolSize(); + return dram_size > application_pool_size ? dram_size - application_pool_size : 0; +} +} + DeviceMemory::DeviceMemory() : buffer{Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize(), - VirtualReserveSize} {} + VirtualReserveSize, ApplicationPoolOffset()} {} DeviceMemory::~DeviceMemory() = default; diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index 3d97fdcc5c..2261a75c00 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -20,6 +20,8 @@ #include "common/scratch_buffer.h" #include "common/virtual_buffer.h" +struct AHardwareBuffer; + namespace Core { constexpr size_t DEVICE_PAGEBITS = 12ULL; @@ -95,6 +97,34 @@ public: ApplyOpOnPAddr(address, buffer, operation); } + u8* GetPhysicalBase() noexcept { + return reinterpret_cast(physical_base); + } + + const u8* GetPhysicalBase() const noexcept { + return reinterpret_cast(physical_base); + } + + size_t GetPhysicalSize() const noexcept { + return physical_size; + } + + std::span GetBackingHardwareBuffers() const noexcept { + return ahb_windows; + } + + size_t GetBackingHardwareBufferWindowSize() const noexcept { + return ahb_window_size; + } + + size_t GetBackingHardwareBufferBase() const noexcept { + return ahb_base; + } + + bool IsBackingShared() const noexcept { + return backing_is_shared; + } + PAddr GetPhysicalRawAddressFromDAddr(DAddr address) const { PAddr subbits = PAddr(address & page_mask); auto paddr = tracked_entries[(address >> page_bits)].compressed_physical_ptr; @@ -126,6 +156,10 @@ public: // New batch API to update multiple ranges with a single lock acquisition. void UpdatePagesCachedBatch(std::span> ranges, s32 delta); + void UpdateTexturePagesCount(DAddr addr, size_t size, s32 delta); + + [[nodiscard]] bool IsRegionTextureCached(DAddr addr, size_t size) const noexcept; + private: struct TranslationEntry { DAddr guest_page{}; @@ -171,6 +205,11 @@ private: std::unique_ptr> impl; const uintptr_t physical_base; + const size_t physical_size; + const std::span ahb_windows; + const size_t ahb_window_size; + const size_t ahb_base; + const bool backing_is_shared; DeviceInterface* device_inter; struct TrackedEntry { @@ -234,6 +273,7 @@ private: (1ULL << (device_virtual_bits - page_bits)) / subentries; using CachedPages = std::array; std::unique_ptr cached_pages; + std::unique_ptr texture_cached_pages; Common::RangeMutex counter_guard; std::mutex mapping_guard; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 43a480cfef..2d00a6e570 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,12 +171,18 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , physical_size{device_memory_.buffer.BackingSize()} + , ahb_windows{device_memory_.buffer.BackingHardwareBuffers()} + , ahb_window_size{device_memory_.buffer.BackingHardwareBufferWindowSize()} + , ahb_base{device_memory_.buffer.BackingHardwareBufferBase()} + , backing_is_shared{device_memory_.buffer.IsBackingShared()} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); + texture_cached_pages = std::make_unique(); const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; for (size_t i = 0; i < total_virtual; i++) { @@ -625,6 +631,28 @@ void DeviceMemoryManager::UpdatePagesCachedCount(DAddr addr, size_t size UpdatePagesCachedCountNoLock(addr, size, delta); } +template +void DeviceMemoryManager::UpdateTexturePagesCount(DAddr addr, size_t size, s32 delta) { + Common::ScopedRangeLock lk(counter_guard, addr, size); + const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); + for (size_t page = addr >> Memory::YUZU_PAGEBITS; page != page_end; ++page) { + CounterAtomicType& count = texture_cached_pages->at(page >> subentries_shift).Count(page); + count.fetch_add(static_cast(delta), std::memory_order_release); + } +} + +template +bool DeviceMemoryManager::IsRegionTextureCached(DAddr addr, size_t size) const noexcept { + const size_t page_end = Common::DivCeil(addr + size, Memory::YUZU_PAGESIZE); + for (size_t page = addr >> Memory::YUZU_PAGEBITS; page != page_end; ++page) { + if (texture_cached_pages->at(page >> subentries_shift).Count(page).load( + std::memory_order_acquire) != 0) { + return true; + } + } + return false; +} + template void DeviceMemoryManager::UpdatePagesCachedBatch(std::span> ranges, s32 delta) { if (ranges.empty()) { diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index dad3ab048c..38667e7777 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -375,39 +375,42 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) { mapping_map.insert_or_assign(params.offset, Mapping(params.handle, device_address, params.offset, size, false, big_page, false)); } - map_buffer_offsets.insert(params.offset); - return NvResult::Success; } NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) { + LOG_DEBUG(Service_NVDRV, "called, offset={:#X}", params.offset); + std::scoped_lock lock(mutex); - if (auto const offset_it = map_buffer_offsets.find(params.offset); offset_it != map_buffer_offsets.end()) { - LOG_DEBUG(Service_NVDRV, "called, offset={:#X}", params.offset); - if (!vm.initialised) { - return NvResult::BadValue; - } - auto const it = mapping_map.find(params.offset); - auto const mapping = it->second; - if (!mapping.fixed) { - auto& allocator{mapping.big_page ? *vm.big_page_allocator : *vm.small_page_allocator}; - u32 page_size_bits{mapping.big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS}; - allocator.Free(u32(mapping.offset >> page_size_bits), u32(mapping.size >> page_size_bits)); - } - - // Sparse mappings shouldn't be fully unmapped, just returned to their sparse state - // Only FreeSpace can unmap them fully - if (mapping.sparse_alloc) { - gmmu->MapSparse(params.offset, mapping.size, mapping.big_page); - } else { - gmmu->Unmap(params.offset, mapping.size); - } - - nvmap.UnpinHandle(mapping.handle); - mapping_map.erase(params.offset); - map_buffer_offsets.erase(params.offset); + if (!vm.initialised) { + return NvResult::BadValue; } + + auto const it = mapping_map.find(params.offset); + if (it == mapping_map.end()) { + LOG_WARNING(Service_NVDRV, "Couldn't find region to unmap at {:#X}", params.offset); + return NvResult::Success; + } + + auto const mapping = it->second; + if (!mapping.fixed) { + auto& allocator{mapping.big_page ? *vm.big_page_allocator : *vm.small_page_allocator}; + u32 page_size_bits{mapping.big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS}; + allocator.Free(u32(mapping.offset >> page_size_bits), u32(mapping.size >> page_size_bits)); + } + + // Sparse mappings shouldn't be fully unmapped, just returned to their sparse state + // Only FreeSpace can unmap them fully + if (mapping.sparse_alloc) { + gmmu->MapSparse(params.offset, mapping.size, mapping.big_page); + } else { + gmmu->Unmap(params.offset, mapping.size); + } + + nvmap.UnpinHandle(mapping.handle); + mapping_map.erase(it); + return NvResult::Success; } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index b8ae57e1b4..46bda127d0 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -13,7 +13,6 @@ #include #include #include -#include #include #include "common/address_space.h" @@ -113,8 +112,6 @@ private: }; static_assert(sizeof(IoctlRemapEntry) == 20, "IoctlRemapEntry is incorrect size"); - ankerl::unordered_dense::set map_buffer_offsets{}; - struct IoctlMapBufferEx { MappingFlags flags{}; // bit0: fixed_offset, bit2: cacheable u32_le kind{}; // -1 is default diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp index b621e5d70d..2a99966027 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include #include "common/assert.h" #include "common/logging.h" @@ -264,7 +265,7 @@ NvResult nvhost_ctrl_gpu::ZCullGetInfo(IoctlNvgpuGpuZcullGetInfoArgs& params) { } NvResult nvhost_ctrl_gpu::ZBCSetTable(IoctlZbcSetTable& params) { - if (params.type > supported_types) { + if (params.type == 0 || params.type > supported_types) { LOG_ERROR(Service_NVDRV, "ZBCSetTable: invalid type {:#X}", params.type); return NvResult::BadParameter; } @@ -279,42 +280,61 @@ NvResult nvhost_ctrl_gpu::ZBCSetTable(IoctlZbcSetTable& params) { color_entry.format = params.format; color_entry.ref_cnt = 1u; - auto color_it = std::ranges::find_if(zbc_colors, - [&](const ZbcColorEntry& color_in_question) { - return color_entry.format == color_in_question.format && - color_entry.color_ds == color_in_question.color_ds && - color_entry.color_l2 == color_in_question.color_l2; - }); + const auto color_end = zbc_colors.begin() + zbc_used_color_entries; + auto color_it = std::find_if(zbc_colors.begin(), color_end, + [&](const ZbcColorEntry& color_in_question) { + return color_entry.format == color_in_question.format && + color_entry.color_ds == color_in_question.color_ds && + color_entry.color_l2 == color_in_question.color_l2; + }); - if (color_it != zbc_colors.end()) { + if (color_it != color_end) { ++color_it->ref_cnt; LOG_DEBUG(Service_NVDRV, "ZBCSetTable: reused color entry fmt={:#X}, ref_cnt={:#X}", params.format, color_it->ref_cnt); - } else { - zbc_colors.push_back(color_entry); - LOG_DEBUG(Service_NVDRV, "ZBCSetTable: added color entry fmt={:#X}, index={:#X}", - params.format, zbc_colors.size() - 1); + break; } + + if (zbc_used_color_entries >= zbc_table_size) { + LOG_WARNING(Service_NVDRV, "ZBCSetTable: color table is full, fmt={:#X}", + params.format); + return NvResult::InsufficientMemory; + } + + zbc_colors[zbc_used_color_entries] = color_entry; + LOG_DEBUG(Service_NVDRV, "ZBCSetTable: added color entry fmt={:#X}, index={:#X}", + params.format, zbc_used_color_entries); + ++zbc_used_color_entries; break; } case ZBCTypes::depth: { ZbcDepthEntry depth_entry{params.depth, params.format, 1u}; - auto depth_it = std::ranges::find_if(zbc_depths, - [&](const ZbcDepthEntry& depth_entry_in_question) { - return depth_entry.format == depth_entry_in_question.format && - depth_entry.depth == depth_entry_in_question.depth; - }); + const auto depth_end = zbc_depths.begin() + zbc_used_depth_entries; + auto depth_it = std::find_if(zbc_depths.begin(), depth_end, + [&](const ZbcDepthEntry& depth_entry_in_question) { + return depth_entry.format == depth_entry_in_question.format && + depth_entry.depth == depth_entry_in_question.depth; + }); - if (depth_it != zbc_depths.end()) { + if (depth_it != depth_end) { ++depth_it->ref_cnt; LOG_DEBUG(Service_NVDRV, "ZBCSetTable: reused depth entry fmt={:#X}, ref_cnt={:#X}", depth_entry.format, depth_it->ref_cnt); - } else { - zbc_depths.push_back(depth_entry); - LOG_DEBUG(Service_NVDRV, "ZBCSetTable: added depth entry fmt={:#X}, index={:#X}", - depth_entry.format, zbc_depths.size() - 1); + break; } + + if (zbc_used_depth_entries >= zbc_table_size) { + LOG_WARNING(Service_NVDRV, "ZBCSetTable: depth table is full, fmt={:#X}", + depth_entry.format); + return NvResult::InsufficientMemory; + } + + zbc_depths[zbc_used_depth_entries] = depth_entry; + LOG_DEBUG(Service_NVDRV, "ZBCSetTable: added depth entry fmt={:#X}, index={:#X}", + depth_entry.format, zbc_used_depth_entries); + ++zbc_used_depth_entries; + break; } } @@ -329,35 +349,34 @@ NvResult nvhost_ctrl_gpu::ZBCQueryTable(IoctlZbcQueryTable& params) { std::scoped_lock lk(zbc_mutex); + if (params.type == 0) { + params.index_size = zbc_table_size; + return NvResult::Success; + } + + if (params.index_size >= zbc_table_size) { + LOG_ERROR(Service_NVDRV, "ZBCQueryTable: invalid index {:#X}", params.index_size); + return NvResult::BadParameter; + } + switch (static_cast(params.type)) { case ZBCTypes::color: { - if (params.index_size >= zbc_colors.size()) { - LOG_ERROR(Service_NVDRV, "ZBCQueryTable: invalid color index {:#X}", params.index_size); - return NvResult::BadParameter; - } - const auto& colors = zbc_colors[params.index_size]; std::copy_n(colors.color_ds.begin(), colors.color_ds.size(), std::begin(params.color_ds)); std::copy_n(colors.color_l2.begin(), colors.color_l2.size(), std::begin(params.color_l2)); params.depth = 0; params.ref_cnt = colors.ref_cnt; params.format = colors.format; - params.index_size = static_cast(zbc_colors.size()); break; } case ZBCTypes::depth: { - if (params.index_size >= zbc_depths.size()) { - LOG_ERROR(Service_NVDRV, "ZBCQueryTable: invalid depth index {:#X}", params.index_size); - return NvResult::BadParameter; - } - const auto& depth_entry = zbc_depths[params.index_size]; std::fill(std::begin(params.color_ds), std::end(params.color_ds), 0); std::fill(std::begin(params.color_l2), std::end(params.color_l2), 0); params.depth = depth_entry.depth; params.ref_cnt = depth_entry.ref_cnt; params.format = depth_entry.format; - params.index_size = static_cast(zbc_depths.size()); + break; } } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h index 80b7abd6a1..8663ba43d7 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl_gpu.h @@ -6,7 +6,7 @@ #pragma once -#include +#include #include "common/common_funcs.h" #include "common/common_types.h" @@ -212,9 +212,13 @@ private: Kernel::KEvent* unknown_event; // ZBC Tables + static constexpr u32 zbc_table_size = 15u; + std::mutex zbc_mutex{}; - std::vector zbc_colors{}; - std::vector zbc_depths{}; + std::array zbc_colors{}; + std::array zbc_depths{}; + u32 zbc_used_color_entries{}; + u32 zbc_used_depth_entries{}; const u32 supported_types = 2u; }; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 5869bec903..46a6198839 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -174,7 +174,9 @@ NvResult nvhost_gpu::SetChannelPriority(IoctlChannelSetPriority& params) { case ChannelPriority::Low: channel_timeslice = 1300; break; case ChannelPriority::Medium: channel_timeslice = 2600; break; case ChannelPriority::High: channel_timeslice = 5200; break; - default : return NvResult::BadParameter; + default: + LOG_WARNING(Service_NVDRV, "unknown channel priority {:#X}", channel_priority); + break; } return NvResult::Success; @@ -278,18 +280,20 @@ NvResult nvhost_gpu::AllocateObjectContext(IoctlAllocObjCtx& params) { params.flags = allowed_mask; } - s32_le ctx_class_number_index = + params.obj_id = 0; + + s32_le ctx_class_number_index = GetObjectContextClassNumberIndex(static_cast(params.class_num)); if (ctx_class_number_index < 0) { - LOG_ERROR(Service_NVDRV, "Invalid class number for object context: {:#X}", - params.class_num); - return NvResult::BadParameter; + LOG_WARNING(Service_NVDRV, "Untracked class number for object context: {:#X}", + params.class_num); + return NvResult::Success; } if (ctxObjs[ctx_class_number_index].has_value()) { - LOG_WARNING(Service_NVDRV, "Object context for class {:#X} already allocated on this channel", - params.class_num); - return NvResult::AlreadyAllocated; + LOG_DEBUG(Service_NVDRV, "Object context for class {:#X} already allocated on this channel", + params.class_num); + return NvResult::Success; } // Defer actual hardware context binding until channel is initialized. @@ -435,10 +439,6 @@ NvResult nvhost_gpu::ChannelSetTimeout(IoctlChannelSetTimeout& params) { NvResult nvhost_gpu::ChannelSetTimeslice(IoctlSetTimeslice& params) { LOG_INFO(Service_NVDRV, "called, timeslice={:#X}", params.timeslice); - if (params.timeslice < 1000 || params.timeslice > 5000) { - return NvResult::BadParameter; - } - channel_timeslice = params.timeslice; return NvResult::Success; diff --git a/src/core/hle/service/nvnflinger/buffer_queue_core.cpp b/src/core/hle/service/nvnflinger/buffer_queue_core.cpp index 30c145e545..a8d87f8850 100644 --- a/src/core/hle/service/nvnflinger/buffer_queue_core.cpp +++ b/src/core/hle/service/nvnflinger/buffer_queue_core.cpp @@ -20,33 +20,23 @@ BufferQueueCore::~BufferQueueCore() = default; void BufferQueueCore::PushHistory(u64 frame_number, s64 queue_time, s64 presentation_time, BufferState state) { std::lock_guard lk(buffer_history_mutex); - auto it = buffer_history_map.find(frame_number); - if (it != buffer_history_map.end()) { - it->second.state = state; - return; - } - - buffer_history_map.emplace(frame_number, BufferHistoryInfo{ + buffer_history_pos = (buffer_history_pos + 1) % BUFFER_HISTORY_SIZE; + buffer_history[buffer_history_pos] = BufferHistoryInfo{ frame_number, queue_time, presentation_time, state - }); - buffer_history_order.push_back(frame_number); - - if (buffer_history_order.size() > BUFFER_HISTORY_SIZE) { - u64 oldest_frame = buffer_history_order.front(); - buffer_history_order.pop_front(); - buffer_history_map.erase(oldest_frame); - } + }; } void BufferQueueCore::UpdateHistory(u64 frame_number, BufferState state) { std::lock_guard lk(buffer_history_mutex); - auto it = buffer_history_map.find(frame_number); - if (it != buffer_history_map.end()) { - it->second.state = state; + for (auto& entry : buffer_history) { + if (entry.frame_number == frame_number) { + entry.state = state; + return; + } } } diff --git a/src/core/hle/service/nvnflinger/buffer_queue_core.h b/src/core/hle/service/nvnflinger/buffer_queue_core.h index 86ca19dad3..2978842a10 100644 --- a/src/core/hle/service/nvnflinger/buffer_queue_core.h +++ b/src/core/hle/service/nvnflinger/buffer_queue_core.h @@ -9,14 +9,13 @@ #pragma once +#include #include -#include #include #include #include #include #include -#include #include #include "core/hle/service/nvnflinger/buffer_item.h" @@ -28,12 +27,15 @@ namespace Service::android { +#pragma pack(push, 1) struct BufferHistoryInfo { - u64 frame_number{}; - s64 queue_time{}; - s64 presentation_time{}; - BufferState state{}; + u64 frame_number; + s64 queue_time; + s64 presentation_time; + BufferState state; }; +#pragma pack(pop) +static_assert(sizeof(BufferHistoryInfo) == 0x1C, "BufferHistoryInfo must be 28 bytes"); class IConsumerListener; class IProducerListener; @@ -88,9 +90,9 @@ private: bool buffer_has_been_queued{}; u64 frame_counter{}; - std::unordered_map buffer_history_map{}; + std::array buffer_history{}; + u32 buffer_history_pos{BUFFER_HISTORY_SIZE - 1}; mutable std::mutex buffer_history_mutex{}; - std::deque buffer_history_order; u32 transform_hint{}; bool is_allocating{}; diff --git a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp index c3eb2796e3..3399e3a9c6 100644 --- a/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp +++ b/src/core/hle/service/nvnflinger/buffer_queue_producer.cpp @@ -507,6 +507,8 @@ Status BufferQueueProducer::QueueBuffer(s32 slot, const QueueBufferInput& input, sticky_transform = sticky_transform_; + const bool track_history = Settings::values.enable_buffer_history.GetValue(); + if (core->queue.empty()) { core->queue.push_back(item); listener_available = core->consumer_listener; @@ -514,7 +516,7 @@ Status BufferQueueProducer::QueueBuffer(s32 slot, const QueueBufferInput& input, auto front = core->queue.begin(); if (front->is_droppable && core->StillTracking(*front)) { slots[front->slot].buffer_state = BufferState::Free; - if (Settings::values.enable_buffer_history.GetValue()) { + if (track_history) { core->UpdateHistory(front->frame_number, BufferState::Free); } slots[front->slot].frame_number = 0; @@ -529,7 +531,7 @@ Status BufferQueueProducer::QueueBuffer(s32 slot, const QueueBufferInput& input, } } - if (Settings::values.enable_buffer_history.GetValue()) { + if (track_history) { core->PushHistory(core->frame_counter, slots[slot].queue_time, slots[slot].presentation_time, BufferState::Queued); } @@ -902,26 +904,31 @@ void BufferQueueProducer::Transact(u32 code, std::span parcel_data, const s32 request = parcel_in.Read(); if (request <= 0) { - parcel_out.Write(Status::BadValue); + status = Status::BadValue; parcel_out.Write(0); break; } - std::vector snapshot; + constexpr u32 history_size = BufferQueueCore::BUFFER_HISTORY_SIZE; + std::array snapshot{}; + s32 count{}; { std::scoped_lock lk(core->buffer_history_mutex); - for (auto& [frame, info] : core->buffer_history_map) { - snapshot.push_back(info); + + const u32 newest = core->buffer_history_pos; + for (u32 i = 0; i < history_size; ++i) { + const auto& entry = core->buffer_history[(newest + history_size - i) % history_size]; + if (entry.frame_number == 0) { + break; + } + + snapshot[count] = entry; + ++count; } } - std::sort(snapshot.begin(), snapshot.end(), [](auto& a, auto& b){ - return a.frame_number > b.frame_number; - }); - - const s32 limit = std::min(request, (s32)snapshot.size()); - parcel_out.Write(Status::NoError); + const s32 limit = (std::min)(request, count); parcel_out.Write(limit); for (s32 i = 0; i < limit; ++i) { parcel_out.Write(snapshot[i]); diff --git a/src/core/hle/service/vi/conductor.cpp b/src/core/hle/service/vi/conductor.cpp index 002c653b8c..d731d55bba 100644 --- a/src/core/hle/service/vi/conductor.cpp +++ b/src/core/hle/service/vi/conductor.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include "common/settings.h" +#include "common/thread.h" #include "core/core.h" #include "core/core_timing.h" #include "core/hle/service/vi/conductor.h" @@ -76,6 +77,7 @@ void Conductor::ProcessVsync() { void Conductor::VsyncThread(std::stop_token token) { Common::SetCurrentThreadName("VSyncThread"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::High); while (!token.stop_requested()) { m_signal.Wait(); diff --git a/src/qt_common/config/shared_translation.cpp b/src/qt_common/config/shared_translation.cpp index 893e4a0854..a81e633597 100644 --- a/src/qt_common/config/shared_translation.cpp +++ b/src/qt_common/config/shared_translation.cpp @@ -230,6 +230,11 @@ std::unique_ptr InitializeTranslations(QObject* parent) { tr("Preserves GPU-modified data by reading it back before uploading.\nSome games require this to render certain effects properly.")); INSERT(Settings, use_asynchronous_shaders, tr("Enable asynchronous shader compilation"), tr("May reduce shader stutter.")); + INSERT(Settings, use_unified_memory, tr("Enable unified memory access (UMA)"), + tr("Lets the GPU write buffer readbacks directly into guest memory.")); + INSERT(Settings, pipeline_worker_count, tr("Pipeline Worker Threads"), + tr("Number of threads used to build Vulkan pipelines.\n" + "Higher values speed up compilation at the cost of heat and power.")); INSERT(Settings, fast_gpu_time, tr("Fast GPU Time"), tr("Overclocks the emulated GPU to increase dynamic resolution and render " "distance.\nUse 256 for maximal performance and 512 for maximal graphics fidelity.")); @@ -287,6 +292,12 @@ std::unique_ptr InitializeTranslations(QObject* parent) { INSERT(Settings, vertex_input_dynamic_state, tr("Vertex Input Dynamic State"), tr("Enables vertex input dynamic state feature for better quality and performance.")); + INSERT(Settings, dynamic_rendering, tr("Dynamic Rendering"), + tr("Renders without render pass and framebuffer objects.\n" + "Results vary by driver: some gain performance, others lose it.")); + + INSERT(Settings, workgroup_memory_explicit_layout, QString(), QString()); + INSERT( Settings, sample_shading, tr("Sample Shading"), tr("Allows the fragment shader to execute per sample in a multi-sampled fragment " diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h index 6572db86dc..6f522ab5c2 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h +++ b/src/shader_recompiler/backend/glasm/emit_glasm_instructions.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -665,6 +665,8 @@ void EmitShuffleDown(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU3 const IR::Value& clamp, const IR::Value& segmentation_mask); void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 index, const IR::Value& clamp, const IR::Value& segmentation_mask); +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 lane); +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 direction); void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a, ScalarF32 op_b, ScalarU32 swizzle); void EmitDPdxFine(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a); diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp index fb6a597d4e..7c07ec3247 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_warp.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -97,6 +100,24 @@ void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, Sca Shuffle(ctx, inst, value, index, clamp, segmentation_mask, "XOR"); } +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 lane) { + const Register ret{ctx.reg_alloc.Define(inst)}; + ctx.Add("AND.U RC.x,{}.threadid,~3;" + "AND.U RC.y,{},3;" + "OR.U RC.x,RC.x,RC.y;" + "SHFIDX.U {},{},RC.x,0x1C03;" + "MOV.U {}.x,{}.y;", + ctx.stage_name, lane, ret, value, ret, ret); +} + +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, ScalarU32 value, ScalarU32 direction) { + const Register ret{ctx.reg_alloc.Define(inst)}; + ctx.Add("ADD.U RC.x,{},1;" + "SHFXOR.U {},{},RC.x,0x1C03;" + "MOV.U {}.x,{}.y;", + direction, ret, value, ret, ret); +} + void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, ScalarF32 op_a, ScalarF32 op_b, ScalarU32 swizzle) { const auto ret{ctx.reg_alloc.Define(inst)}; diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h index d508b84cec..96aac502d5 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h +++ b/src/shader_recompiler/backend/glsl/emit_glsl_instructions.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -743,6 +743,10 @@ void EmitShuffleDown(EmitContext& ctx, IR::Inst& inst, std::string_view value, void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, std::string_view value, std::string_view index, std::string_view clamp, std::string_view segmentation_mask); +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view lane); +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view direction); void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, std::string_view op_a, std::string_view op_b, std::string_view swizzle); void EmitDPdxFine(EmitContext& ctx, IR::Inst& inst, std::string_view op_a); diff --git a/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp b/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp index f9be5de1c0..b2f65550fb 100644 --- a/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp +++ b/src/shader_recompiler/backend/glsl/emit_glsl_warp.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -200,6 +203,18 @@ void EmitShuffleButterfly(EmitContext& ctx, IR::Inst& inst, std::string_view val ctx.AddU32("{}=shfl_in_bounds?shfl_result:{};", inst, value); } +void EmitQuadBroadcast(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view lane) { + const auto src_thread_id{fmt::format("(({}&~3)|({}& 3))", THREAD_ID, lane)}; + ctx.AddU32("{}=readInvocationARB({},{});", inst, value, src_thread_id); +} + +void EmitQuadSwap(EmitContext& ctx, IR::Inst& inst, std::string_view value, + std::string_view direction) { + const auto src_thread_id{fmt::format("({}^({}+1))", THREAD_ID, direction)}; + ctx.AddU32("{}=readInvocationARB({},{});", inst, value, src_thread_id); +} + void EmitFSwizzleAdd(EmitContext& ctx, IR::Inst& inst, std::string_view op_a, std::string_view op_b, std::string_view swizzle) { const auto mask{fmt::format("({}>>((gl_SubGroupInvocationARB&3)<<1))&3", swizzle)}; diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp index 53453e6ab7..d9e77db169 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp @@ -322,6 +322,11 @@ void DefineEntryPoint(const IR::Program& program, EmitContext& ctx, Id main) { if (ctx.runtime_info.force_early_z) { ctx.AddExecutionMode(main, spv::ExecutionMode::EarlyFragmentTests); } + if (ctx.profile.support_shader_quad_control && program.info.uses_quad_shuffles) { + ctx.AddExtension("SPV_KHR_quad_control"); + ctx.AddCapability(spv::Capability::QuadControlKHR); + ctx.AddExecutionMode(main, spv::ExecutionMode::RequireFullQuadsKHR); + } break; default: throw NotImplementedException("Stage {}", program.stage); @@ -443,6 +448,12 @@ void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ct ctx.AddCapability(spv::Capability::GroupNonUniformVote); } } + if (info.uses_quad_shuffles) { + if (profile.support_quad_shuffles) { + ctx.AddCapability(spv::Capability::GroupNonUniformQuad); + } + ctx.AddCapability(spv::Capability::GroupNonUniformShuffle); + } if (info.uses_int64_bit_atomics && profile.support_int64_atomics) { ctx.AddCapability(spv::Capability::Int64Atomics); } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h index 171977c85c..165ec3fe84 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -622,6 +622,8 @@ Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clam Id segmentation_mask); Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp, Id segmentation_mask); +Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane); +Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction); Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle); Id EmitDPdxFine(EmitContext& ctx, Id op_a); Id EmitDPdyFine(EmitContext& ctx, Id op_a); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp index 242426ec08..ffd5d9c29f 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp @@ -260,6 +260,21 @@ Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id return SelectValue(ctx, in_range, value, src_thread_id); } +Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) { + if (ctx.profile.support_quad_shuffles) { + return ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, lane); + } + const Id base{ctx.OpBitwiseAnd(ctx.U32[1], GetThreadId(ctx), ctx.Const(~3u))}; + const Id local_lane{ctx.OpBitwiseAnd(ctx.U32[1], lane, ctx.Const(3u))}; + const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], base, local_lane)}; + return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id); +} + +Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction) { + const Id xor_mask{ctx.OpIAdd(ctx.U32[1], direction, ctx.Const(1u))}; + return ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, xor_mask); +} + Id EmitFSwizzleAdd(EmitContext& ctx, Id op_a, Id op_b, Id swizzle) { const Id three{ctx.Const(3U)}; Id mask{GetThreadId(ctx)}; diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp index 171161ffd1..95815d7d51 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp +++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -2100,6 +2100,14 @@ U32 IREmitter::ShuffleButterfly(const IR::U32& value, const IR::U32& index, cons return Inst(Opcode::ShuffleButterfly, value, index, clamp, seg_mask); } +U32 IREmitter::QuadBroadcast(const IR::U32& value, const IR::U32& lane) { + return Inst(Opcode::QuadBroadcast, value, lane); +} + +U32 IREmitter::QuadSwap(const IR::U32& value, const IR::U32& direction) { + return Inst(Opcode::QuadSwap, value, direction); +} + F32 IREmitter::FSwizzleAdd(const F32& a, const F32& b, const U32& swizzle, FpControl control) { return Inst(Opcode::FSwizzleAdd, Flags{control}, a, b, swizzle); } diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h index 725eac3951..f22f0f75e2 100644 --- a/src/shader_recompiler/frontend/ir/ir_emitter.h +++ b/src/shader_recompiler/frontend/ir/ir_emitter.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -394,6 +394,8 @@ public: const IR::U32& seg_mask); [[nodiscard]] U32 ShuffleButterfly(const IR::U32& value, const IR::U32& index, const IR::U32& clamp, const IR::U32& seg_mask); + [[nodiscard]] U32 QuadBroadcast(const IR::U32& value, const IR::U32& lane); + [[nodiscard]] U32 QuadSwap(const IR::U32& value, const IR::U32& direction); [[nodiscard]] F32 FSwizzleAdd(const F32& a, const F32& b, const U32& swizzle, FpControl control = {}); diff --git a/src/shader_recompiler/frontend/ir/opcodes.cpp b/src/shader_recompiler/frontend/ir/opcodes.cpp index 7fb6694a9b..6918d4c15d 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.cpp +++ b/src/shader_recompiler/frontend/ir/opcodes.cpp @@ -10,7 +10,7 @@ namespace Shader::IR { namespace Detail { -OpcodeMeta META_TABLE[532] = { +OpcodeMeta META_TABLE[] = { #define OPCODE(name_token, type_token, ...) \ { \ .name{#name_token}, \ @@ -21,7 +21,7 @@ OpcodeMeta META_TABLE[532] = { #undef OPCODE }; -u8 NUM_ARGS[532] = { +u8 NUM_ARGS[] = { #define OPCODE(name_token, type_token, ...) u8(CalculateNumArgsOf(Opcode::name_token)), #include "opcodes.inc" #undef OPCODE diff --git a/src/shader_recompiler/frontend/ir/opcodes.h b/src/shader_recompiler/frontend/ir/opcodes.h index 5549edc9d7..6f5c09dfcb 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.h +++ b/src/shader_recompiler/frontend/ir/opcodes.h @@ -57,12 +57,12 @@ static constexpr Type F64x2{Type::F64x2}; static constexpr Type F64x3{Type::F64x3}; static constexpr Type F64x4{Type::F64x4}; -extern OpcodeMeta META_TABLE[532]; +extern OpcodeMeta META_TABLE[]; constexpr size_t CalculateNumArgsOf(Opcode op) noexcept { const auto& arg_types = META_TABLE[size_t(op)].arg_types; return size_t(std::distance(arg_types.begin(), std::ranges::find(arg_types, Type::Void))); } -extern u8 NUM_ARGS[532]; +extern u8 NUM_ARGS[]; } // namespace Detail /// Get return type of an opcode diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc index 960009d89e..fee0afaf33 100644 --- a/src/shader_recompiler/frontend/ir/opcodes.inc +++ b/src/shader_recompiler/frontend/ir/opcodes.inc @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -579,6 +582,8 @@ OPCODE(ShuffleIndex, U32, U32, OPCODE(ShuffleUp, U32, U32, U32, U32, U32, ) OPCODE(ShuffleDown, U32, U32, U32, U32, U32, ) OPCODE(ShuffleButterfly, U32, U32, U32, U32, U32, ) +OPCODE(QuadBroadcast, U32, U32, U32, ) +OPCODE(QuadSwap, U32, U32, U32, ) OPCODE(FSwizzleAdd, F32, F32, F32, U32, ) OPCODE(DPdxFine, F32, F32, ) OPCODE(DPdyFine, F32, F32, ) diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp index e6345de65e..e4c39d7974 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/internal_stage_buffer_entry_read.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -17,39 +17,12 @@ enum class Mode : u64 { Attr, }; -enum class SZ : u64 { - U8, - U16, - U32, - F32 -}; - enum class Shift : u64 { Default, U16, B32, }; -IR::U32 scaleIndex(IR::IREmitter& ir, IR::U32 index, Shift shift) { - switch (shift) { - case Shift::Default: return index; - case Shift::U16: return ir.ShiftLeftLogical(index, ir.Imm32(1)); - case Shift::B32: return ir.ShiftLeftLogical(index, ir.Imm32(2)); - default: UNREACHABLE(); - } -} - -IR::U32 skewBytes(IR::IREmitter& ir, SZ sizeRead) { - const IR::U32 lane = ir.LaneId(); - switch (sizeRead) { - case SZ::U8: return lane; - case SZ::U16: return ir.ShiftLeftLogical(lane, ir.Imm32(1)); - case SZ::U32: - case SZ::F32: return ir.ShiftLeftLogical(lane, ir.Imm32(2)); - default: UNREACHABLE(); - } -} - } // Anonymous namespace void TranslatorVisitor::ISBERD(u64 insn) { @@ -64,65 +37,28 @@ void TranslatorVisitor::ISBERD(u64 insn) { BitField<31, 1, u64> skew; BitField<32, 1, u64> o; BitField<33, 2, Mode> mode; - BitField<36, 4, SZ> sz; BitField<47, 2, Shift> shift; } const isberd{insn}; - IR::U32 index{}; - if (isberd.src_reg_num.Value() == 0xFF) { - index = ir.Imm32(isberd.imm.Value()); - } else { - const IR::U32 scaledIndex = scaleIndex(ir, X(isberd.src_reg.Value()), isberd.shift.Value()); - index = ir.IAdd(scaledIndex, ir.Imm32(isberd.imm.Value())); + if (isberd.skew != 0) { + throw NotImplementedException("ISBERD SKEW"); + } + if (isberd.o != 0) { + throw NotImplementedException("ISBERD O"); } - if (isberd.o.Value()) { - if (isberd.skew.Value()) { - index = ir.IAdd(index, skewBytes(ir, isberd.sz.Value())); - } - - const IR::U64 index64 = ir.UConvert(64, index); - IR::U32 globalLoaded{}; - switch (isberd.sz.Value()) { - case SZ::U8: globalLoaded = ir.LoadGlobalU8 (index64); break; - case SZ::U16: globalLoaded = ir.LoadGlobalU16(index64); break; - case SZ::U32: - case SZ::F32: globalLoaded = ir.LoadGlobal32(index64); break; - default: UNREACHABLE(); - } - X(isberd.dest_reg.Value(), globalLoaded); - + switch (isberd.mode.Value()) { + case Mode::Default: + X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); return; - } - - if (isberd.mode.Value() != Mode::Default) { - if (isberd.skew.Value()) { - index = ir.IAdd(index, skewBytes(ir, SZ::U32)); - } - - IR::F32 float_index{}; - switch (isberd.mode.Value()) { - case Mode::Patch: float_index = ir.GetPatch(index.Patch()); - break; - case Mode::Prim: float_index = ir.GetAttribute(index.Attribute()); - break; - case Mode::Attr: float_index = ir.GetAttributeIndexed(index); - break; - default: UNREACHABLE(); - } - X(isberd.dest_reg.Value(), ir.BitCast(float_index)); - + case Mode::Attr: + LOG_DEBUG(Shader, "(STUBBED) ISBERD Mode Attr"); + X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); return; + default: + throw NotImplementedException("ISBERD Mode {}", + static_cast(isberd.mode.Value())); } - - if (isberd.skew.Value()) { - X(isberd.dest_reg.Value(), ir.IAdd(X(isberd.src_reg.Value()), ir.LaneId())); - - return; - } - - // Fallback copy - X(isberd.dest_reg.Value(), X(isberd.src_reg.Value())); } } // namespace Shader::Maxwell diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp index 972eec8276..2a19237ab2 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/warp_shuffle.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -36,7 +36,10 @@ enum class ShuffleMode : u64 { } } -void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32& mask) { +constexpr u32 QUAD_MASK = (28u << 8) | 3u; + +void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32& mask, + bool index_is_imm, u32 index_imm, bool mask_is_imm, u32 mask_imm) { union { u64 insn; BitField<0, 8, IR::Reg> dest_reg; @@ -45,6 +48,21 @@ void Shuffle(TranslatorVisitor& v, u64 insn, const IR::U32& index, const IR::U32 BitField<48, 3, IR::Pred> pred; } const shfl{insn}; + const bool is_quad_candidate{mask_is_imm && mask_imm == QUAD_MASK && index_is_imm && + v.env.ShaderStage() == Stage::Fragment}; + if (is_quad_candidate) { + if (shfl.mode == ShuffleMode::IDX && index_imm <= 3) { + v.X(shfl.dest_reg, v.ir.QuadBroadcast(v.X(shfl.src_reg), v.ir.Imm32(index_imm))); + v.ir.SetPred(shfl.pred, v.ir.Imm1(true)); + return; + } + if (shfl.mode == ShuffleMode::BFLY && index_imm >= 1 && index_imm <= 3) { + v.X(shfl.dest_reg, v.ir.QuadSwap(v.X(shfl.src_reg), v.ir.Imm32(index_imm - 1))); + v.ir.SetPred(shfl.pred, v.ir.Imm1(true)); + return; + } + } + const IR::U32 result{ShuffleOperation(v.ir, v.X(shfl.src_reg), index, mask, shfl.mode)}; v.ir.SetPred(shfl.pred, v.ir.GetInBoundsFromOp(result)); v.X(shfl.dest_reg, result); @@ -59,11 +77,14 @@ void TranslatorVisitor::SHFL(u64 insn) { BitField<29, 1, u64> src_b_flag; BitField<34, 13, u64> src_b_imm; } const flags{insn}; - const IR::U32 src_a{flags.src_a_flag != 0 ? ir.Imm32(static_cast(flags.src_a_imm)) - : GetReg20(insn)}; - const IR::U32 src_b{flags.src_b_flag != 0 ? ir.Imm32(static_cast(flags.src_b_imm)) - : GetReg39(insn)}; - Shuffle(*this, insn, src_a, src_b); + const bool index_is_imm{flags.src_a_flag != 0}; + const bool mask_is_imm{flags.src_b_flag != 0}; + const IR::U32 src_a{index_is_imm ? ir.Imm32(static_cast(flags.src_a_imm)) + : GetReg20(insn)}; + const IR::U32 src_b{mask_is_imm ? ir.Imm32(static_cast(flags.src_b_imm)) + : GetReg39(insn)}; + Shuffle(*this, insn, src_a, src_b, index_is_imm, static_cast(flags.src_a_imm), + mask_is_imm, static_cast(flags.src_b_imm)); } } // namespace Shader::Maxwell diff --git a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp index 5089f3a101..9010c2bf68 100644 --- a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp +++ b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp @@ -498,6 +498,10 @@ void VisitUsages(Info& info, IR::Inst& inst) { case IR::Opcode::ShuffleButterfly: info.uses_subgroup_shuffles = true; break; + case IR::Opcode::QuadBroadcast: + case IR::Opcode::QuadSwap: + info.uses_quad_shuffles = true; + break; case IR::Opcode::GetCbufU8: case IR::Opcode::GetCbufS8: case IR::Opcode::GetCbufU16: diff --git a/src/shader_recompiler/profile.h b/src/shader_recompiler/profile.h index eb7b2a02b2..bd8db0d6f4 100644 --- a/src/shader_recompiler/profile.h +++ b/src/shader_recompiler/profile.h @@ -37,6 +37,8 @@ struct Profile { bool support_explicit_workgroup_layout{}; bool support_workgroup_layout_8bit_access{}; bool support_workgroup_layout_16bit_access{}; + bool support_shader_quad_control{}; + bool support_quad_shuffles{}; bool support_vote{}; u32 supported_subgroup_stages{0x7F}; bool support_viewport_index_layer_non_geometry{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 0c11a0d72a..71309e040e 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -252,6 +252,7 @@ struct Info { bool uses_is_helper_invocation{}; bool uses_subgroup_invocation_id{}; bool uses_subgroup_shuffles{}; + bool uses_quad_shuffles{}; std::array uses_patches{}; std::array interpolation{}; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 5c02b5ed90..999955baf9 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -33,7 +33,7 @@ add_library(video_core STATIC control/channel_state_cache.h control/scheduler.cpp control/scheduler.h - delayed_destruction_ring.h + deferred_destruction_queue.h dirty_flags.cpp dirty_flags.h dma_pusher.cpp @@ -158,6 +158,8 @@ add_library(video_core STATIC renderer_vulkan/vk_compute_pass.h renderer_vulkan/vk_compute_pipeline.cpp renderer_vulkan/vk_compute_pipeline.h + renderer_vulkan/vk_descriptor_buffer.cpp + renderer_vulkan/vk_descriptor_buffer.h renderer_vulkan/vk_descriptor_pool.cpp renderer_vulkan/vk_descriptor_pool.h renderer_vulkan/vk_fence_manager.cpp diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index e40aad1fb5..d9fb98f58b 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include @@ -31,44 +32,89 @@ BufferCache

::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R immediately_free = (Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive); #endif if (!runtime.CanReportMemoryUsage()) { - minimum_memory = DEFAULT_EXPECTED_MEMORY; - critical_memory = DEFAULT_CRITICAL_MEMORY; + memory_budget = FALLBACK_MEMORY_BUDGET; return; } - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - minimum_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); + memory_budget = runtime.GetDeviceLocalMemory(); } template BufferCache

::~BufferCache() = default; template -void BufferCache

::RunGarbageCollector() { - const bool aggressive_gc = total_used_memory >= critical_memory; - const u64 ticks_to_destroy = aggressive_gc ? 60 : 120; - int num_iterations = aggressive_gc ? 64 : 32; - const auto clean_up = [this, &num_iterations](BufferId buffer_id) { - if (num_iterations == 0) { +u64 BufferCache

::DeviceUsage(bool force_refresh) { + if (!runtime.CanReportMemoryUsage()) { + return total_used_memory; + } + if (force_refresh || usage_refresh_countdown == 0) { + cached_device_usage = runtime.GetDeviceAllocationUsage(); + usage_refresh_countdown = USAGE_REFRESH_INTERVAL; + } else { + --usage_refresh_countdown; + } + return cached_device_usage; +} + +template +u64 BufferCache

::ReclaimMemory(u64 target_bytes, bool allow_download) { + if (target_bytes == 0 || in_reclaim) { + return 0; + } + in_reclaim = true; + u64 freed = 0; + const auto clean_up = [&](BufferId buffer_id) { + if (freed >= target_bytes) { return true; } - --num_iterations; auto& buffer = slot_buffers[buffer_id]; + if (!allow_download && IsRegionGpuModified(buffer.CpuAddr(), buffer.SizeBytes())) { + return false; + } + const u64 buffer_bytes = Common::AlignUp(buffer.SizeBytes(), 1024); DownloadBufferMemory(buffer); DeleteBuffer(buffer_id); + freed += buffer_bytes; return false; }; - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, clean_up); + const u64 cold_tick = + frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0; + lru_cache.ForEachItemBelow(cold_tick, clean_up); + if (freed == 0) { + lru_cache.ForEachItemBelow(frame_tick > 0 ? frame_tick - 1 : 0, clean_up); + } + in_reclaim = false; + usage_refresh_countdown = 0; + reclaim_stalled = freed == 0; + if (freed > 0) { + reclaim_wait_sync_point = runtime.CurrentSyncPoint(); + } + return freed; +} + +template +void BufferCache

::ReclaimDeferredResources(u64 completed_sync_point) { + sentenced_buffers.Reclaim(completed_sync_point); +} + +template +void BufferCache

::EnsureHeadroom(bool allow_download) { + if (reclaim_stalled) { + return; + } + if (runtime.CompletedSyncPoint() < reclaim_wait_sync_point) { + return; + } + const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0; + const u64 usage = DeviceUsage(false); + if (usage <= limit) { + return; + } + const u64 target = (limit / 100) * RECLAIM_TARGET_PERCENT; + const u64 excess = usage - target; + const u64 usage_mib = (std::max)(usage >> 20, u64{1}); + const u64 share = (((excess >> 20) * (total_used_memory >> 20)) / usage_mib) << 20; + ReclaimMemory((std::min)(share, total_used_memory), allow_download); } template @@ -96,15 +142,11 @@ void BufferCache

::TickFrame() { const bool skip_preferred = hits * 256 < shots * 251; channel_state->uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0; - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); - } - if (total_used_memory >= minimum_memory) { - RunGarbageCollector(); - } + usage_refresh_countdown = 0; + reclaim_stalled = false; + ReclaimDeferredResources(runtime.CompletedSyncPoint()); + EnsureHeadroom(true); ++frame_tick; - delayed_destruction_ring.Tick(); for (auto& buffer : async_buffers_death_ring) { runtime.FreeDeferredStagingBuffer(buffer); @@ -175,9 +217,71 @@ std::optional BufferCache

::GetFlushArea(DA template void BufferCache

::DownloadMemory(DAddr device_addr, u64 size) { - ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) { - DownloadBufferMemory(buffer, device_addr, size); + if constexpr (!USE_MEMORY_MAPS) { + std::scoped_lock lock{mutex}; + ForEachBufferInRange(device_addr, size, [&](BufferId, Buffer& buffer) { + DownloadBufferMemory(buffer, device_addr, size); + }); + return; + } + + boost::container::small_vector, 8> downloads; + u64 total_size_bytes = 0; + u64 largest_copy = 0; + + std::unique_lock lock{mutex}; + ForEachBufferInRange(device_addr, size, [&](BufferId buffer_id, Buffer& buffer) { + memory_tracker.ForEachDownloadRangeAndClear( + device_addr, size, [&](u64 device_addr_out, u64 range_size) { + const DAddr buffer_addr = buffer.CpuAddr(); + const auto add_download = [&](DAddr start, DAddr end) { + const u64 new_offset = start - buffer_addr; + const u64 new_size = end - start; + downloads.push_back({ + BufferCopy{ + .src_offset = new_offset, + .dst_offset = total_size_bytes, + .size = new_size, + }, + buffer_id, + }); + constexpr u64 align = 64ULL; + constexpr u64 mask = ~(align - 1ULL); + total_size_bytes += (new_size + align - 1) & mask; + largest_copy = (std::max)(largest_copy, new_size); + }; + gpu_modified_ranges.ForEachInRange(device_addr_out, range_size, add_download); + ClearDownload(device_addr_out, range_size); + gpu_modified_ranges.Subtract(device_addr_out, range_size); + }); }); + if (total_size_bytes == 0) { + return; + } + + auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); + boost::container::small_vector writebacks; + runtime.PreCopyBarrier(); + for (auto& [copy, buffer_id] : downloads) { + copy.dst_offset += download_staging.offset; + Buffer& buffer = slot_buffers[buffer_id]; + buffer.MarkUsage(copy.src_offset, copy.size); + const std::array copies{copy}; + runtime.CopyBuffer(download_staging.buffer, buffer, copies, false); + BufferCopy writeback{copy}; + writeback.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; + writebacks.push_back(writeback); + } + runtime.PostCopyBarrier(); + lock.unlock(); + + runtime.Finish(); + const u8* const base = download_staging.mapped_span.data(); + for (const BufferCopy& writeback : writebacks) { + const u64 staging_offset = writeback.dst_offset - download_staging.offset; + device_memory.WriteBlockUnsafe(static_cast(writeback.src_offset), + base + staging_offset, writeback.size); + } } template @@ -214,7 +318,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am auto& src_buffer = slot_buffers[buffer_a]; auto& dest_buffer = slot_buffers[buffer_b]; SynchronizeBuffer(src_buffer, *cpu_src_address, static_cast(amount)); - SynchronizeBuffer(dest_buffer, *cpu_dest_address, static_cast(amount)); + memory_tracker.UnmarkRegionAsCpuModified(*cpu_dest_address, static_cast(amount)); std::array copies{BufferCopy{ .src_offset = src_buffer.Offset(*cpu_src_address), .dst_offset = dest_buffer.Offset(*cpu_dest_address), @@ -571,7 +675,11 @@ void BufferCache

::AccumulateFlushes() { template bool BufferCache

::ShouldWaitAsyncFlushes() const noexcept { - return (!async_buffers.empty() && async_buffers.front().has_value()); + if (async_buffers.empty()) { + return false; + } + return async_buffers.front().has_value() || + !pending_downloads.front().unified_copies.empty(); } template @@ -579,6 +687,7 @@ void BufferCache

::CommitAsyncFlushesHigh() { AccumulateFlushes(); if (committed_gpu_modified_ranges.empty()) { + pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; } @@ -638,27 +747,83 @@ void BufferCache

::CommitAsyncFlushesHigh() { } committed_gpu_modified_ranges.clear(); if (downloads.empty()) { + pending_downloads.emplace_back(); async_buffers.emplace_back(std::optional{}); return; } - auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes, true); - boost::container::small_vector normalized_copies; - runtime.PreCopyBarrier(); + + struct QueuedUnifiedCopy { + u64 window; + BufferId buffer_id; + boost::container::small_vector copies; + }; + + AsyncDownloadBatch batch; + boost::container::small_vector, 16> staging_downloads; + boost::container::small_vector unified_copy_queue; + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + u64 staging_size_bytes = 0; for (auto& [copy, buffer_id] : downloads) { - copy.dst_offset += download_staging.offset; - const std::array copies{copy}; - BufferCopy second_copy{copy}; Buffer& buffer = slot_buffers[buffer_id]; - second_copy.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; - const DAddr orig_device_addr = static_cast(second_copy.src_offset); + const DAddr orig_device_addr = buffer.CpuAddr() + copy.src_offset; + bool unified = false; + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory()) { + window_ids.clear(); + groups.clear(); + unified = ResolveUnifiedWindows(orig_device_addr, copy.src_offset, copy.size, + window_ids, groups); + } + } + BufferCopy record{copy}; + record.src_offset = static_cast(orig_device_addr); + if (unified) { + async_downloads.Add(orig_device_addr, copy.size); + buffer.MarkUsage(copy.src_offset, copy.size); + for (size_t i = 0; i < window_ids.size(); ++i) { + unified_copy_queue.push_back( + QueuedUnifiedCopy{window_ids[i], buffer_id, std::move(groups[i])}); + } + batch.unified_copies.push_back(record); + continue; + } + copy.dst_offset = staging_size_bytes; + constexpr u64 align = 64ULL; + staging_size_bytes += (copy.size + align - 1) & ~(align - 1ULL); + staging_downloads.push_back({copy, buffer_id}); + } + + std::optional download_staging; + if (!staging_downloads.empty()) { + download_staging = runtime.DownloadStagingBuffer(staging_size_bytes, true); + } + runtime.PreCopyBarrier(); + for (auto& [copy, buffer_id] : staging_downloads) { + copy.dst_offset += download_staging->offset; + const std::array copies{copy}; + Buffer& buffer = slot_buffers[buffer_id]; + BufferCopy record{copy}; + record.src_offset = static_cast(buffer.CpuAddr()) + copy.src_offset; + const DAddr orig_device_addr = static_cast(record.src_offset); async_downloads.Add(orig_device_addr, copy.size); buffer.MarkUsage(copy.src_offset, copy.size); - runtime.CopyBuffer(download_staging.buffer, buffer, copies, false); - normalized_copies.push_back(second_copy); + runtime.CopyBuffer(download_staging->buffer, buffer, copies, false); + batch.staging_copies.push_back(record); + } + if constexpr (USE_UNIFIED_MEMORY) { + for (const auto& queued : unified_copy_queue) { + const std::span group_span(queued.copies.data(), + queued.copies.size()); + runtime.CopyToUnifiedMemory(queued.window, slot_buffers[queued.buffer_id], group_span); + } + if (!unified_copy_queue.empty()) { + runtime.UnifiedMemoryHostBarrier(); + } } runtime.PostCopyBarrier(); - pending_downloads.emplace_back(std::move(normalized_copies)); - async_buffers.emplace_back(download_staging); + pending_downloads.emplace_back(std::move(batch)); + async_buffers.emplace_back(std::move(download_staging)); } template @@ -673,32 +838,49 @@ void BufferCache

::PopAsyncFlushes() { template void BufferCache

::PopAsyncBuffers() { - if (async_buffers.empty()) { - return; - } - if (!async_buffers.front().has_value()) { + struct Writeback { + DAddr addr; + const u8* src; + u64 size; + }; + boost::container::small_vector writebacks; + { + std::scoped_lock lock{mutex}; + if (async_buffers.empty()) { + return; + } + auto& batch = pending_downloads.front(); + auto& async_buffer = async_buffers.front(); + if (async_buffer.has_value()) { + const u8* base = async_buffer->mapped_span.data(); + const size_t base_offset = async_buffer->offset; + for (const auto& copy : batch.staging_copies) { + const DAddr device_addr = static_cast(copy.src_offset); + const u64 dst_offset = copy.dst_offset - base_offset; + const u8* read_mapped_memory = base + dst_offset; + async_downloads.ForEachInRange( + device_addr, copy.size, [&](DAddr start, DAddr end, s32) { + writebacks.push_back( + {start, &read_mapped_memory[start - device_addr], end - start}); + }); + async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + }); + } + async_buffers_death_ring.emplace_back(*async_buffer); + } + for (const auto& copy : batch.unified_copies) { + const DAddr device_addr = static_cast(copy.src_offset); + async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { + gpu_modified_ranges.Subtract(start, end - start); + }); + } async_buffers.pop_front(); - return; + pending_downloads.pop_front(); } - auto& downloads = pending_downloads.front(); - auto& async_buffer = async_buffers.front(); - u8* base = async_buffer->mapped_span.data(); - const size_t base_offset = async_buffer->offset; - for (const auto& copy : downloads) { - const DAddr device_addr = static_cast(copy.src_offset); - const u64 dst_offset = copy.dst_offset - base_offset; - const u8* read_mapped_memory = base + dst_offset; - async_downloads.ForEachInRange(device_addr, copy.size, [&](DAddr start, DAddr end, s32) { - device_memory.WriteBlockUnsafe(start, &read_mapped_memory[start - device_addr], - end - start); - }); - async_downloads.Subtract(device_addr, copy.size, [&](DAddr start, DAddr end) { - gpu_modified_ranges.Subtract(start, end - start); - }); + for (const auto& wb : writebacks) { + device_memory.WriteBlockUnsafe(wb.addr, wb.src, wb.size); } - async_buffers_death_ring.emplace_back(*async_buffer); - async_buffers.pop_front(); - pending_downloads.pop_front(); } template @@ -809,46 +991,46 @@ void BufferCache

::BindHostVertexBuffers() { if (use_optimized_vertex_buffers) { auto& flags = maxwell3d->dirty.flags; - u32 enabled_mask = enabled_vertex_buffers_mask; - HostBindings bindings{}; - u32 last_index = (std::numeric_limits::max)(); - const auto flush_bindings = [&]() { - if (bindings.buffers.empty()) { - return; - } - bindings.max_index = bindings.min_index + static_cast(bindings.buffers.size()); - runtime.BindVertexBuffers(bindings); - bindings = HostBindings{}; - last_index = (std::numeric_limits::max)(); - }; - while (enabled_mask != 0) { - const u32 index = std::countr_zero(enabled_mask); - enabled_mask &= (enabled_mask - 1); + const u32 enabled_mask = enabled_vertex_buffers_mask; + bool any_dirty = false; + u32 pending_mask = enabled_mask; + while (pending_mask != 0) { + const u32 index = std::countr_zero(pending_mask); + pending_mask &= (pending_mask - 1); const Binding& binding = VertexBufferSlot(index); Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); SynchronizeBuffer(buffer, binding.device_addr, binding.size); - if (!flags[Dirty::VertexBuffer0 + index]) { - flush_bindings(); - continue; - } + any_dirty |= flags[Dirty::VertexBuffer0 + index]; + } + if (enabled_mask == 0 || !any_dirty) { + return; + } + const u32 min_index = static_cast(std::countr_zero(enabled_mask)); + const u32 max_index = 32u - static_cast(std::countl_zero(enabled_mask)); + HostBindings bindings{}; + bindings.min_index = min_index; + bindings.max_index = max_index; + for (u32 index = min_index; index < max_index; ++index) { flags[Dirty::VertexBuffer0 + index] = false; const u32 stride = maxwell3d->regs.vertex_streams[index].stride; + if ((enabled_mask & (1u << index)) == 0) { + bindings.buffers.push_back(&slot_buffers[NULL_BUFFER_ID]); + bindings.offsets.push_back(0); + bindings.sizes.push_back(0); + bindings.strides.push_back(stride); + continue; + } + const Binding& binding = VertexBufferSlot(index); + Buffer& buffer = slot_buffers[binding.buffer_id]; const u32 offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, binding.size); - if (!bindings.buffers.empty() && index != last_index + 1) { - flush_bindings(); - } - if (bindings.buffers.empty()) { - bindings.min_index = index; - } bindings.buffers.push_back(&buffer); bindings.offsets.push_back(offset); bindings.sizes.push_back(binding.size); bindings.strides.push_back(stride); - last_index = index; } - flush_bindings(); + runtime.BindVertexBuffers(bindings); } else { HostBindings host_bindings; bool any_valid{false}; @@ -921,7 +1103,6 @@ void BufferCache

::BindHostGraphicsUniformBuffers(size_t stage) { template void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 binding_index, bool needs_bind) { - ++channel_state->uniform_cache_shots[0]; const Binding& binding = channel_state->uniform_buffers[stage][index]; const DAddr device_addr = binding.device_addr; const u32 size = (std::min)(binding.size, (*channel_state->uniform_buffer_sizes)[stage][index]); @@ -940,8 +1121,12 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 return alignment > 1 && (offset % alignment) != 0; } }(); + + const bool cached_buffer_is_current = + has_host_buffer && !memory_tracker.IsRegionCpuModified(device_addr, size); const bool use_fast_buffer = needs_alignment_stream - || (has_host_buffer && size <= channel_state->uniform_buffer_skip_cache_size + || (has_host_buffer && !cached_buffer_is_current + && size <= channel_state->uniform_buffer_skip_cache_size && !memory_tracker.IsRegionGpuModified(device_addr, size)); if (use_fast_buffer) { if constexpr (IS_OPENGL) { @@ -968,7 +1153,7 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 device_memory.ReadBlockUnsafe(device_addr, span.data(), size); return; } - // Classic cached path + ++channel_state->uniform_cache_shots[0]; if (SynchronizeBuffer(buffer, device_addr, size)) { ++channel_state->uniform_cache_hits[0]; } @@ -1576,6 +1761,7 @@ void BufferCache

::JoinOverlap(BufferId new_buffer_id, BufferId overlap_id, template BufferId BufferCache

::CreateBuffer(DAddr device_addr, u32 wanted_size) { + EnsureHeadroom(false); DAddr device_addr_end = Common::AlignUp(device_addr + wanted_size, CACHING_PAGESIZE); device_addr = Common::AlignDown(device_addr, CACHING_PAGESIZE); wanted_size = static_cast(device_addr_end - device_addr); @@ -1613,7 +1799,7 @@ void BufferCache

::ChangeRegister(BufferId buffer_id) { total_used_memory += Common::AlignUp(size, 1024); buffer.setLRUID(lru_cache.Insert(buffer_id, frame_tick)); } else { - total_used_memory -= Common::AlignUp(size, 1024); + total_used_memory -= std::min(total_used_memory, Common::AlignUp(size, 1024)); lru_cache.Free(buffer.getLRUID()); } const DAddr device_addr_begin = buffer.CpuAddr(); @@ -1699,6 +1885,98 @@ void BufferCache

::ImmediateUploadMemory([[maybe_unused]] Buffer& buffer, } } +template +bool BufferCache

::ResolveUnifiedWindows( + [[maybe_unused]] DAddr device_addr, [[maybe_unused]] u64 buffer_offset, + [[maybe_unused]] u64 size, [[maybe_unused]] boost::container::small_vector& window_ids, + [[maybe_unused]] UnifiedWindowGroups& groups) { + if constexpr (USE_UNIFIED_MEMORY) { + const u8* const physical_base = device_memory.GetPhysicalBase(); + const u64 unified_base = runtime.UnifiedMemoryBase(); + const u64 unified_size = runtime.UnifiedMemorySize(); + const u64 window_size = runtime.UnifiedMemoryWindowSize(); + if (window_size == 0) { + return false; + } + const auto group_for = [&](u64 window) -> boost::container::small_vector& { + for (size_t i = 0; i < window_ids.size(); ++i) { + if (window_ids[i] == window) { + return groups[i]; + } + } + window_ids.push_back(window); + groups.emplace_back(); + return groups.back(); + }; + u64 downloaded = 0; + while (downloaded < size) { + const DAddr page_addr = device_addr + downloaded; + const u8* const ptr = device_memory.GetPointer(page_addr); + if (ptr == nullptr) { + return false; + } + const u64 page_offset = page_addr & Core::DEVICE_PAGEMASK; + u64 chunk = (std::min)(size - downloaded, + static_cast(Core::DEVICE_PAGESIZE) - page_offset); + const u64 phys_offset = static_cast(ptr - physical_base); + if (phys_offset < unified_base || phys_offset - unified_base + chunk > unified_size) { + return false; + } + const u64 relative = phys_offset - unified_base; + const u64 window = relative / window_size; + const u64 local_offset = relative % window_size; + chunk = (std::min)(chunk, window_size - local_offset); + auto& group = group_for(window); + if (!group.empty()) { + BufferCopy& last = group.back(); + if (last.src_offset + last.size == buffer_offset + downloaded && + last.dst_offset + last.size == local_offset) { + last.size += chunk; + downloaded += chunk; + continue; + } + } + group.push_back(BufferCopy{ + .src_offset = buffer_offset + downloaded, + .dst_offset = local_offset, + .size = chunk, + }); + downloaded += chunk; + } + return true; + } else { + return false; + } +} + +template +bool BufferCache

::TryUnifiedDownloadMemory([[maybe_unused]] Buffer& buffer, + [[maybe_unused]] std::span copies) { + if constexpr (USE_UNIFIED_MEMORY) { + boost::container::small_vector window_ids; + UnifiedWindowGroups groups; + for (const BufferCopy& copy : copies) { + if (!ResolveUnifiedWindows(buffer.CpuAddr() + copy.src_offset, copy.src_offset, + copy.size, window_ids, groups)) { + return false; + } + } + for (const BufferCopy& copy : copies) { + buffer.MarkUsage(copy.src_offset, copy.size); + } + runtime.PreCopyBarrier(); + for (size_t i = 0; i < window_ids.size(); ++i) { + const std::span group_span(groups[i].data(), groups[i].size()); + runtime.CopyToUnifiedMemory(window_ids[i], buffer, group_span); + } + runtime.UnifiedMemoryHostBarrier(); + runtime.Finish(); + return true; + } else { + return false; + } +} + template void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] u64 total_size_bytes, @@ -1802,6 +2080,12 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 } if constexpr (USE_MEMORY_MAPS) { + if constexpr (USE_UNIFIED_MEMORY) { + if (runtime.HasUnifiedMemory() && + TryUnifiedDownloadMemory(buffer, std::span(copies.data(), copies.size()))) { + return; + } + } auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); const u8* const mapped_memory = download_staging.mapped_span.data(); const std::span copies_span(copies.data(), copies.data() + copies.size()); @@ -1872,7 +2156,7 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { #ifdef YUZU_LEGACY if (!do_not_mark || !immediately_free) #endif - delayed_destruction_ring.Push(std::move(slot_buffers[buffer_id])); + sentenced_buffers.Push(std::move(slot_buffers[buffer_id]), runtime.CurrentSyncPoint()); slot_buffers.erase(buffer_id); diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 14ab3e6ebc..804a5ca056 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -30,7 +31,7 @@ #include "common/slot_vector.h" #include "video_core/buffer_cache/buffer_base.h" #include "video_core/control/channel_state_cache.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/dirty_flags.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/engines/kepler_compute.h" @@ -180,15 +181,18 @@ class BufferCache : public VideoCommon::ChannelSetupCaches copies); + bool TryUnifiedDownloadMemory(Buffer& buffer, std::span copies); + + using UnifiedWindowGroups = + boost::container::small_vector, 4>; + + bool ResolveUnifiedWindows(DAddr device_addr, u64 buffer_offset, u64 size, + boost::container::small_vector& window_ids, + UnifiedWindowGroups& groups); + void DownloadBufferMemory(Buffer& buffer_id); void DownloadBufferMemory(Buffer& buffer_id, DAddr device_addr, u64 size); @@ -475,12 +494,7 @@ private: Tegra::MaxwellDeviceMemoryManager& device_memory; Common::SlotVector slot_buffers; -#ifdef YUZU_LEGACY - static constexpr size_t TICKS_TO_DESTROY = 6; -#else - static constexpr size_t TICKS_TO_DESTROY = 8; -#endif - DelayedDestructionRing delayed_destruction_ring; + DeferredDestructionQueue sentenced_buffers; const Tegra::Engines::Maxwell3D::DrawManager::IndirectParams* current_draw_indirect{}; @@ -498,9 +512,14 @@ private: std::deque> committed_gpu_modified_ranges; // Async Buffers + struct AsyncDownloadBatch { + boost::container::small_vector staging_copies; + boost::container::small_vector unified_copies; + }; + Common::OverlapRangeSet async_downloads; std::deque> async_buffers; - std::deque> pending_downloads; + std::deque pending_downloads; std::optional current_buffer; std::deque async_buffers_death_ring; @@ -515,8 +534,14 @@ private: Common::LeastRecentlyUsedCache lru_cache; u64 frame_tick = 0; u64 total_used_memory = 0; - u64 minimum_memory = 0; - u64 critical_memory = 0; + u64 memory_budget = 0; + u64 cached_device_usage = 0; + /// Sync point the last reclaim's evictions were queued at. Their memory is not back with the + /// device until this completes, so reclaiming again before then measures stale usage. + u64 reclaim_wait_sync_point = 0; + u32 usage_refresh_countdown = 0; + bool in_reclaim = false; + bool reclaim_stalled = false; BufferId inline_buffer_id; #ifdef YUZU_LEGACY bool immediately_free = false; diff --git a/src/video_core/deferred_destruction_queue.h b/src/video_core/deferred_destruction_queue.h new file mode 100644 index 0000000000..d5faf1e6aa --- /dev/null +++ b/src/video_core/deferred_destruction_queue.h @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include +#include + +#include "common/common_types.h" + +namespace VideoCommon { + +template +class DeferredDestructionQueue { +public: + void Push(T&& object, u64 sync_point) { + entries.emplace_back(std::move(object), sync_point); + } + + void Reclaim(u64 completed_sync_point) { + while (!entries.empty() && entries.front().sync_point <= completed_sync_point) { + entries.pop_front(); + } + } + + void Clear() { + entries.clear(); + } + + [[nodiscard]] size_t Size() const noexcept { + return entries.size(); + } + + [[nodiscard]] bool Empty() const noexcept { + return entries.empty(); + } + +private: + struct Entry { + Entry(T&& object_, u64 sync_point_) noexcept + : object{std::move(object_)}, sync_point{sync_point_} {} + + T object; + u64 sync_point; + }; + + using EntryDequeOptions = + boost::container::deque_options>::type; + + boost::container::deque entries; +}; + +} // namespace VideoCommon diff --git a/src/video_core/delayed_destruction_ring.h b/src/video_core/delayed_destruction_ring.h deleted file mode 100644 index 1f7af86d26..0000000000 --- a/src/video_core/delayed_destruction_ring.h +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include -#include -#include -#include - -namespace VideoCommon { - -/// Container to push objects to be destroyed a few ticks in the future -template -class DelayedDestructionRing { -public: - void Tick() { - index = (index + 1) % TICKS_TO_DESTROY; - elements[index].clear(); - } - - void Push(T&& object) { - elements[index].push_back(std::move(object)); - } - -private: - size_t index = 0; - std::array, TICKS_TO_DESTROY> elements; -}; - -} // namespace VideoCommon diff --git a/src/video_core/engines/fermi_2d.cpp b/src/video_core/engines/fermi_2d.cpp index 8b02c91e03..b1f92fe9ca 100644 --- a/src/video_core/engines/fermi_2d.cpp +++ b/src/video_core/engines/fermi_2d.cpp @@ -71,7 +71,8 @@ void Fermi2D::Blit() { constexpr s64 null_derivative = 1ULL << 32; Surface src = regs.src; const auto bytes_per_pixel = BytesPerBlock(PixelFormatFromRenderTargetFormat(src.format)); - const bool delegate_to_gpu = src.width > 512 && src.height > 512 && bytes_per_pixel <= 8 && + const u64 src_area = static_cast(src.width) * static_cast(src.height); + const bool delegate_to_gpu = src_area > 512ULL * 512ULL && bytes_per_pixel <= 8 && src.format != regs.dst.format; auto srcX = args.src_x0; diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 16f19bf009..a7d45acc61 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -18,7 +18,7 @@ #include "common/common_types.h" #include "common/settings.h" #include "common/thread.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/gpu.h" #include "video_core/host1x/host1x.h" #include "video_core/host1x/syncpoint_manager.h" @@ -50,7 +50,8 @@ public: /// Notify the fence manager about a new frame void TickFrame() { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Tick(); + ++retire_tick; + sentenced_fences.Reclaim(retire_tick > RETIRE_DELAY ? retire_tick - RETIRE_DELAY : 0); } // Unlike other fences, this one doesn't @@ -91,9 +92,6 @@ public: func(); } fences.push(std::move(new_fence)); - if (should_flush) { - rasterizer.FlushCommands(); - } if constexpr (can_async_check) { guard.unlock(); cv.notify_all(); @@ -186,7 +184,7 @@ private: } { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Push(std::move(current_fence)); + sentenced_fences.Push(std::move(current_fence), retire_tick); } fences.pop(); } @@ -219,7 +217,7 @@ private: } { std::unique_lock lock(ring_guard); - delayed_destruction_ring.Push(std::move(current_fence)); + sentenced_fences.Push(std::move(current_fence), retire_tick); } } } @@ -238,10 +236,10 @@ private: void PopAsyncFlushes() { { - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + std::scoped_lock lock{texture_cache.mutex}; texture_cache.PopAsyncFlushes(); - buffer_cache.PopAsyncFlushes(); } + buffer_cache.PopAsyncFlushes(); query_cache.PopAsyncFlushes(); } @@ -264,7 +262,9 @@ private: std::jthread fence_thread; - DelayedDestructionRing delayed_destruction_ring; + static constexpr u64 RETIRE_DELAY = 8; + u64 retire_tick = 1; + DeferredDestructionQueue sentenced_fences; }; } // namespace VideoCommon diff --git a/src/video_core/gpu_thread.cpp b/src/video_core/gpu_thread.cpp index 50570b596d..2acf9ac458 100644 --- a/src/video_core/gpu_thread.cpp +++ b/src/video_core/gpu_thread.cpp @@ -30,6 +30,7 @@ void ThreadManager::StartThread(VideoCore::RendererBase& renderer, Core::Fronten thread = std::jthread([&](std::stop_token stop_token) { Common::SetCurrentThreadName("GPU"); Common::SetCurrentThreadPriority(Common::ThreadPriority::Critical); + Common::SetCurrentThreadToPerformanceCores(); system.RegisterHostThread(); auto current_context = context.Acquire(); diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index 0d342c463d..9949e90e09 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -17,11 +17,13 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/astc_decoder.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_float.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d.comp + ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_2d_buffer.comp ${CMAKE_CURRENT_SOURCE_DIR}/blit_color_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_bcn.comp + ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_buffer.comp ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d24s8.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d32f.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_d32f_to_abgr8.frag @@ -32,6 +34,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/convert_msaa_to_non_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.comp ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa.frag + ${CMAKE_CURRENT_SOURCE_DIR}/convert_non_msaa_to_msaa_depth.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_s8d24_to_abgr8.frag ${CMAKE_CURRENT_SOURCE_DIR}/full_screen_triangle.vert ${CMAKE_CURRENT_SOURCE_DIR}/fxaa.frag diff --git a/src/video_core/host_shaders/astc_decoder.comp b/src/video_core/host_shaders/astc_decoder.comp index ce03e589ad..86a9c0a4d2 100644 --- a/src/video_core/host_shaders/astc_decoder.comp +++ b/src/video_core/host_shaders/astc_decoder.comp @@ -77,14 +77,8 @@ uvec4 local_buff; uvec4 color_endpoint_data; int color_bitsread = 0; -// Global "vector" to be pushed into when decoding -// At most will require BLOCK_WIDTH x BLOCK_HEIGHT in single plane mode -// At most will require BLOCK_WIDTH x BLOCK_HEIGHT x 2 in dual plane mode -// So the maximum would be 144 (12 x 12) elements, x 2 for two planes -#define DIVCEIL(number, divisor) (number + divisor - 1) / divisor -#define ARRAY_NUM_ELEMENTS 144 -#define VECTOR_ARRAY_SIZE DIVCEIL(ARRAY_NUM_ELEMENTS * 2, 4) -uint result_vector[ARRAY_NUM_ELEMENTS * 2]; +#define MAX_WEIGHT_VALUES 64 +uint result_vector[MAX_WEIGHT_VALUES]; int result_index = 0; uint result_vector_max_index; @@ -492,7 +486,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o A = ReplicateBitTo9((bitval & 1)); switch (encoding) { case JUST_BITS: - color_values[++out_index] = FastReplicateTo8(bitval, bitlen); + color_values[out_index++] = FastReplicateTo8(bitval, bitlen); break; case TRIT: { D = QuintTritValue(val); @@ -571,7 +565,7 @@ void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, o uint T = (D * C) + B; T ^= A; T = (A & 0x80) | (T >> 2); - color_values[++out_index] = T; + color_values[out_index++] = T; } } } @@ -753,12 +747,12 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui #define READ_UINT_VALUES(N) \ uvec4 V[2]; \ for (uint i = 0; i < N; i++) { \ - V[i / 4][i % 4] = color_values[++colvals_index]; \ + V[i / 4][i % 4] = color_values[colvals_index++]; \ } #define READ_INT_VALUES(N) \ ivec4 V[2]; \ for (uint i = 0; i < N; i++) { \ - V[i / 4][i % 4] = int(color_values[++colvals_index]); \ + V[i / 4][i % 4] = int(color_values[colvals_index++]); \ } switch (color_endpoint_mode) { @@ -1225,6 +1219,10 @@ void DecompressBlock(ivec3 coord) { FillError(coord); return; } + if (GetNumWeightValues(size_params, dual_plane) > MAX_WEIGHT_VALUES) { + FillError(coord); + return; + } uint partition_index = 1; uvec4 color_endpoint_mode = uvec4(0); uint ced_pointer = 0; @@ -1384,11 +1382,7 @@ void DecompressBlock(ivec3 coord) { p = Cf / 65535.0f; } -#ifdef VULKAN - imageStore(dest_image, coord + ivec3(i, j, 0), p.gbar); -#else imageStore(dest_image, coord + ivec3(i, j, 0), clamp(p, 0.0f, 1.0f).gbar); -#endif } } } diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d_buffer.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d_buffer.comp new file mode 100644 index 0000000000..3fc71e05b6 --- /dev/null +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d_buffer.comp @@ -0,0 +1,104 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 430 + +#extension GL_EXT_shader_16bit_storage : require +#extension GL_EXT_shader_8bit_storage : require + +#define BINDING_INPUT_BUFFER 0 +#define BINDING_OUTPUT_BUFFER 1 + +layout(push_constant) uniform PushConstants { + uvec3 dim; + uint bytes_per_block_log2; + + uvec3 origin; + uint layer_stride; + + uint block_size; + uint x_shift; + uint block_height; + uint block_height_mask; +} pc; + +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; }; + +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer { + uint out_u32[]; +}; + +layout(local_size_x = 16, local_size_y = 8, local_size_z = 1) in; + +const uint GOB_SIZE_X = 64; +const uint GOB_SIZE_Y = 8; + +const uint GOB_SIZE_X_SHIFT = 6; +const uint GOB_SIZE_Y_SHIFT = 3; +const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT; + +const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u); + +uint SwizzleTable(uint pos) { + const uint t[8] = uint[]( + 0x12100200, 0x13110301, 0x16140604, 0x17150705, + 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d + ); + const uint i = pos >> 4; + const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; + return (h << 4) | (pos & 0xf); +} + +uint SwizzleOffset(uvec2 pos) { + pos = pos & SWIZZLE_MASK; + return SwizzleTable(pos.y * 64u + pos.x); +} + +uvec4 ReadTexel(uint offset) { + switch (pc.bytes_per_block_log2) { + case 2u: + return uvec4(u32data[offset / 4u], 0u, 0u, 0u); + case 3u: + return uvec4(u64data[offset / 8u], 0u, 0u); + case 4u: + return u128data[offset / 16u]; + } + return uvec4(0u); +} + +void main() { + uvec3 coord = gl_GlobalInvocationID; + if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) { + return; + } + + uvec3 pos = coord + pc.origin; + pos.x <<= pc.bytes_per_block_log2; + + uint swizzle = SwizzleOffset(pos.xy); + uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; + + uint offset = 0u; + offset += pos.z * pc.layer_stride; + offset += (block_y >> pc.block_height) * pc.block_size; + offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT; + offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift; + offset += swizzle; + + uvec4 texel = ReadTexel(offset); + + uint words = 1u << (pc.bytes_per_block_log2 - 2u); + uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y; + uint out_idx = linear_index * words; + + out_u32[out_idx] = texel.x; + if (words > 1u) { + out_u32[out_idx + 1u] = texel.y; + } + if (words > 2u) { + out_u32[out_idx + 2u] = texel.z; + out_u32[out_idx + 3u] = texel.w; + } +} diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_buffer.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_buffer.comp new file mode 100644 index 0000000000..7333568eb7 --- /dev/null +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d_buffer.comp @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 430 + +#define BINDING_INPUT_BUFFER 0 +#define BINDING_OUTPUT_BUFFER 1 + +layout(push_constant) uniform PushConstants { + uvec3 dim; + uint bytes_per_block_log2; + + uvec3 origin; + uint slice_size; + + uint block_size; + uint x_shift; + uint block_height; + uint block_height_mask; + + uint block_depth; + uint block_depth_mask; +} pc; + +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; }; +layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; }; + +layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer { + uint out_u32[]; +}; + +layout(local_size_x = 8, local_size_y = 8, local_size_z = 4) in; + +const uint GOB_SIZE_X = 64; +const uint GOB_SIZE_Y = 8; + +const uint GOB_SIZE_X_SHIFT = 6; +const uint GOB_SIZE_Y_SHIFT = 3; +const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT; + +const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u); + +uint SwizzleTable(uint pos) { + const uint t[8] = uint[]( + 0x12100200, 0x13110301, 0x16140604, 0x17150705, + 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d + ); + const uint i = pos >> 4; + const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; + return (h << 4) | (pos & 0xf); +} + +uint SwizzleOffset(uvec2 pos) { + pos = pos & SWIZZLE_MASK; + return SwizzleTable(pos.y * 64u + pos.x); +} + +uvec4 ReadTexel(uint offset) { + switch (pc.bytes_per_block_log2) { + case 2u: + return uvec4(u32data[offset / 4u], 0u, 0u, 0u); + case 3u: + return uvec4(u64data[offset / 8u], 0u, 0u); + case 4u: + return u128data[offset / 16u]; + } + return uvec4(0u); +} + +void main() { + uvec3 coord = gl_GlobalInvocationID; + if (coord.x >= pc.dim.x || coord.y >= pc.dim.y || coord.z >= pc.dim.z) { + return; + } + + uvec3 pos = coord + pc.origin; + pos.x <<= pc.bytes_per_block_log2; + + uint swizzle = SwizzleOffset(pos.xy); + uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; + + uint offset = 0u; + offset += (pos.z >> pc.block_depth) * pc.slice_size; + offset += (pos.z & pc.block_depth_mask) << (GOB_SIZE_SHIFT + pc.block_height); + offset += (block_y >> pc.block_height) * pc.block_size; + offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT; + offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift; + offset += swizzle; + + uvec4 texel = ReadTexel(offset); + + uint words = 1u << (pc.bytes_per_block_log2 - 2u); + uint linear_index = coord.x + coord.y * pc.dim.x + coord.z * pc.dim.x * pc.dim.y; + uint out_idx = linear_index * words; + + out_u32[out_idx] = texel.x; + if (words > 1u) { + out_u32[out_idx + 1u] = texel.y; + } + if (words > 2u) { + out_u32[out_idx + 2u] = texel.z; + out_u32[out_idx + 3u] = texel.w; + } +} diff --git a/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag new file mode 100644 index 0000000000..f883b3196f --- /dev/null +++ b/src/video_core/host_shaders/convert_non_msaa_to_msaa_depth.frag @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 450 core + +layout(binding = 0) uniform sampler2D img_in; + +layout(push_constant) uniform PushConstants { + ivec2 dst_offset; + ivec2 src_offset; + ivec2 scale; +}; + +void main() { + const ivec2 msaa_coord = ivec2(gl_FragCoord.xy) - dst_offset; + const ivec2 sample_offset = ivec2(gl_SampleID % scale.x, gl_SampleID / scale.x); + const ivec2 coord = msaa_coord * scale + sample_offset + src_offset; + gl_FragDepth = texelFetch(img_in, coord, 0).r; +} diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index c583fbdf7d..a78486c39e 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -370,8 +370,37 @@ inline void MemoryManager::MemoryOperation(GPUVAddr gpu_src_addr, std::size_t si template void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which) const { + const u8* run_src{nullptr}; + u8* run_dst{nullptr}; + std::size_t run_size{0}; + auto flush_run = [&] { + if (run_size == 0) { + return; + } + std::memcpy(run_dst, run_src, run_size); + run_src = nullptr; + run_dst = nullptr; + run_size = 0; + }; + auto append_run = [&](const u8* physical, std::size_t copy_amount) { + if (physical == nullptr) [[unlikely]] { + flush_run(); + std::memset(dest_buffer, 0, copy_amount); + return; + } + if (run_size != 0 && run_src + run_size == physical && + run_dst + run_size == static_cast(dest_buffer)) { + run_size += copy_amount; + return; + } + flush_run(); + run_src = physical; + run_dst = static_cast(dest_buffer); + run_size = copy_amount; + }; auto set_to_zero = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + flush_run(); std::memset(dest_buffer, 0, copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; @@ -381,8 +410,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: if constexpr (is_safe) { rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(dest_buffer, physical, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); dest_buffer = static_cast(dest_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -392,10 +420,10 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: rasterizer->FlushRegion(dev_addr_base, copy_amount, which); } if (!IsBigPageContinuous(page_index)) [[unlikely]] { + flush_run(); memory.ReadBlockUnsafe(dev_addr_base, dest_buffer, copy_amount); } else { - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(dest_buffer, physical, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); } dest_buffer = static_cast(dest_buffer) + copy_amount; }; @@ -405,6 +433,7 @@ void MemoryManager::ReadBlockImpl(GPUVAddr gpu_src_addr, void* dest_buffer, std: MemoryOperation(base, copy_amount, mapped_normal, set_to_zero, set_to_zero); }; MemoryOperation(gpu_src_addr, size, mapped_big, set_to_zero, read_short_pages); + flush_run(); } void MemoryManager::ReadBlock(GPUVAddr gpu_src_addr, void* dest_buffer, std::size_t size, @@ -420,8 +449,36 @@ void MemoryManager::ReadBlockUnsafe(GPUVAddr gpu_src_addr, void* dest_buffer, template void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, [[maybe_unused]] VideoCommon::CacheType which) { + const u8* run_src{nullptr}; + u8* run_dst{nullptr}; + std::size_t run_size{0}; + auto flush_run = [&] { + if (run_size == 0) { + return; + } + std::memcpy(run_dst, run_src, run_size); + run_src = nullptr; + run_dst = nullptr; + run_size = 0; + }; + auto append_run = [&](u8* physical, std::size_t copy_amount) { + if (physical == nullptr) [[unlikely]] { + flush_run(); + return; + } + if (run_size != 0 && run_dst + run_size == physical && + run_src + run_size == static_cast(src_buffer)) { + run_size += copy_amount; + return; + } + flush_run(); + run_src = static_cast(src_buffer); + run_dst = physical; + run_size = copy_amount; + }; auto just_advance = [&]([[maybe_unused]] std::size_t page_index, [[maybe_unused]] std::size_t offset, std::size_t copy_amount) { + flush_run(); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_normal = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -430,8 +487,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe if constexpr (is_safe) { rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(physical, src_buffer, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); src_buffer = static_cast(src_buffer) + copy_amount; }; auto mapped_big = [&](std::size_t page_index, std::size_t offset, std::size_t copy_amount) { @@ -441,10 +497,10 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe rasterizer->InvalidateRegion(dev_addr_base, copy_amount, which); } if (!IsBigPageContinuous(page_index)) [[unlikely]] { + flush_run(); memory.WriteBlockUnsafe(dev_addr_base, src_buffer, copy_amount); } else { - u8* physical = memory.GetPointer(dev_addr_base); - std::memcpy(physical, src_buffer, copy_amount); + append_run(memory.GetPointer(dev_addr_base), copy_amount); } src_buffer = static_cast(src_buffer) + copy_amount; }; @@ -454,6 +510,7 @@ void MemoryManager::WriteBlockImpl(GPUVAddr gpu_dest_addr, const void* src_buffe MemoryOperation(base, copy_amount, mapped_normal, just_advance, just_advance); }; MemoryOperation(gpu_dest_addr, size, mapped_big, just_advance, write_short_pages); + flush_run(); } void MemoryManager::WriteBlock(GPUVAddr gpu_dest_addr, const void* src_buffer, std::size_t size, @@ -605,7 +662,7 @@ bool MemoryManager::IsGranularRange(GPUVAddr gpu_addr, std::size_t size) const { if (GetEntry(gpu_addr) == EntryType::Mapped) [[likely]] { size_t page_index = gpu_addr >> big_page_bits; if (IsBigPageContinuous(page_index)) [[likely]] { - const std::size_t page{(page_index & big_page_mask) + size}; + const std::size_t page{(gpu_addr & big_page_mask) + size}; return page <= big_page_size; } const std::size_t page{(gpu_addr & Core::DEVICE_PAGEMASK) + size}; diff --git a/src/video_core/renderer_opengl/gl_buffer_cache.h b/src/video_core/renderer_opengl/gl_buffer_cache.h index a0acfc48c1..7c8d4de287 100644 --- a/src/video_core/renderer_opengl/gl_buffer_cache.h +++ b/src/video_core/renderer_opengl/gl_buffer_cache.h @@ -93,7 +93,17 @@ public: void PostCopyBarrier(); void Finish(); - void TickFrame(Common::SlotVector&) noexcept {} + void TickFrame(Common::SlotVector&) noexcept { + ++sync_point; + } + + u64 CurrentSyncPoint() const noexcept { + return sync_point; + } + + u64 CompletedSyncPoint() const noexcept { + return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0; + } void ClearBuffer(Buffer& dest_buffer, u32 offset, size_t size, u32 value); @@ -128,6 +138,10 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const { + return GetDeviceMemoryUsage(); + } + void BindFastUniformBuffer(size_t stage, u32 binding_index, u32 size) { const GLuint handle = fast_uniforms[stage][binding_index].handle; const GLsizeiptr gl_size = static_cast(size); @@ -213,9 +227,13 @@ private: GL_FRAGMENT_PROGRAM_PARAMETER_BUFFER_NV, }; + static constexpr u64 SYNC_POINT_DELAY = 8; + const Device& device; StagingBufferPool& staging_buffer_pool; + u64 sync_point = 1; + bool has_fast_buffer_sub_data = false; bool use_assembly_shaders = false; bool has_unified_vertex_buffers = false; @@ -261,6 +279,7 @@ struct BufferCacheParams { // TODO: Investigate why OpenGL seems to perform worse with persistently mapped buffer uploads static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = false; + static constexpr bool USE_UNIFIED_MEMORY = false; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 1d77d28c46..e660f75dae 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -485,7 +485,6 @@ void RasterizerOpenGL::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType texture_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::BufferCache))) { - std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::QueryCache))) { diff --git a/src/video_core/renderer_opengl/gl_texture_cache.h b/src/video_core/renderer_opengl/gl_texture_cache.h index dfcef4b0b6..942ffdd514 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.h +++ b/src/video_core/renderer_opengl/gl_texture_cache.h @@ -87,6 +87,10 @@ public: u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const { + return GetDeviceMemoryUsage(); + } + bool CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -139,7 +143,19 @@ public: bool HasNativeASTC() const noexcept; - void TickFrame() {} + void TickFrame() { + ++sync_point; + } + + u64 CurrentSyncPoint() const noexcept { + return sync_point; + } + + u64 CompletedSyncPoint() const noexcept { + return sync_point > SYNC_POINT_DELAY ? sync_point - SYNC_POINT_DELAY : 0; + } + + void WaitSyncPoint(u64) {} StateTracker& GetStateTracker() { return state_tracker; @@ -174,6 +190,9 @@ private: std::array rescale_read_fbos; const Settings::ResolutionScalingInfo& resolution; u64 device_access_memory; + + static constexpr u64 SYNC_POINT_DELAY = 8; + u64 sync_point = 1; }; class Image : public VideoCommon::ImageBase { @@ -370,6 +389,7 @@ struct TextureCacheParams { static constexpr bool HAS_EMULATED_COPIES = true; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true; + static constexpr bool HAS_TIMELINE_SYNC_POINTS = false; using Runtime = OpenGL::TextureCacheRuntime; using Image = OpenGL::Image; diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index 89083d29fb..f4af87fb00 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -21,6 +21,7 @@ #include "video_core/host_shaders/convert_depth_to_float_frag_spv.h" #include "video_core/host_shaders/convert_float_to_depth_frag_spv.h" #include "video_core/host_shaders/convert_msaa_to_non_msaa_frag_spv.h" +#include "video_core/host_shaders/convert_non_msaa_to_msaa_depth_frag_spv.h" #include "video_core/host_shaders/convert_non_msaa_to_msaa_frag_spv.h" #include "video_core/host_shaders/convert_s8d24_to_abgr8_frag_spv.h" #include "video_core/host_shaders/full_screen_triangle_vert_spv.h" @@ -519,7 +520,8 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) } [[nodiscard]] vk::ImageView MakeMSAACopyView(const vk::Device& device, VkImage image, - VkFormat format, u32 base_level) { + VkFormat format, u32 base_level, + VkImageAspectFlags aspect_mask) { return device.CreateImageView(VkImageViewCreateInfo{ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, .pNext = nullptr, @@ -534,7 +536,7 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) .a = VK_COMPONENT_SWIZZLE_IDENTITY, }, .subresourceRange{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect_mask, .baseMipLevel = base_level, .levelCount = 1, .baseArrayLayer = 0, @@ -545,6 +547,10 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) { const VkRenderPass render_pass = framebuffer->RenderPass(); + if (!render_pass) { + framebuffer->BeginRendering(cmdbuf); + return; + } const VkFramebuffer framebuffer_handle = framebuffer->Handle(); const VkExtent2D render_area = framebuffer->RenderArea(); const VkRenderPassBeginInfo renderpass_bi{ @@ -561,6 +567,31 @@ void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) }; cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); } + +void EndRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) { + if (framebuffer->RenderPass()) { + cmdbuf.EndRenderPass(); + } else { + cmdbuf.EndRendering(); + } +} + +[[nodiscard]] VkPipelineRenderingCreateInfo MakePipelineRenderingCreateInfo( + const Framebuffer* framebuffer) { + return VkPipelineRenderingCreateInfo{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, + .pNext = nullptr, + .viewMask = 0, + .colorAttachmentCount = framebuffer->NumColorAttachments(), + .pColorAttachmentFormats = framebuffer->ColorAttachmentFormats().data(), + .depthAttachmentFormat = framebuffer->HasAspectDepthBit() + ? framebuffer->DepthAttachmentFormat() + : VK_FORMAT_UNDEFINED, + .stencilAttachmentFormat = framebuffer->HasAspectStencilBit() + ? framebuffer->DepthAttachmentFormat() + : VK_FORMAT_UNDEFINED, + }; +} } // Anonymous namespace BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, @@ -610,6 +641,8 @@ BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, convert_s8d24_to_abgr8_frag(BuildShader(device, CONVERT_S8D24_TO_ABGR8_FRAG_SPV)), convert_msaa_to_non_msaa_frag(BuildShader(device, CONVERT_MSAA_TO_NON_MSAA_FRAG_SPV)), convert_non_msaa_to_msaa_frag(BuildShader(device, CONVERT_NON_MSAA_TO_MSAA_FRAG_SPV)), + convert_non_msaa_to_msaa_depth_frag( + BuildShader(device, CONVERT_NON_MSAA_TO_MSAA_DEPTH_FRAG_SPV)), linear_sampler(device.GetLogical().CreateSampler(SAMPLER_CREATE_INFO)), nearest_sampler(device.GetLogical().CreateSampler(SAMPLER_CREATE_INFO)) {} @@ -623,10 +656,12 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, const ImageV const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = operation, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *one_texture_pipeline_layout; const VkSampler sampler = is_linear ? *linear_sampler : *nearest_sampler; - const VkPipeline pipeline = FindOrEmplaceColorPipeline(key); + const VkPipeline pipeline = FindOrEmplaceColorPipeline(key, dst_framebuffer); const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D); RecordShaderReadBarrier(scheduler, src_image_view); @@ -651,9 +686,11 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *one_texture_pipeline_layout; - const VkPipeline pipeline = FindOrEmplaceColorPipeline(key); + const VkPipeline pipeline = FindOrEmplaceColorPipeline(key, dst_framebuffer); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, dst_framebuffer, src_image_view, src_image, src_sampler, dst_region, src_region, src_size, pipeline, layout](vk::CommandBuffer cmdbuf) { @@ -666,7 +703,7 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView nullptr); BindBlitState(cmdbuf, layout, dst_region, src_region, src_size); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + EndRenderPass(cmdbuf, dst_framebuffer); }); } @@ -676,10 +713,12 @@ void BlitImageHelper::BlitColorMSAA(const Framebuffer* dst_framebuffer, const BlitMSAAPipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .samples = dst_framebuffer->Samples(), + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *one_texture_pipeline_layout; const VkSampler sampler = *nearest_sampler; - const VkPipeline pipeline = FindOrEmplaceBlitColorMSAAPipeline(key); + const VkPipeline pipeline = FindOrEmplaceBlitColorMSAAPipeline(key, dst_framebuffer); const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D); RecordShaderReadBarrier(scheduler, src_image_view); @@ -703,7 +742,7 @@ void BlitImageHelper::ResolveDepthStencil(const Framebuffer* dst_framebuffer, const bool resolve_stencil = dst_framebuffer->HasAspectStencilBit() && device.IsExtShaderStencilExportSupported(); const VkPipeline pipeline = - FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer->RenderPass(), resolve_stencil); + FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer, resolve_stencil); const VkPipelineLayout layout = resolve_stencil ? *two_textures_pipeline_layout : *one_texture_pipeline_layout; const VkSampler sampler = *nearest_sampler; @@ -747,10 +786,12 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = operation, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; const VkPipelineLayout layout = *two_textures_pipeline_layout; const VkSampler sampler = *nearest_sampler; - const VkPipeline pipeline = FindOrEmplaceDepthStencilPipeline(key); + const VkPipeline pipeline = FindOrEmplaceDepthStencilPipeline(key, dst_framebuffer); const VkImageView src_depth_view = src_image_view.DepthView(); const VkImageView src_stencil_view = src_image_view.StencilView(); @@ -772,25 +813,25 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, void BlitImageHelper::ConvertD32ToR32(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer->RenderPass()); + ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer); Convert(*convert_d32_to_r32_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertR32ToD32(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer->RenderPass()); + ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer); Convert(*convert_r32_to_d32_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD16ToR16(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer->RenderPass()); + ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer); Convert(*convert_d16_to_r16_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertR16ToD16(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer->RenderPass()); + ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer); Convert(*convert_r16_to_d16_pipeline, dst_framebuffer, src_image_view); } @@ -801,35 +842,35 @@ void BlitImageHelper::ConvertABGR8ToD24S8(const Framebuffer* dst_framebuffer, LOG_WARNING(Render_Vulkan, "ConvertABGR8ToD24S8 requires shader_stencil_export, skipping"); return; } - ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer, convert_abgr8_to_d24s8_frag); Convert(*convert_abgr8_to_d24s8_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertABGR8ToD32F(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer, convert_abgr8_to_d32f_frag); Convert(*convert_abgr8_to_d32f_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD32FToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer, convert_d32f_to_abgr8_frag); ConvertDepthStencil(*convert_d32f_to_abgr8_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD24S8ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer, convert_d24s8_to_abgr8_frag); ConvertDepthStencil(*convert_d24s8_to_abgr8_pipeline, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertS8D24ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer->RenderPass(), + ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer, convert_s8d24_to_abgr8_frag); ConvertDepthStencil(*convert_s8d24_to_abgr8_pipeline, dst_framebuffer, src_image_view); } @@ -840,8 +881,10 @@ void BlitImageHelper::ClearColor(const Framebuffer* dst_framebuffer, u8 color_ma const BlitImagePipelineKey key{ .renderpass = dst_framebuffer->RenderPass(), .operation = Tegra::Engines::Fermi2D::Operation::BlendPremult, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; - const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key); + const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key, dst_framebuffer); const VkPipelineLayout layout = *clear_color_pipeline_layout; scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record( @@ -867,8 +910,10 @@ void BlitImageHelper::ClearDepthStencil(const Framebuffer* dst_framebuffer, bool .stencil_mask = stencil_mask, .stencil_compare_mask = stencil_compare_mask, .stencil_ref = stencil_ref, + .color_formats = dst_framebuffer->ColorAttachmentFormats(), + .depth_format = dst_framebuffer->DepthAttachmentFormat(), }; - const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key); + const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key, dst_framebuffer); const VkPipelineLayout layout = *clear_color_pipeline_layout; scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record([pipeline, layout, clear_depth, dst_region](vk::CommandBuffer cmdbuf) { @@ -895,16 +940,34 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i const s32 scale_y = 1 << samples_y; const VkSampleCountFlagBits samples = msaa_to_non_msaa ? VK_SAMPLE_COUNT_1_BIT : SampleCountFlag(num_samples); + const auto dst_surface_type = VideoCore::Surface::GetFormatType(dst_format); + const bool is_depth = dst_surface_type == VideoCore::Surface::SurfaceType::Depth || + dst_surface_type == VideoCore::Surface::SurfaceType::DepthStencil; + const bool has_stencil = dst_surface_type == VideoCore::Surface::SurfaceType::DepthStencil; + const VkImageAspectFlags view_aspect = + is_depth ? VK_IMAGE_ASPECT_DEPTH_BIT : VK_IMAGE_ASPECT_COLOR_BIT; + VkImageAspectFlags barrier_aspect = VK_IMAGE_ASPECT_COLOR_BIT; + if (is_depth) { + barrier_aspect = VK_IMAGE_ASPECT_DEPTH_BIT; + if (has_stencil) { + barrier_aspect |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + } RenderPassKey renderpass_key{}; renderpass_key.color_formats.fill(VideoCore::Surface::PixelFormat::Invalid); - renderpass_key.color_formats[0] = dst_format; - renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid; + if (is_depth) { + renderpass_key.depth_format = dst_format; + } else { + renderpass_key.color_formats[0] = dst_format; + renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid; + } renderpass_key.samples = samples; const VkRenderPass renderpass = render_pass_cache.Get(renderpass_key); const MSAACopyPipelineKey key{ .renderpass = renderpass, .samples = samples, .msaa_to_non_msaa = msaa_to_non_msaa, + .is_depth = is_depth, }; const VkPipeline pipeline = FindOrEmplaceMSAACopyPipeline(key); const VkPipelineLayout layout = *msaa_copy_pipeline_layout; @@ -920,10 +983,10 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i ASSERT(copy.dst_subresource.num_layers == 1); vk::ImageView src_view = MakeMSAACopyView(device.GetLogical(), src_image, src_vk_format, - static_cast(copy.src_subresource.base_level)); + static_cast(copy.src_subresource.base_level), view_aspect); vk::ImageView dst_view = MakeMSAACopyView(device.GetLogical(), dst_image, dst_vk_format, - static_cast(copy.dst_subresource.base_level)); + static_cast(copy.dst_subresource.base_level), view_aspect); const VkOffset2D dst_offset{copy.dst_offset.x, copy.dst_offset.y}; const VkExtent2D dst_extent{copy.extent.width, copy.extent.height}; const VkRect2D render_area{ @@ -949,50 +1012,64 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, pipeline, layout, sampler, renderpass, framebuffer_handle = *framebuffer, src_view_handle = *src_view, - src = src_image, dst = dst_image, render_area, + src = src_image, dst = dst_image, render_area, is_depth, barrier_aspect, push_constants](vk::CommandBuffer cmdbuf) { - constexpr VkImageSubresourceRange color_range{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + const VkImageSubresourceRange src_range{ + .aspectMask = barrier_aspect, .baseMipLevel = 0, .levelCount = VK_REMAINING_MIP_LEVELS, .baseArrayLayer = 0, .layerCount = VK_REMAINING_ARRAY_LAYERS, }; + const VkImageSubresourceRange dst_range = src_range; + const VkAccessFlags attachment_read = + is_depth ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT + : VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; + const VkAccessFlags attachment_write = + is_depth ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + : VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const VkPipelineStageFlags depth_stage = + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; + const VkPipelineStageFlags attachment_stage = + is_depth ? depth_stage + : static_cast( + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT); const std::array pre_barriers{ VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .srcAccessMask = attachment_write | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = src, - .subresourceRange = color_range, + .subresourceRange = src_range, }, VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .srcAccessMask = attachment_write | VK_ACCESS_SHADER_WRITE_BIT | + VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = attachment_read | attachment_write, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst, - .subresourceRange = color_range, + .subresourceRange = dst_range, }, }; cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | attachment_stage, 0, nullptr, nullptr, pre_barriers); const VkRenderPassBeginInfo renderpass_bi{ .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, @@ -1025,16 +1102,16 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i const VkImageMemoryBarrier post_barrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .srcAccessMask = attachment_write, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst, - .subresourceRange = color_range, + .subresourceRange = dst_range, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + cmdbuf.PipelineBarrier(attachment_stage, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, @@ -1140,12 +1217,14 @@ void BlitImageHelper::ConvertDepthStencil(VkPipeline pipeline, const Framebuffer scheduler.InvalidateState(); } -VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(blit_color_keys, key); if (it != blit_color_keys.end()) { return *blit_color_pipelines[std::distance(blit_color_keys.begin(), it)]; } blit_color_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *blit_color_to_color_frag); const VkPipelineColorBlendAttachmentState blend_attachment{ @@ -1173,7 +1252,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); blit_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1191,21 +1270,23 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *blit_color_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(blit_depth_stencil_keys, key); if (it != blit_depth_stencil_keys.end()) { return *blit_depth_stencil_pipelines[std::distance(blit_depth_stencil_keys.begin(), it)]; } blit_depth_stencil_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *blit_depth_stencil_frag); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); blit_depth_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1223,42 +1304,50 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *blit_depth_stencil_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(clear_color_keys, key); if (it != clear_color_keys.end()) { return *clear_color_pipelines[std::distance(clear_color_keys.begin(), it)]; } clear_color_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*clear_color_vert, *clear_color_frag); - const VkPipelineColorBlendAttachmentState color_blend_attachment_state{ - .blendEnable = VK_TRUE, - .srcColorBlendFactor = VK_BLEND_FACTOR_CONSTANT_COLOR, - .dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR, - .colorBlendOp = VK_BLEND_OP_ADD, - .srcAlphaBlendFactor = VK_BLEND_FACTOR_CONSTANT_ALPHA, - .dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA, - .alphaBlendOp = VK_BLEND_OP_ADD, - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | - VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - }; + const u32 num_color = framebuffer->NumColorAttachments(); + constexpr VkColorComponentFlags full_write_mask = + VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | + VK_COLOR_COMPONENT_A_BIT; + std::array blend_attachments{}; + for (u32 index = 0; index < num_color; ++index) { + blend_attachments[index] = VkPipelineColorBlendAttachmentState{ + .blendEnable = index == 0 ? VK_TRUE : VK_FALSE, + .srcColorBlendFactor = VK_BLEND_FACTOR_CONSTANT_COLOR, + .dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR, + .colorBlendOp = VK_BLEND_OP_ADD, + .srcAlphaBlendFactor = VK_BLEND_FACTOR_CONSTANT_ALPHA, + .dstAlphaBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA, + .alphaBlendOp = VK_BLEND_OP_ADD, + .colorWriteMask = index == 0 ? full_write_mask : VkColorComponentFlags{0}, + }; + } const VkPipelineColorBlendStateCreateInfo color_blend_state_generic_create_info{ .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, .pNext = nullptr, .flags = 0, .logicOpEnable = VK_FALSE, .logicOp = VK_LOGIC_OP_CLEAR, - .attachmentCount = 1, - .pAttachments = &color_blend_attachment_state, + .attachmentCount = num_color, + .pAttachments = blend_attachments.data(), .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, }; const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); clear_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1276,18 +1365,31 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *clear_color_pipelines.back(); } VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( - const BlitDepthStencilPipelineKey& key) { + const BlitDepthStencilPipelineKey& key, const Framebuffer* framebuffer) { const auto it = std::ranges::find(clear_stencil_keys, key); if (it != clear_stencil_keys.end()) { return *clear_stencil_pipelines[std::distance(clear_stencil_keys.begin(), it)]; } clear_stencil_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*clear_color_vert, *clear_stencil_frag); + const u32 num_color = framebuffer->NumColorAttachments(); + std::array blend_attachments{}; + const VkPipelineColorBlendStateCreateInfo color_blend_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = num_color, + .pAttachments = blend_attachments.data(), + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; const auto stencil = VkStencilOpState{ .failOp = VK_STENCIL_OP_KEEP, .passOp = VK_STENCIL_OP_REPLACE, @@ -1314,7 +1416,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); clear_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1325,23 +1427,25 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, .pDepthStencilState = &depth_stencil_ci, - .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, + .pColorBlendState = &color_blend_ci, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *clear_color_pipeline_layout, .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *clear_stencil_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key) { +VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key, + const Framebuffer* framebuffer) { const auto it = std::ranges::find(blit_msaa_color_keys, key); if (it != blit_msaa_color_keys.end()) { return *blit_msaa_color_pipelines[std::distance(blit_msaa_color_keys.begin(), it)]; } blit_msaa_color_keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *blit_color_msaa_frag); const VkPipelineMultisampleStateCreateInfo multisample_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, @@ -1357,7 +1461,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); blit_msaa_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = key.renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1375,26 +1479,32 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *blit_msaa_color_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass, - bool resolve_stencil) { +VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline( + const Framebuffer* framebuffer, bool resolve_stencil) { + const VkRenderPass renderpass = framebuffer->RenderPass(); + const ResolveDepthStencilPipelineKey key{ + .renderpass = renderpass, + .depth_format = framebuffer->DepthAttachmentFormat(), + }; auto& keys = resolve_stencil ? resolve_depth_stencil_keys : resolve_depth_keys; auto& pipelines = resolve_stencil ? resolve_depth_stencil_pipelines : resolve_depth_pipelines; - const auto it = std::ranges::find(keys, renderpass); + const auto it = std::ranges::find(keys, key); if (it != keys.end()) { return *pipelines[std::distance(keys.begin(), it)]; } - keys.push_back(renderpass); + keys.push_back(key); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, resolve_stencil ? *blit_depth_stencil_msaa_frag : *blit_depth_msaa_frag); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1413,7 +1523,7 @@ VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPas .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *pipelines.back(); } @@ -1423,9 +1533,36 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline return *msaa_copy_pipelines[std::distance(msaa_copy_keys.begin(), it)]; } msaa_copy_keys.push_back(key); - const std::array stages = MakeStages(*clear_color_vert, key.msaa_to_non_msaa - ? *convert_msaa_to_non_msaa_frag - : *convert_non_msaa_to_msaa_frag); + const VkShaderModule frag_module = + key.msaa_to_non_msaa + ? *convert_msaa_to_non_msaa_frag + : (key.is_depth ? *convert_non_msaa_to_msaa_depth_frag + : *convert_non_msaa_to_msaa_frag); + const std::array stages = MakeStages(*clear_color_vert, frag_module); + const VkPipelineDepthStencilStateCreateInfo depth_stencil_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .depthTestEnable = VK_TRUE, + .depthWriteEnable = VK_TRUE, + .depthCompareOp = VK_COMPARE_OP_ALWAYS, + .depthBoundsTestEnable = VK_FALSE, + .stencilTestEnable = VK_FALSE, + .front = {}, + .back = {}, + .minDepthBounds = 0.0f, + .maxDepthBounds = 0.0f, + }; + static constexpr VkPipelineColorBlendStateCreateInfo no_color_blend_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = 0, + .pAttachments = nullptr, + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; const VkPipelineMultisampleStateCreateInfo multisample_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, .pNext = nullptr, @@ -1450,37 +1587,42 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pMultisampleState = &multisample_ci, - .pDepthStencilState = nullptr, - .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, + .pDepthStencilState = key.is_depth ? &depth_stencil_ci : nullptr, + .pColorBlendState = key.is_depth ? &no_color_blend_ci + : &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *msaa_copy_pipeline_layout, .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - })); + }, device.StaticPipelineCache())); return *msaa_copy_pipelines.back(); } -void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) { - ConvertPipeline(pipeline, renderpass, false); +void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline, + const Framebuffer* framebuffer) { + ConvertPipeline(pipeline, framebuffer, false); } -void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) { - ConvertPipeline(pipeline, renderpass, true); +void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline, + const Framebuffer* framebuffer) { + ConvertPipeline(pipeline, framebuffer, true); } -void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module, bool single_texture, bool is_target_depth) { if (pipeline) { return; } + const VkRenderPass renderpass = framebuffer->RenderPass(); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); const std::array stages = MakeStages(*full_screen_vert, *module); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1499,31 +1641,35 @@ void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass ren .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } -void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, + const Framebuffer* framebuffer, vk::ShaderModule& module) { - ConvertPipelineEx(pipeline, renderpass, module, false, false); + ConvertPipelineEx(pipeline, framebuffer, module, false, false); } -void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, + const Framebuffer* framebuffer, vk::ShaderModule& module) { - ConvertPipelineEx(pipeline, renderpass, module, true, true); + ConvertPipelineEx(pipeline, framebuffer, module, true, true); } -void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, +void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer, bool is_target_depth) { if (pipeline) { return; } + const VkRenderPass renderpass = framebuffer->RenderPass(); + const VkPipelineRenderingCreateInfo rendering_ci = MakePipelineRenderingCreateInfo(framebuffer); VkShaderModule frag_shader = is_target_depth ? *convert_float_to_depth_frag : *convert_depth_to_float_frag; const std::array stages = MakeStages(*full_screen_vert, frag_shader); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = renderpass ? nullptr : &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1542,7 +1688,7 @@ void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass rende .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 67294d594b..defd1caf4e 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -33,6 +33,8 @@ struct BlitImagePipelineKey { VkRenderPass renderpass; Tegra::Engines::Fermi2D::Operation operation; + std::array color_formats; + VkFormat depth_format; }; struct BlitDepthStencilPipelineKey { @@ -43,6 +45,8 @@ struct BlitDepthStencilPipelineKey { u8 stencil_mask; u32 stencil_compare_mask; u32 stencil_ref; + std::array color_formats; + VkFormat depth_format; }; struct MSAACopyPipelineKey { @@ -51,6 +55,7 @@ struct MSAACopyPipelineKey { VkRenderPass renderpass; VkSampleCountFlagBits samples; bool msaa_to_non_msaa; + bool is_depth; }; struct BlitMSAAPipelineKey { @@ -58,6 +63,15 @@ struct BlitMSAAPipelineKey { VkRenderPass renderpass; VkSampleCountFlagBits samples; + std::array color_formats; + VkFormat depth_format; +}; + +struct ResolveDepthStencilPipelineKey { + constexpr auto operator<=>(const ResolveDepthStencilPipelineKey&) const noexcept = default; + + VkRenderPass renderpass; + VkFormat depth_format; }; class BlitImageHelper { @@ -123,31 +137,36 @@ private: void ConvertDepthStencil(VkPipeline pipeline, const Framebuffer* dst_framebuffer, ImageView& src_image_view); - [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer); - [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceDepthStencilPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer); - [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key); + [[nodiscard]] VkPipeline FindOrEmplaceClearColorPipeline(const BlitImagePipelineKey& key, + const Framebuffer* framebuffer); [[nodiscard]] VkPipeline FindOrEmplaceClearStencilPipeline( - const BlitDepthStencilPipelineKey& key); + const BlitDepthStencilPipelineKey& key, const Framebuffer* framebuffer); [[nodiscard]] VkPipeline FindOrEmplaceMSAACopyPipeline(const MSAACopyPipelineKey& key); - [[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key); - [[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass, + [[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key, + const Framebuffer* framebuffer); + [[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(const Framebuffer* framebuffer, bool resolve_stencil); - void ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, bool is_target_depth); + void ConvertPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer, + bool is_target_depth); - void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass); + void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer); - void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass); + void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, const Framebuffer* framebuffer); - void ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass, + void ConvertPipelineEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module, bool single_texture, bool is_target_depth); - void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, + void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module); - void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, + void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, const Framebuffer* framebuffer, vk::ShaderModule& module); const Device& device; @@ -180,6 +199,7 @@ private: vk::ShaderModule convert_s8d24_to_abgr8_frag; vk::ShaderModule convert_msaa_to_non_msaa_frag; vk::ShaderModule convert_non_msaa_to_msaa_frag; + vk::ShaderModule convert_non_msaa_to_msaa_depth_frag; vk::Sampler linear_sampler; vk::Sampler nearest_sampler; @@ -195,9 +215,9 @@ private: std::vector msaa_copy_pipelines; std::vector blit_msaa_color_keys; std::vector blit_msaa_color_pipelines; - std::vector resolve_depth_keys; + std::vector resolve_depth_keys; std::vector resolve_depth_pipelines; - std::vector resolve_depth_stencil_keys; + std::vector resolve_depth_stencil_keys; std::vector resolve_depth_stencil_pipelines; struct MSAACopyResources { u64 tick; diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp index 0ced890929..02ee3e1e6e 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -164,7 +164,9 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe } provoking_vertex_last.Assign(use_last_provoking_vertex ? 1 : 0); - conservative_raster_enable.Assign(regs.conservative_raster_enable != 0 ? 1 : 0); + if (!features.has_dynamic_state3_conservative_raster_mode) { + conservative_raster_enable.Assign(regs.conservative_raster_enable != 0 ? 1 : 0); + } smooth_lines.Assign(regs.line_anti_alias_enable != 0 ? 1 : 0); alpha_to_coverage_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_coverage != 0 ? 1 : 0); alpha_to_one_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_one != 0 ? 1 : 0); @@ -360,18 +362,35 @@ void FixedPipelineState::DynamicState::Refresh2(const Maxwell& regs, depth_bias_enable.Assign(enabled_lut[POLYGON_OFFSET_ENABLE_LUT[topology_index]] != 0 ? 1 : 0); } +bool IsDepthClipEnabled(const Maxwell& regs) { + const auto clip = regs.viewport_clip_control.geometry_clip.Value(); + return clip == Maxwell::ViewportClipControl::GeometryClip::Passthrough || + clip == Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ || + clip == Maxwell::ViewportClipControl::GeometryClip::FrustumZ; +} + +bool IsDepthClampEnabled(const Maxwell& regs, bool has_depth_clip_enable) { + if (!IsDepthClipEnabled(regs)) { + return true; + } + if (!has_depth_clip_enable) { + return false; + } + return regs.viewport_clip_control.pixel_min_z.Value() != 0 || + regs.viewport_clip_control.pixel_max_z.Value() != 0; +} + void FixedPipelineState::DynamicState::Refresh3(const Maxwell& regs, const DynamicFeatures& features) { if (!features.has_dynamic_state3_logic_op_enable) { logic_op_enable.Assign(regs.logic_op.enable != 0 ? 1 : 0); } + if (features.has_depth_clip_enable) { + depth_clip_disabled.Assign(IsDepthClipEnabled(regs) ? 0 : 1); + } if (!features.has_dynamic_state3_depth_clamp_enable) { - depth_clamp_disabled.Assign(regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::Passthrough || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumZ); + depth_clamp_disabled.Assign( + IsDepthClampEnabled(regs, features.has_depth_clip_enable) ? 0 : 1); } if (!features.has_dynamic_state3_line_stipple_enable) { line_stipple_enable.Assign(regs.line_stipple_enable); diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.h b/src/video_core/renderer_vulkan/fixed_pipeline_state.h index 36a4b4b7a2..45920042f9 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.h +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.h @@ -30,6 +30,8 @@ struct DynamicFeatures { bool has_extended_dynamic_state_3_blend; bool has_extended_dynamic_state_3_enables; bool has_dynamic_state3_depth_clamp_enable; + bool has_dynamic_state3_conservative_raster_mode; + bool has_depth_clip_enable; bool has_dynamic_state3_logic_op_enable; bool has_dynamic_state3_line_stipple_enable; bool has_dynamic_vertex_input; @@ -165,6 +167,7 @@ struct FixedPipelineState { BitField<10, 1, u32> logic_op_enable; BitField<11, 1, u32> depth_clamp_disabled; BitField<12, 1, u32> line_stipple_enable; + BitField<13, 1, u32> depth_clip_disabled; }; union { u32 raw2; @@ -298,6 +301,9 @@ static_assert(std::has_unique_object_representations_v); static_assert(std::is_trivially_copyable_v); static_assert(std::is_trivially_constructible_v); +bool IsDepthClipEnabled(const Maxwell& regs); +bool IsDepthClampEnabled(const Maxwell& regs, bool has_depth_clip_enable); + } // namespace Vulkan namespace std { diff --git a/src/video_core/renderer_vulkan/pipeline_helper.h b/src/video_core/renderer_vulkan/pipeline_helper.h index 882372aee2..0773277e5c 100644 --- a/src/video_core/renderer_vulkan/pipeline_helper.h +++ b/src/video_core/renderer_vulkan/pipeline_helper.h @@ -47,6 +47,93 @@ using Shader::Backend::SPIRV::NUM_TEXTURE_AND_IMAGE_SCALING_WORDS; return std::nullopt; } +[[nodiscard]] inline VkDeviceSize DescriptorSizeForType(const Device& device, + VkDescriptorType type) { + const auto& props = device.DescriptorBufferProperties(); + const bool robust = device.IsRobustBufferAccessEnabled(); + switch (type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + return robust ? props.robustUniformBufferDescriptorSize : props.uniformBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + return robust ? props.robustStorageBufferDescriptorSize : props.storageBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + return robust ? props.robustUniformTexelBufferDescriptorSize + : props.uniformTexelBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + return robust ? props.robustStorageTexelBufferDescriptorSize + : props.storageTexelBufferDescriptorSize; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + return props.combinedImageSamplerDescriptorSize; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + return props.storageImageDescriptorSize; + default: + return 0; + } +} + +struct DescriptorBufferBinding { + VkDescriptorType type; + u32 count; + VkDeviceSize offset; + VkDeviceSize stride; +}; + +struct DescriptorBufferLayout { + VkDeviceSize size{}; + boost::container::small_vector bindings; + + [[nodiscard]] bool Empty() const noexcept { + return bindings.empty(); + } +}; + +inline void WriteDescriptorBuffer(const Device& device, const DescriptorBufferLayout& layout, + const DescriptorUpdateEntry* payload, u8* host) { + const vk::Device& dev = device.GetLogical(); + for (const DescriptorBufferBinding& binding : layout.bindings) { + for (u32 index = 0; index < binding.count; ++index) { + const DescriptorUpdateEntry& entry = *(payload++); + const VkDescriptorAddressInfoEXT address_info{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_ADDRESS_INFO_EXT, + .pNext = nullptr, + .address = entry.address.address, + .range = entry.address.range, + .format = entry.address.format, + }; + VkDescriptorGetInfoEXT get_info{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_GET_INFO_EXT, + .pNext = nullptr, + .type = binding.type, + .data{}, + }; + switch (binding.type) { + case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: + get_info.data.pUniformBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: + get_info.data.pStorageBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: + get_info.data.pUniformTexelBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: + get_info.data.pStorageTexelBuffer = &address_info; + break; + case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: + get_info.data.pCombinedImageSampler = &entry.image; + break; + case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: + get_info.data.pStorageImage = &entry.image; + break; + default: + continue; + } + dev.GetDescriptorEXT(get_info, binding.stride, + host + binding.offset + index * binding.stride); + } + } +} + [[nodiscard]] inline u32 NumDescriptorEntries(const Shader::Info& info) { return Shader::NumDescriptors(info.constant_buffer_descriptors) + Shader::NumDescriptors(info.storage_buffers_descriptors) + @@ -65,16 +152,59 @@ public: num_descriptors <= device->MaxPushDescriptors(); } - // TODO(crueter): utilize layout binding flags - vk::DescriptorSetLayout CreateDescriptorSetLayout(bool use_push_descriptor) const { + bool CanUseDescriptorBuffer() const noexcept { + return device->IsExtDescriptorBufferSupported() && !bindings.empty() && + !CanUsePushDescriptor() && + device->DescriptorBufferProperties().combinedImageSamplerDescriptorSingleArray; + } + + DescriptorBufferLayout MakeDescriptorBufferLayout(VkDescriptorSetLayout layout) const { + DescriptorBufferLayout result; + if (!layout) { + return result; + } + const vk::Device& dev = device->GetLogical(); + result.size = dev.GetDescriptorSetLayoutSizeEXT(layout); + result.bindings.reserve(bindings.size()); + for (const VkDescriptorSetLayoutBinding& binding : bindings) { + result.bindings.push_back(DescriptorBufferBinding{ + .type = binding.descriptorType, + .count = binding.descriptorCount, + .offset = dev.GetDescriptorSetLayoutBindingOffsetEXT(layout, binding.binding), + .stride = DescriptorSizeForType(*device, binding.descriptorType), + }); + } + return result; + } + + vk::DescriptorSetLayout CreateDescriptorSetLayout(bool use_push_descriptor, + bool use_descriptor_buffer = false) const { if (bindings.empty()) { return nullptr; } - const VkDescriptorSetLayoutCreateFlags flags = - use_push_descriptor ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR : 0; + VkDescriptorSetLayoutCreateFlags flags = 0; + if (use_push_descriptor) { + flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR; + } + if (use_descriptor_buffer) { + flags |= VK_DESCRIPTOR_SET_LAYOUT_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + } + boost::container::small_vector binding_flags; + VkDescriptorSetLayoutBindingFlagsCreateInfo binding_flags_ci{}; + const void* pnext = nullptr; + if (!use_push_descriptor && device->IsDescriptorBindingPartiallyBoundSupported()) { + binding_flags.assign(bindings.size(), VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT); + binding_flags_ci = { + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO, + .pNext = nullptr, + .bindingCount = static_cast(binding_flags.size()), + .pBindingFlags = binding_flags.data(), + }; + pnext = &binding_flags_ci; + } return device->GetLogical().CreateDescriptorSetLayout({ .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO, - .pNext = nullptr, + .pNext = pnext, .flags = flags, .bindingCount = static_cast(bindings.size()), .pBindings = bindings.data(), diff --git a/src/video_core/renderer_vulkan/present/util.cpp b/src/video_core/renderer_vulkan/present/util.cpp index 2b83902bc0..33b69c06d4 100644 --- a/src/video_core/renderer_vulkan/present/util.cpp +++ b/src/video_core/renderer_vulkan/present/util.cpp @@ -491,7 +491,7 @@ static vk::Pipeline CreateWrappedPipelineImpl( .subpass = 0, .basePipelineHandle = 0, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass, diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 4402550af9..e67a4ada3b 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -69,6 +69,9 @@ vk::Buffer CreateBuffer(const Device& device, const MemoryAllocator& memory_allo if (device.IsExtConditionalRendering()) { flags |= VK_BUFFER_USAGE_CONDITIONAL_RENDERING_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + flags |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } const VkBufferCreateInfo buffer_ci = { .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = nullptr, @@ -91,6 +94,9 @@ Buffer::Buffer(BufferCacheRuntime& runtime, VideoCommon::NullBufferParams null_p device = &runtime.device; buffer = runtime.CreateNullBuffer(); is_null = true; + if (device->IsBufferDeviceAddressSupported()) { + device_address = device->GetLogical().GetBufferDeviceAddress(*buffer); + } } Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_) @@ -100,6 +106,9 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_) if (runtime.device.HasDebuggingToolAttached()) { buffer.SetObjectNameEXT(fmt::format("Buffer 0x{:x}", CpuAddr()).c_str()); } + if (device->IsBufferDeviceAddressSupported()) { + device_address = device->GetLogical().GetBufferDeviceAddress(*buffer); + } } void Buffer::MarkUsage(u64 offset, u64 size) noexcept { @@ -246,7 +255,6 @@ protected: StagingBufferPool& staging_pool; vk::Buffer buffer{}; - MemoryCommit memory_commit{}; VkIndexType index_type{}; u32 num_indices = 0; }; @@ -356,6 +364,93 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m scheduler_, staging_pool_); } +void BufferCacheRuntime::TryEnableUnifiedMemory(void* base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, + size_t hardware_buffer_base) { + unified_memory = std::make_unique( + device, base, size, hardware_buffers, hardware_buffer_window, hardware_buffer_base); + if (!unified_memory->IsValid()) { + unified_memory.reset(); + } +} + +void BufferCacheRuntime::CopyToUnifiedMemory( + size_t window_index, VkBuffer src_buffer, + std::span copies) { + if (!unified_memory || src_buffer == VK_NULL_HANDLE || copies.empty() || + window_index >= unified_memory->GetWindowCount()) { + return; + } + const VkBuffer dst_buffer = unified_memory->GetWindowBuffer(window_index); + if (dst_buffer == VK_NULL_HANDLE) { + return; + } + VkDeviceSize covered_begin = std::numeric_limits::max(); + VkDeviceSize covered_end = 0; + for (const VideoCommon::BufferCopy& copy : copies) { + covered_begin = (std::min)(covered_begin, static_cast(copy.dst_offset)); + covered_end = (std::max)(covered_end, + static_cast(copy.dst_offset + copy.size)); + } + + boost::container::small_vector vk_copies(copies.size()); + std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); + + const bool foreign = unified_memory->NeedsForeignOwnershipTransfer(); + const u32 queue_family = device.GetGraphicsFamily(); + + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([src_buffer, dst_buffer, vk_copies, foreign, queue_family, covered_begin, + covered_end](vk::CommandBuffer cmdbuf) { + if (foreign) { + const VkBufferMemoryBarrier acquire{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .dstQueueFamilyIndex = queue_family, + .buffer = dst_buffer, + .offset = covered_begin, + .size = covered_end - covered_begin, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, acquire); + } + cmdbuf.CopyBuffer(src_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies)); + if (foreign) { + const VkBufferMemoryBarrier release{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = 0, + .srcQueueFamilyIndex = queue_family, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT, + .buffer = dst_buffer, + .offset = covered_begin, + .size = covered_end - covered_begin, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, 0, release); + } + }); +} + +void BufferCacheRuntime::UnifiedMemoryHostBarrier() { + static constexpr VkMemoryBarrier HOST_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_HOST_READ_BIT, + }; + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, 0, + HOST_BARRIER); + }); +} + StagingBufferRef BufferCacheRuntime::UploadStagingBuffer(size_t size) { return staging_pool.Request(size, MemoryUsage::Upload); } @@ -364,6 +459,10 @@ StagingBufferRef BufferCacheRuntime::DownloadStagingBuffer(size_t size, bool def return staging_pool.Request(size, MemoryUsage::Download, deferred); } +VkFormat BufferCacheRuntime::TexelBufferFormat(VideoCore::Surface::PixelFormat format) const { + return MaxwellToVK::SurfaceFormat(device, FormatType::Buffer, false, format).format; +} + void BufferCacheRuntime::FreeDeferredStagingBuffer(StagingBufferRef& ref) { staging_pool.FreeDeferred(ref); } @@ -376,6 +475,10 @@ u64 BufferCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 BufferCacheRuntime::GetDeviceAllocationUsage() const { + return device.GetMemoryBudgetInfo().allocation_bytes; +} + bool BufferCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -404,6 +507,16 @@ u64 BufferCacheRuntime::KnownGpuTick() { return scheduler.GetMasterSemaphore().KnownGpuTick(); } +u64 BufferCacheRuntime::CurrentSyncPoint() const noexcept { + return scheduler.GetMasterSemaphore().CurrentTick(); +} + +u64 BufferCacheRuntime::CompletedSyncPoint() const { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + return master_semaphore.KnownGpuTick(); +} + void BufferCacheRuntime::Wait(u64 buffer_tick) { scheduler.Wait(buffer_tick); } @@ -641,6 +754,7 @@ void BufferCacheRuntime::BindTransformFeedbackBuffer(u32 index, VkBuffer buffer, offset = 0; size = 0; } + scheduler.MarkTransformFeedbackUsed(); scheduler.Record([index, buffer, offset, size](vk::CommandBuffer cmdbuf) { const VkDeviceSize vk_offset = offset; const VkDeviceSize vk_size = size; @@ -653,19 +767,26 @@ void BufferCacheRuntime::BindTransformFeedbackBuffers(VideoCommon::HostBindings< // Already logged in the rasterizer return; } - boost::container::static_vector buffer_handles(bindings.buffers.size()); - for (u32 i = 0; i < bindings.buffers.size(); ++i) { + const u32 count = std::min(static_cast(bindings.buffers.size()), + VideoCommon::NUM_TRANSFORM_FEEDBACK_BUFFERS); + std::array handles{}; + std::array offsets{}; + std::array sizes{}; + for (u32 i = 0; i < count; ++i) { auto handle = bindings.buffers[i]->Handle(); if (handle == VK_NULL_HANDLE) { ReserveNullBuffer(); handle = *null_buffer; - bindings.offsets[i] = 0; - bindings.sizes[i] = 0; + } else { + offsets[i] = bindings.offsets[i]; + sizes[i] = bindings.sizes[i]; } - buffer_handles[i] = handle; + handles[i] = handle; } - scheduler.Record([bindings_ = std::move(bindings), buffer_handles_ = std::move(buffer_handles)](vk::CommandBuffer cmdbuf) { - cmdbuf.BindTransformFeedbackBuffersEXT(0, u32(buffer_handles_.size()), buffer_handles_.data(), bindings_.offsets.data(), bindings_.sizes.data()); + scheduler.MarkTransformFeedbackUsed(); + scheduler.Record([count, handles, offsets, sizes](vk::CommandBuffer cmdbuf) { + cmdbuf.BindTransformFeedbackBuffersEXT(0, count, handles.data(), offsets.data(), + sizes.data()); }); } @@ -690,6 +811,9 @@ vk::Buffer BufferCacheRuntime::CreateNullBuffer() { if (device.IsExtTransformFeedbackSupported()) { create_info.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + create_info.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } vk::Buffer ret = memory_allocator.CreateBuffer(create_info, MemoryUsage::DeviceLocal); if (device.HasDebuggingToolAttached()) { ret.SetObjectNameEXT("Null buffer"); diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index d4ad156073..4fc316850f 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include "video_core/buffer_cache/buffer_cache_base.h" #include "video_core/buffer_cache/memory_tracker_base.h" @@ -39,6 +41,10 @@ public: return *buffer; } + [[nodiscard]] VkDeviceAddress DeviceAddress() const noexcept { + return device_address; + } + [[nodiscard]] bool IsRegionUsed(u64 offset, u64 size) const noexcept { return tracker.IsUsed(offset, size); } @@ -70,6 +76,7 @@ private: vk::Buffer buffer; std::vector views; VideoCommon::UsageTracker tracker; + VkDeviceAddress device_address{}; u64 last_usage_tick{}; bool is_null{}; }; @@ -92,6 +99,31 @@ public: void TickFrame(Common::SlotVector& slot_buffers) noexcept; + void TryEnableUnifiedMemory(void* base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base); + + [[nodiscard]] bool HasUnifiedMemory() const noexcept { + return unified_memory != nullptr && unified_memory->IsValid(); + } + + [[nodiscard]] u64 UnifiedMemorySize() const noexcept { + return unified_memory ? unified_memory->GetSize() : 0; + } + + [[nodiscard]] u64 UnifiedMemoryBase() const noexcept { + return unified_memory ? unified_memory->GetBaseOffset() : 0; + } + + [[nodiscard]] u64 UnifiedMemoryWindowSize() const noexcept { + return unified_memory ? unified_memory->GetWindowSize() : 0; + } + + void CopyToUnifiedMemory(size_t window_index, VkBuffer src_buffer, + std::span copies); + + void UnifiedMemoryHostBarrier(); + u64 CurrentTick(); u64 KnownGpuTick(); @@ -100,10 +132,16 @@ public: void Finish(); + u64 CurrentSyncPoint() const noexcept; + + u64 CompletedSyncPoint() const; + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const; + bool CanReportMemoryUsage() const; u32 GetUniformBufferAlignment() const; @@ -145,22 +183,25 @@ public: [[maybe_unused]] u32 binding_index, u32 size) { const StagingBufferRef ref = staging_pool.Request(size, MemoryUsage::Upload); - BindBuffer(ref.buffer, static_cast(ref.offset), size); + guest_descriptor_queue.AddBuffer(ref.buffer, ref.device_address, + static_cast(ref.offset), size); return ref.mapped_span; } - void BindUniformBuffer(VkBuffer buffer, u32 offset, u32 size) { + void BindUniformBuffer(const Buffer& buffer, u32 offset, u32 size) { BindBuffer(buffer, offset, size); } - void BindStorageBuffer(VkBuffer buffer, u32 offset, u32 size, + void BindStorageBuffer(const Buffer& buffer, u32 offset, u32 size, [[maybe_unused]] bool is_written) { BindBuffer(buffer, offset, size); } void BindTextureBuffer(Buffer& buffer, u32 offset, u32 size, VideoCore::Surface::PixelFormat format) { - guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format)); + guest_descriptor_queue.AddTexelBuffer(buffer.View(offset, size, format), + buffer.DeviceAddress(), offset, size, + TexelBufferFormat(format)); } bool ShouldLimitDynamicStorageBuffers() const { @@ -172,14 +213,17 @@ public: } private: - void BindBuffer(VkBuffer buffer, u32 offset, u32 size) { - if (buffer == VK_NULL_HANDLE) { - guest_descriptor_queue.AddBuffer(buffer, 0, VK_WHOLE_SIZE); + void BindBuffer(const Buffer& buffer, u32 offset, u32 size) { + const VkBuffer handle = buffer.Handle(); + if (handle == VK_NULL_HANDLE) { + guest_descriptor_queue.AddBuffer(handle, 0, 0, VK_WHOLE_SIZE); } else { - guest_descriptor_queue.AddBuffer(buffer, offset, size); + guest_descriptor_queue.AddBuffer(handle, buffer.DeviceAddress(), offset, size); } } + VkFormat TexelBufferFormat(VideoCore::Surface::PixelFormat format) const; + void ReserveNullBuffer(); vk::Buffer CreateNullBuffer(); @@ -193,6 +237,7 @@ private: std::shared_ptr quad_strip_index_buffer; vk::Buffer null_buffer; + std::unique_ptr unified_memory; std::unique_ptr uint8_pass; QuadIndexedPass quad_index_pass; @@ -215,6 +260,7 @@ struct BufferCacheParams { static constexpr bool USE_MEMORY_MAPS = true; static constexpr bool SEPARATE_IMAGE_BUFFER_BINDINGS = false; static constexpr bool USE_MEMORY_MAPS_FOR_UPLOADS = true; + static constexpr bool USE_UNIFIED_MEMORY = true; }; using BufferCache = VideoCommon::BufferCache; diff --git a/src/video_core/renderer_vulkan/vk_command_pool.cpp b/src/video_core/renderer_vulkan/vk_command_pool.cpp index d0dbf7ca54..7ba35e0e20 100644 --- a/src/video_core/renderer_vulkan/vk_command_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_command_pool.cpp @@ -1,9 +1,13 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later #include #include "video_core/renderer_vulkan/vk_command_pool.h" +#include "video_core/renderer_vulkan/vk_master_semaphore.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -14,32 +18,52 @@ constexpr size_t COMMAND_BUFFER_POOL_SIZE = 4; struct CommandPool::Pool { vk::CommandPool handle; vk::CommandBuffers cmdbufs; + u64 tick; }; CommandPool::CommandPool(MasterSemaphore& master_semaphore_, const Device& device_) - : ResourcePool(master_semaphore_, COMMAND_BUFFER_POOL_SIZE), device{device_} {} + : master_semaphore{master_semaphore_}, device{device_} {} CommandPool::~CommandPool() = default; -void CommandPool::Allocate(size_t begin, size_t end) { - // Command buffers are going to be committed, recorded, executed every single usage cycle. - // They are also going to be reset when committed. +void CommandPool::AllocatePool() { Pool& pool = pools.emplace_back(); pool.handle = device.GetLogical().CreateCommandPool({ .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO, .pNext = nullptr, - .flags = - VK_COMMAND_POOL_CREATE_TRANSIENT_BIT | VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, + .flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT, .queueFamilyIndex = device.GetGraphicsFamily(), }); pool.cmdbufs = pool.handle.Allocate(COMMAND_BUFFER_POOL_SIZE); + pool.tick = 0; +} + +void CommandPool::AcquirePool() { + if (!pools.empty()) { + master_semaphore.Refresh(); + const u64 gpu_tick = master_semaphore.KnownGpuTick(); + for (size_t i = 0; i < pools.size(); ++i) { + const size_t candidate = (current_pool + 1 + i) % pools.size(); + if (gpu_tick >= pools[candidate].tick) { + current_pool = candidate; + current_index = 0; + pools[current_pool].handle.Reset(); + return; + } + } + } + AllocatePool(); + current_pool = pools.size() - 1; + current_index = 0; } VkCommandBuffer CommandPool::Commit() { - const size_t index = CommitResource(); - const auto pool_index = index / COMMAND_BUFFER_POOL_SIZE; - const auto sub_index = index % COMMAND_BUFFER_POOL_SIZE; - return pools[pool_index].cmdbufs[sub_index]; + if (pools.empty() || current_index >= COMMAND_BUFFER_POOL_SIZE) { + AcquirePool(); + } + Pool& pool = pools[current_pool]; + pool.tick = master_semaphore.CurrentTick(); + return pool.cmdbufs[current_index++]; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_command_pool.h b/src/video_core/renderer_vulkan/vk_command_pool.h index ec1647f012..362ec0610d 100644 --- a/src/video_core/renderer_vulkan/vk_command_pool.h +++ b/src/video_core/renderer_vulkan/vk_command_pool.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -6,7 +9,7 @@ #include #include -#include "video_core/renderer_vulkan/vk_resource_pool.h" +#include "common/common_types.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { @@ -14,20 +17,24 @@ namespace Vulkan { class Device; class MasterSemaphore; -class CommandPool final : public ResourcePool { +class CommandPool final { public: explicit CommandPool(MasterSemaphore& master_semaphore_, const Device& device_); - ~CommandPool() override; - - void Allocate(size_t begin, size_t end) override; + ~CommandPool(); VkCommandBuffer Commit(); private: struct Pool; + void AllocatePool(); + void AcquirePool(); + + MasterSemaphore& master_semaphore; const Device& device; std::vector pools; + size_t current_pool = 0; + size_t current_index = 0; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index 80c434dc8f..2e65161868 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -5,10 +5,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include #include #include +#include #include "video_core/renderer_vulkan/vk_texture_cache.h" @@ -22,7 +24,9 @@ #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" #include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h" #include "video_core/host_shaders/vulkan_uint8_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_2d_buffer_comp_spv.h" #include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_3d_buffer_comp_spv.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" #include "video_core/surface.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" @@ -31,6 +35,7 @@ #include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/texture_cache/accelerated_swizzle.h" #include "video_core/texture_cache/types.h" +#include "video_core/texture_cache/util.h" #include "video_core/textures/decoders.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -268,7 +273,7 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor .layout = *layout, .basePipelineHandle = {}, .basePipelineIndex = 0, - }); + }, device.StaticPipelineCache()); } ComputePass::~ComputePass() = default; @@ -570,7 +575,7 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; // To unswizzle the ASTC data - const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(swizzle, image.info); ASSERT(params.origin == (std::array{0, 0, 0})); ASSERT(params.destination == (std::array{0, 0, 0})); ASSERT(params.bytes_per_block_log2 == 4); @@ -713,7 +718,7 @@ void BlockLinearUnswizzle3DPass::Unswizzle( ASSERT(swizzles.size() == 1); const auto& sw = swizzles[0]; - const auto params = MakeBlockLinearSwizzle3DParams(sw, image.info); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info); const u32 blocks_x = (image.info.size.width + 3) / 4; const u32 blocks_y = (image.info.size.height + 3) / 4; @@ -872,4 +877,626 @@ void BlockLinearUnswizzle3DPass::UnswizzleChunk( }); } +namespace { +constexpr u32 BL2D_BINDING_INPUT_BUFFER = 0; +constexpr u32 BL2D_BINDING_OUTPUT_BUFFER = 1; + +struct alignas(16) BlockLinearUnswizzle2DPushConstants { + std::array dim; + u32 bytes_per_block_log2; + std::array origin; + u32 layer_stride; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; +}; +static_assert(sizeof(BlockLinearUnswizzle2DPushConstants) <= 128); + +constexpr std::array BL2D_BINDINGS{{ + { + .binding = BL2D_BINDING_INPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = BL2D_BINDING_OUTPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, +}}; + +constexpr std::array BL2D_TEMPLATE{{ + { + .dstBinding = BL2D_BINDING_INPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL2D_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = BL2D_BINDING_OUTPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL2D_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, +}}; + +constexpr DescriptorBankInfo BL2D_BANK_INFO{ + .uniform_buffers = 0, + .storage_buffers = 2, + .texture_buffers = 0, + .image_buffers = 0, + .textures = 0, + .images = 0, + .score = 2, +}; + +constexpr bool BL2D_VERIFY_AGAINST_CPU = false; +} // Anonymous namespace + +BlockLinearUnswizzle2DPass::BlockLinearUnswizzle2DPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, BL2D_BINDINGS, BL2D_TEMPLATE, + BL2D_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_2D_BUFFER_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle2DPass::~BlockLinearUnswizzle2DPass() = default; + +bool BlockLinearUnswizzle2DPass::IsSupported(const VideoCommon::ImageInfo& info) { + if (info.type != VideoCommon::ImageType::e2D) { + return false; + } + if (info.resources.levels != 1 || info.resources.layers != 1) { + return false; + } + if (info.num_samples > 1) { + return false; + } + if (VideoCore::Surface::IsPixelFormatASTC(info.format) || + VideoCore::Surface::IsPixelFormatBCn(info.format)) { + return false; + } + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format); + if (bytes_per_block != 4 && bytes_per_block != 8 && bytes_per_block != 16) { + return false; + } + return VideoCore::Surface::DefaultBlockWidth(info.format) == 1 && + VideoCore::Surface::DefaultBlockHeight(info.format) == 1; +} + +void BlockLinearUnswizzle2DPass::Unswizzle( + Image& image, const StagingBufferRef& swizzled, + std::span swizzles) { + using namespace VideoCommon::Accelerated; + + if (swizzles.empty()) { + return; + } + const VideoCommon::SwizzleParameters& sw = swizzles.front(); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle2DParams(sw, image.info); + + const u32 width = sw.num_tiles.width; + const u32 height = sw.num_tiles.height; + const u32 depth = image.info.resources.layers; + const u32 bytes_per_block = 1u << params.bytes_per_block_log2; + const VkDeviceSize output_size = + static_cast(width) * height * depth * bytes_per_block; + + const StagingBufferRef output = staging_buffer_pool.Request( + static_cast(output_size), MemoryUsage::DeviceLocal); + + BlockLinearUnswizzle2DPushConstants pc{}; + pc.dim = {width, height, depth}; + pc.bytes_per_block_log2 = params.bytes_per_block_log2; + pc.origin = params.origin; + pc.layer_stride = params.layer_stride; + pc.block_size = params.block_size; + pc.x_shift = params.x_shift; + pc.block_height = params.block_height; + pc.block_height_mask = params.block_height_mask; + + scheduler.RequestOutsideRenderPassOperationContext(); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, + sw.buffer_offset + swizzled.offset, + image.guest_size_bytes - sw.buffer_offset); + compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size); + + const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); + const VkDescriptorSet set = descriptor_allocator.Commit(); + + const u32 gx = Common::DivCeil(width, 16u); + const u32 gy = Common::DivCeil(height, 8u); + const bool is_initialized = image.ExchangeInitialization(); + + const VkBuffer out_buffer = output.buffer; + const VkDeviceSize out_offset = output.offset; + const VkImage dst_image = image.Handle(); + const VkImageAspectFlags aspect = image.AspectMask(); + + scheduler.Record([this, set, descriptor_data, pc, gx, gy, depth, output_size, out_buffer, + out_offset, dst_image, aspect, width, height, + is_initialized](vk::CommandBuffer cmdbuf) { + if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { + return; + } + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); + cmdbuf.Dispatch(gx, gy, depth); + + const VkBufferMemoryBarrier buffer_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = out_offset, + .size = output_size, + }; + const VkImageMemoryBarrier pre_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast( + is_initialized ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + (is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE + : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy); + + const VkBufferImageCopy copy{ + .bufferOffset = out_offset, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource{ + .aspectMask = aspect, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = depth, + }, + .imageOffset = {0, 0, 0}, + .imageExtent = {width, height, 1}, + }; + cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); + + const VkImageMemoryBarrier post_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, post_copy); + }); + + if constexpr (BL2D_VERIFY_AGAINST_CPU) { + VerifyAgainstCpu(swizzled, sw, output, output_size, width, height, depth, + bytes_per_block); + } +} + +void BlockLinearUnswizzle2DPass::VerifyAgainstCpu(const StagingBufferRef& swizzled, + const VideoCommon::SwizzleParameters& sw, + const StagingBufferRef& gpu_output, + VkDeviceSize output_size, u32 width, u32 height, + u32 depth, u32 bytes_per_block) { + const StagingBufferRef readback = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::Download); + + const VkBuffer src = gpu_output.buffer; + const VkDeviceSize src_offset = gpu_output.offset; + const VkBuffer dst = readback.buffer; + const VkDeviceSize dst_offset = readback.offset; + scheduler.Record([src, src_offset, dst, dst_offset, output_size](vk::CommandBuffer cmdbuf) { + const VkBufferMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = src, + .offset = src_offset, + .size = output_size, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, {}, barrier, {}); + const VkBufferCopy copy{ + .srcOffset = src_offset, + .dstOffset = dst_offset, + .size = output_size, + }; + cmdbuf.CopyBuffer(src, dst, copy); + }); + scheduler.Finish(); + + const size_t size = static_cast(output_size); + std::vector reference(size); + const std::span input{swizzled.mapped_span.data() + sw.buffer_offset, + swizzled.mapped_span.size() - sw.buffer_offset}; + Tegra::Texture::UnswizzleTexture(reference, input, bytes_per_block, width, height, depth, + sw.block.height, sw.block.depth); + + const u8* gpu_data = readback.mapped_span.data(); + if (std::memcmp(reference.data(), gpu_data, size) == 0) { + LOG_INFO(Render_Vulkan, "BL2D verify OK: {}x{}x{} bpb={} ({} bytes)", width, height, depth, + bytes_per_block, size); + return; + } + size_t first_diff = size; + size_t num_diff = 0; + for (size_t i = 0; i < size; ++i) { + if (reference[i] != gpu_data[i]) { + if (first_diff == size) { + first_diff = i; + } + ++num_diff; + } + } + LOG_CRITICAL(Render_Vulkan, + "BL2D verify FAILED: {}x{}x{} bpb={} block_height={} first_diff={} " + "num_diff={}/{} cpu=0x{:02x} gpu=0x{:02x}", + width, height, depth, bytes_per_block, sw.block.height, first_diff, + num_diff, size, reference[first_diff], gpu_data[first_diff]); +} + +namespace { +constexpr u32 BL3DB_BINDING_INPUT_BUFFER = 0; +constexpr u32 BL3DB_BINDING_OUTPUT_BUFFER = 1; + +struct alignas(16) BlockLinearUnswizzle3DBufferPushConstants { + std::array dim; + u32 bytes_per_block_log2; + std::array origin; + u32 slice_size; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; + u32 block_depth; + u32 block_depth_mask; +}; +static_assert(sizeof(BlockLinearUnswizzle3DBufferPushConstants) <= 128); + +constexpr std::array BL3DB_BINDINGS{{ + { + .binding = BL3DB_BINDING_INPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, + { + .binding = BL3DB_BINDING_OUTPUT_BUFFER, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .descriptorCount = 1, + .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, + .pImmutableSamplers = nullptr, + }, +}}; + +constexpr std::array BL3DB_TEMPLATE{{ + { + .dstBinding = BL3DB_BINDING_INPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL3DB_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, + { + .dstBinding = BL3DB_BINDING_OUTPUT_BUFFER, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, + .offset = BL3DB_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry), + .stride = sizeof(DescriptorUpdateEntry), + }, +}}; + +constexpr DescriptorBankInfo BL3DB_BANK_INFO{ + .uniform_buffers = 0, + .storage_buffers = 2, + .texture_buffers = 0, + .image_buffers = 0, + .textures = 0, + .images = 0, + .score = 2, +}; + +constexpr bool BL3DB_VERIFY_AGAINST_CPU = false; +} // Anonymous namespace + +BlockLinearUnswizzle3DBufferPass::BlockLinearUnswizzle3DBufferPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, BL3DB_BINDINGS, BL3DB_TEMPLATE, + BL3DB_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_3D_BUFFER_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +BlockLinearUnswizzle3DBufferPass::~BlockLinearUnswizzle3DBufferPass() = default; + +bool BlockLinearUnswizzle3DBufferPass::IsSupported(const Device& device, + const VideoCommon::ImageInfo& info) { + if (info.type != VideoCommon::ImageType::e3D) { + return false; + } + if (info.resources.levels != 1 || info.resources.layers != 1) { + return false; + } + if (info.num_samples > 1) { + return false; + } + if (info.size.depth <= 1) { + return false; + } + if (VideoCore::Surface::IsPixelFormatASTC(info.format)) { + return false; + } + if (VideoCore::Surface::IsPixelFormatBCn(info.format) && !device.IsOptimalBcnSupported()) { + return false; + } + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(info.format); + return bytes_per_block == 4 || bytes_per_block == 8 || bytes_per_block == 16; +} + +void BlockLinearUnswizzle3DBufferPass::Unswizzle( + Image& image, const StagingBufferRef& swizzled, + std::span swizzles) { + if (swizzles.empty()) { + return; + } + + const VideoCommon::SwizzleParameters& sw = swizzles.front(); + const auto params = VideoCommon::Accelerated::MakeBlockLinearSwizzle3DParams(sw, image.info); + + const u32 blocks_x = sw.num_tiles.width; + const u32 blocks_y = sw.num_tiles.height; + const u32 blocks_z = sw.num_tiles.depth; + const u32 bytes_per_block = 1u << params.bytes_per_block_log2; + const VkDeviceSize output_size = + static_cast(blocks_x) * blocks_y * blocks_z * bytes_per_block; + + const StagingBufferRef output = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::DeviceLocal); + + BlockLinearUnswizzle3DBufferPushConstants pc{}; + pc.dim = {blocks_x, blocks_y, blocks_z}; + pc.bytes_per_block_log2 = params.bytes_per_block_log2; + pc.origin = params.origin; + pc.slice_size = params.slice_size; + pc.block_size = params.block_size; + pc.x_shift = params.x_shift; + pc.block_height = params.block_height; + pc.block_height_mask = params.block_height_mask; + pc.block_depth = params.block_depth; + pc.block_depth_mask = params.block_depth_mask; + + scheduler.RequestOutsideRenderPassOperationContext(); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, sw.buffer_offset + swizzled.offset, + image.guest_size_bytes - sw.buffer_offset); + compute_pass_descriptor_queue.AddBuffer(output.buffer, output.offset, output_size); + + const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); + const VkDescriptorSet set = descriptor_allocator.Commit(); + + const u32 gx = Common::DivCeil(blocks_x, 8u); + const u32 gy = Common::DivCeil(blocks_y, 8u); + const u32 gz = Common::DivCeil(blocks_z, 4u); + const bool is_initialized = image.ExchangeInitialization(); + + const VkBuffer out_buffer = output.buffer; + const VkDeviceSize out_offset = output.offset; + const VkImage dst_image = image.Handle(); + const VkImageAspectFlags aspect = image.AspectMask(); + const VkExtent3D extent{ + .width = image.info.size.width, + .height = image.info.size.height, + .depth = image.info.size.depth, + }; + + scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, output_size, out_buffer, + out_offset, dst_image, aspect, extent, + is_initialized](vk::CommandBuffer cmdbuf) { + if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { + return; + } + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); + cmdbuf.Dispatch(gx, gy, gz); + + const VkBufferMemoryBarrier buffer_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = out_buffer, + .offset = out_offset, + .size = output_size, + }; + const VkImageMemoryBarrier pre_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = static_cast( + is_initialized ? VK_ACCESS_SHADER_READ_BIT : VK_ACCESS_NONE), + .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | + (is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE + : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), + VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, buffer_barrier, pre_copy); + + const VkBufferImageCopy copy{ + .bufferOffset = out_offset, + .bufferRowLength = 0, + .bufferImageHeight = 0, + .imageSubresource{ + .aspectMask = aspect, + .mipLevel = 0, + .baseArrayLayer = 0, + .layerCount = 1, + }, + .imageOffset = {0, 0, 0}, + .imageExtent = extent, + }; + cmdbuf.CopyBufferToImage(out_buffer, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); + + const VkImageMemoryBarrier post_copy{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, + vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, {}, {}, post_copy); + }); + + if constexpr (BL3DB_VERIFY_AGAINST_CPU) { + VerifyAgainstCpu(swizzled, sw, image.info, output, output_size, blocks_x, blocks_y, + blocks_z, bytes_per_block); + } +} + +void BlockLinearUnswizzle3DBufferPass::VerifyAgainstCpu( + const StagingBufferRef& swizzled, const VideoCommon::SwizzleParameters& sw, + const VideoCommon::ImageInfo& info, const StagingBufferRef& gpu_output, + VkDeviceSize output_size, u32 blocks_x, u32 blocks_y, u32 blocks_z, u32 bytes_per_block) { + const StagingBufferRef readback = + staging_buffer_pool.Request(static_cast(output_size), MemoryUsage::Download); + + const VkBuffer src = gpu_output.buffer; + const VkDeviceSize src_offset = gpu_output.offset; + const VkBuffer dst = readback.buffer; + const VkDeviceSize dst_offset = readback.offset; + scheduler.Record([src, src_offset, dst, dst_offset, output_size](vk::CommandBuffer cmdbuf) { + const VkBufferMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .buffer = src, + .offset = src_offset, + .size = output_size, + }; + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, + 0, {}, barrier, {}); + const VkBufferCopy copy{ + .srcOffset = src_offset, + .dstOffset = dst_offset, + .size = output_size, + }; + cmdbuf.CopyBuffer(src, dst, copy); + }); + scheduler.Finish(); + + const size_t size = static_cast(output_size); + std::vector reference(size); + const std::span input{swizzled.mapped_span.data() + sw.buffer_offset, + swizzled.mapped_span.size() - sw.buffer_offset}; + const u32 stride_alignment = VideoCommon::CalculateLevelStrideAlignment(info, sw.level); + Tegra::Texture::UnswizzleTexture(reference, input, bytes_per_block, blocks_x, blocks_y, blocks_z, + sw.block.height, sw.block.depth, stride_alignment); + + const u8* gpu_data = readback.mapped_span.data(); + if (std::memcmp(reference.data(), gpu_data, size) == 0) { + LOG_INFO(Render_Vulkan, "BL3D verify OK: {}x{}x{} bpb={} ({} bytes)", blocks_x, blocks_y, + blocks_z, bytes_per_block, size); + return; + } + size_t first_diff = size; + size_t num_diff = 0; + for (size_t i = 0; i < size; ++i) { + if (reference[i] != gpu_data[i]) { + if (first_diff == size) { + first_diff = i; + } + ++num_diff; + } + } + LOG_CRITICAL(Render_Vulkan, + "BL3D verify FAILED: {}x{}x{} bpb={} block_height={} block_depth={} " + "first_diff={} num_diff={}/{} cpu=0x{:02x} gpu=0x{:02x}", + blocks_x, blocks_y, blocks_z, bytes_per_block, sw.block.height, sw.block.depth, + first_diff, num_diff, size, reference[first_diff], gpu_data[first_diff]); +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 3d30aa6bdc..70a02c8d8a 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -164,4 +164,54 @@ private: ComputePassDescriptorQueue& compute_pass_descriptor_queue; }; +class BlockLinearUnswizzle2DPass final : public ComputePass { +public: + explicit BlockLinearUnswizzle2DPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle2DPass(); + + [[nodiscard]] static bool IsSupported(const VideoCommon::ImageInfo& info); + + void Unswizzle(Image& image, const StagingBufferRef& swizzled, + std::span swizzles); + +private: + void VerifyAgainstCpu(const StagingBufferRef& swizzled, + const VideoCommon::SwizzleParameters& sw, + const StagingBufferRef& gpu_output, VkDeviceSize output_size, u32 width, + u32 height, u32 depth, u32 bytes_per_block); + + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzle3DBufferPass final : public ComputePass { +public: + explicit BlockLinearUnswizzle3DBufferPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzle3DBufferPass(); + + [[nodiscard]] static bool IsSupported(const Device& device, + const VideoCommon::ImageInfo& info); + + void Unswizzle(Image& image, const StagingBufferRef& swizzled, + std::span swizzles); + +private: + void VerifyAgainstCpu(const StagingBufferRef& swizzled, + const VideoCommon::SwizzleParameters& sw, + const VideoCommon::ImageInfo& info, + const StagingBufferRef& gpu_output, VkDeviceSize output_size, + u32 blocks_x, u32 blocks_y, u32 blocks_z, u32 bytes_per_block); + + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 006bfc0a09..545c19b273 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -34,12 +34,14 @@ using Tegra::Texture::TexturePair; ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk::PipelineCache& pipeline_cache_, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue_, + DescriptorBufferRing& descriptor_buffer_ring_, Common::ThreadWorker* thread_worker, PipelineStatistics* pipeline_statistics, VideoCore::ShaderNotify* shader_notify, const Shader::Info& info_, vk::ShaderModule spv_module_, u64 shader_hash_) : device{device_}, - pipeline_cache(pipeline_cache_), guest_descriptor_queue{guest_descriptor_queue_}, info{info_}, + pipeline_cache(pipeline_cache_), guest_descriptor_queue{guest_descriptor_queue_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, info{info_}, shader_hash{shader_hash_}, spv_module(std::move(spv_module_)) { if (shader_notify) { shader_notify->MarkShaderBuilding(); @@ -48,18 +50,35 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk uniform_buffer_sizes.begin()); num_descriptor_entries = NumDescriptorEntries(info); - auto func{[this, &scheduler, &descriptor_pool, shader_notify, pipeline_statistics] { - DescriptorLayoutBuilder builder{device}; - builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT); + DescriptorLayoutBuilder builder{device}; + builder.Add(info, VK_SHADER_STAGE_COMPUTE_BIT); - uses_push_descriptor = builder.CanUsePushDescriptor(); - descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor); - pipeline_layout = builder.CreatePipelineLayout(*descriptor_set_layout); + uses_push_descriptor = builder.CanUsePushDescriptor(); + uses_descriptor_buffer = builder.CanUseDescriptorBuffer() && descriptor_buffer_ring.IsValid(); + descriptor_set_layout = + builder.CreateDescriptorSetLayout(uses_push_descriptor, uses_descriptor_buffer); + if (uses_descriptor_buffer) { + descriptor_buffer_layout = builder.MakeDescriptorBufferLayout(*descriptor_set_layout); + if (descriptor_buffer_layout.size > DescriptorBufferRing::MaxAllocationSize()) { + LOG_WARNING(Render_Vulkan, + "Compute shader {:016X} needs {} descriptor bytes, falling back to sets", + shader_hash, descriptor_buffer_layout.size); + uses_descriptor_buffer = false; + descriptor_buffer_layout = {}; + descriptor_set_layout = builder.CreateDescriptorSetLayout(false); + } + } + pipeline_layout = builder.CreatePipelineLayout(*descriptor_set_layout); + if (!uses_descriptor_buffer) { descriptor_update_template = builder.CreateTemplate(*descriptor_set_layout, *pipeline_layout, uses_push_descriptor); if (!uses_push_descriptor) { - descriptor_allocator = descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, info); + descriptor_allocator = + descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, info); } + } + + auto func{[this, shader_notify, pipeline_statistics] { const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT subgroup_size_ci{ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT, .pNext = nullptr, @@ -69,9 +88,20 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) { flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR; } + if (uses_descriptor_buffer) { + flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + } + VkPipelineCreationFeedback creation_feedback{}; + const VkPipelineCreationFeedbackCreateInfo feedback_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO, + .pNext = nullptr, + .pPipelineCreationFeedback = &creation_feedback, + .pipelineStageCreationFeedbackCount = 0, + .pPipelineStageCreationFeedbacks = nullptr, + }; const VkComputePipelineCreateInfo compute_ci{ .sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = device.IsExtPipelineCreationFeedbackSupported() ? &feedback_ci : nullptr, .flags = flags, .stage{ .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, @@ -101,6 +131,14 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk return; } + if ((creation_feedback.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) != 0) { + const bool cache_hit = + (creation_feedback.flags & + VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; + LOG_DEBUG(Render_Vulkan, "Compute pipeline {:016X} cache_hit={} duration={}us", + shader_hash, cache_hit, creation_feedback.duration / 1000); + } + // Log compute pipeline creation if (GPU::Logging::IsActive()) { GPU::Logging::GPULogger::GetInstance().LogPipelineStateChange( @@ -128,7 +166,7 @@ ComputePipeline::ComputePipeline(const Device& device_, Scheduler& scheduler, vk void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, Tegra::MemoryManager& gpu_memory, Scheduler& scheduler, BufferCache& buffer_cache, TextureCache& texture_cache) { - guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries); + guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); buffer_cache.SetComputeUniformBufferState(info.constant_buffer_mask, &uniform_buffer_sizes); buffer_cache.UnbindComputeStorageBuffers(); @@ -249,9 +287,22 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, GPU::Logging::GPULogger::GetInstance().LogPipelineBind(true, "compute pipeline"); } - const void* const descriptor_data{guest_descriptor_queue.UpdateData()}; + const DescriptorUpdateEntry* const descriptor_data{guest_descriptor_queue.UpdateData()}; + VkDeviceSize descriptor_buffer_offset{}; + bool descriptor_buffer_ready{false}; + if (uses_descriptor_buffer) { + const DescriptorBufferRing::Allocation alloc{ + descriptor_buffer_ring.Allocate(scheduler, descriptor_buffer_layout.size)}; + if (alloc.host) { + WriteDescriptorBuffer(device, descriptor_buffer_layout, descriptor_data, alloc.host); + descriptor_buffer_offset = alloc.offset; + descriptor_buffer_ready = true; + } + } + const bool is_rescaling = !info.texture_descriptors.empty() || !info.image_descriptors.empty(); - scheduler.Record([this, descriptor_data, is_rescaling, + scheduler.Record([this, descriptor_data, is_rescaling, descriptor_buffer_offset, + descriptor_buffer_ready, rescaling_data = rescaling.Data()](vk::CommandBuffer cmdbuf) { if (!pipeline) { return; @@ -265,7 +316,17 @@ void ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, RESCALING_LAYOUT_WORDS_OFFSET, sizeof(rescaling_data), rescaling_data.data()); } - if (uses_push_descriptor) { + if (uses_descriptor_buffer) { + if (!descriptor_buffer_ready) { + return; + } + const VkDescriptorBufferBindingInfoEXT binding_info{ + descriptor_buffer_ring.BindingInfo()}; + cmdbuf.BindDescriptorBuffersEXT(binding_info); + const u32 buffer_index{}; + cmdbuf.SetDescriptorBufferOffsetsEXT(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline_layout, + 0, buffer_index, descriptor_buffer_offset); + } else if (uses_push_descriptor) { cmdbuf.PushDescriptorSetWithTemplateKHR(*descriptor_update_template, *pipeline_layout, 0, descriptor_data); } else { diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.h b/src/video_core/renderer_vulkan/vk_compute_pipeline.h index 073e4b5079..56c4707e69 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.h @@ -13,7 +13,9 @@ #include "common/common_types.h" #include "common/thread_worker.h" #include "shader_recompiler/shader_info.h" +#include "video_core/renderer_vulkan/pipeline_helper.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" @@ -34,6 +36,7 @@ public: explicit ComputePipeline(const Device& device, Scheduler& scheduler, vk::PipelineCache& pipeline_cache, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue, + DescriptorBufferRing& descriptor_buffer_ring, Common::ThreadWorker* thread_worker, PipelineStatistics* pipeline_statistics, VideoCore::ShaderNotify* shader_notify, const Shader::Info& info, @@ -56,6 +59,7 @@ private: const Device& device; vk::PipelineCache& pipeline_cache; GuestDescriptorQueue& guest_descriptor_queue; + DescriptorBufferRing& descriptor_buffer_ring; Shader::Info info; u64 shader_hash{}; u32 num_descriptor_entries{}; @@ -65,6 +69,8 @@ private: vk::ShaderModule spv_module; vk::DescriptorSetLayout descriptor_set_layout; bool uses_push_descriptor{false}; + bool uses_descriptor_buffer{false}; + DescriptorBufferLayout descriptor_buffer_layout; DescriptorAllocator descriptor_allocator; vk::PipelineLayout pipeline_layout; vk::DescriptorUpdateTemplate descriptor_update_template; diff --git a/src/video_core/renderer_vulkan/vk_descriptor_buffer.cpp b/src/video_core/renderer_vulkan/vk_descriptor_buffer.cpp new file mode 100644 index 0000000000..60047faec0 --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_descriptor_buffer.cpp @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include "common/alignment.h" +#include "common/assert.h" +#include "common/logging.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" +#include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/vulkan_common/vulkan_device.h" + +namespace Vulkan { + +DescriptorBufferRing::DescriptorBufferRing(const Device& device_, + MemoryAllocator& memory_allocator) + : device{device_} { + if (!device.IsExtDescriptorBufferSupported() || !device.IsBufferDeviceAddressSupported()) { + return; + } + alignment = std::max( + device.DescriptorBufferProperties().descriptorBufferOffsetAlignment, 1); + const VkDeviceSize total = FRAME_SIZE * FRAMES_IN_FLIGHT + alignment; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = total, + .usage = VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + buffer = memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::Upload); + if (!buffer.IsHostVisible()) { + LOG_WARNING(Render_Vulkan, "Descriptor buffer is not host visible, disabling"); + buffer.reset(); + return; + } + if (device.HasDebuggingToolAttached()) { + buffer.SetObjectNameEXT("Descriptor buffer"); + } + const VkDeviceAddress raw_address = device.GetLogical().GetBufferDeviceAddress(*buffer); + base_address = Common::AlignUp(raw_address, alignment); + base_host = buffer.Mapped().data() + (base_address - raw_address); +} + +DescriptorBufferRing::~DescriptorBufferRing() = default; + +void DescriptorBufferRing::TickFrame() { + if (++frame_index >= FRAMES_IN_FLIGHT) { + frame_index = 0; + } + frame_start = static_cast(frame_index) * FRAME_SIZE; + cursor = 0; + frame_reused = true; +} + +DescriptorBufferRing::Allocation DescriptorBufferRing::Allocate(Scheduler& scheduler, + VkDeviceSize size) { + ASSERT(buffer); + const VkDeviceSize needed = Common::AlignUp(size, alignment); + if (needed > FRAME_SIZE) { + LOG_ERROR(Render_Vulkan, "Descriptor set of {} bytes exceeds frame capacity {}", needed, + FRAME_SIZE); + return Allocation{}; + } + if (frame_reused) { + frame_reused = false; + scheduler.Wait(frame_ticks[frame_index]); + } + if (cursor + needed > FRAME_SIZE) { + LOG_WARNING(Render_Vulkan, "Descriptor buffer frame exhausted, stalling on the GPU"); + scheduler.Finish(); + cursor = 0; + } + const VkDeviceSize offset = frame_start + cursor; + cursor += needed; + frame_ticks[frame_index] = scheduler.CurrentTick(); + return Allocation{ + .host = base_host + offset, + .offset = offset, + }; +} + +VkDescriptorBufferBindingInfoEXT DescriptorBufferRing::BindingInfo() const noexcept { + return VkDescriptorBufferBindingInfoEXT{ + .sType = VK_STRUCTURE_TYPE_DESCRIPTOR_BUFFER_BINDING_INFO_EXT, + .pNext = nullptr, + .address = base_address, + .usage = VK_BUFFER_USAGE_RESOURCE_DESCRIPTOR_BUFFER_BIT_EXT | + VK_BUFFER_USAGE_SAMPLER_DESCRIPTOR_BUFFER_BIT_EXT, + }; +} + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_descriptor_buffer.h b/src/video_core/renderer_vulkan/vk_descriptor_buffer.h new file mode 100644 index 0000000000..25e0d1380f --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_descriptor_buffer.h @@ -0,0 +1,57 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include "common/common_types.h" +#include "video_core/vulkan_common/vulkan_memory_allocator.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +namespace Vulkan { + +class Device; +class Scheduler; + +class DescriptorBufferRing final { + static constexpr size_t FRAMES_IN_FLIGHT = 8; + static constexpr VkDeviceSize FRAME_SIZE = 512 * 1024; + +public: + explicit DescriptorBufferRing(const Device& device_, MemoryAllocator& memory_allocator); + ~DescriptorBufferRing(); + + struct Allocation { + u8* host{}; + VkDeviceSize offset{}; + }; + + [[nodiscard]] static constexpr VkDeviceSize MaxAllocationSize() noexcept { + return FRAME_SIZE; + } + + void TickFrame(); + + [[nodiscard]] Allocation Allocate(Scheduler& scheduler, VkDeviceSize size); + + [[nodiscard]] VkDescriptorBufferBindingInfoEXT BindingInfo() const noexcept; + + [[nodiscard]] bool IsValid() const noexcept { + return static_cast(buffer); + } + +private: + const Device& device; + vk::Buffer buffer; + VkDeviceAddress base_address{}; + u8* base_host{}; + VkDeviceSize alignment{1}; + size_t frame_index{}; + VkDeviceSize frame_start{}; + VkDeviceSize cursor{}; + std::array frame_ticks{}; + bool frame_reused{}; +}; + +} // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_fence_manager.cpp b/src/video_core/renderer_vulkan/vk_fence_manager.cpp index fad9e38324..bdda8582e9 100644 --- a/src/video_core/renderer_vulkan/vk_fence_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_fence_manager.cpp @@ -37,7 +37,7 @@ void InnerFence::Wait() { if (is_stubbed) { return; } - scheduler.Wait(wait_tick); + scheduler.WaitSubmitted(wait_tick); } FenceManager::FenceManager(VideoCore::RasterizerInterface& rasterizer_, Tegra::GPU& gpu_, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 757e0f3ccf..3db07ad5e1 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -5,6 +5,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include @@ -250,13 +251,15 @@ GraphicsPipeline::GraphicsPipeline( Scheduler& scheduler_, BufferCache& buffer_cache_, TextureCache& texture_cache_, vk::PipelineCache& pipeline_cache_, VideoCore::ShaderNotify* shader_notify, const Device& device_, DescriptorPool& descriptor_pool, - GuestDescriptorQueue& guest_descriptor_queue_, Common::ThreadWorker* worker_thread, + GuestDescriptorQueue& guest_descriptor_queue_, DescriptorBufferRing& descriptor_buffer_ring_, + Common::ThreadWorker* worker_thread, PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache, const GraphicsPipelineCacheKey& key_, std::array stages, const std::array& infos) : key{key_}, device{device_}, texture_cache{texture_cache_}, buffer_cache{buffer_cache_}, pipeline_cache(pipeline_cache_), scheduler{scheduler_}, - guest_descriptor_queue{guest_descriptor_queue_}, spv_modules{std::move(stages)} { + guest_descriptor_queue{guest_descriptor_queue_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, spv_modules{std::move(stages)} { if (shader_notify) { shader_notify->MarkShaderBuilding(); } @@ -276,21 +279,40 @@ GraphicsPipeline::GraphicsPipeline( num_descriptor_entries += NumDescriptorEntries(*info); } fragment_has_color0_output = stage_infos[NUM_STAGES - 1].stores_frag_color[0]; - auto func{[this, shader_notify, &render_pass_cache, &descriptor_pool, pipeline_statistics] { - DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)}; - uses_push_descriptor = builder.CanUsePushDescriptor(); - descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor); - if (!uses_push_descriptor) { - descriptor_allocator = descriptor_pool.Allocator(device, scheduler, *descriptor_set_layout, stage_infos); + DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)}; + uses_push_descriptor = builder.CanUsePushDescriptor(); + uses_descriptor_buffer = builder.CanUseDescriptorBuffer() && descriptor_buffer_ring.IsValid(); + descriptor_set_layout = + builder.CreateDescriptorSetLayout(uses_push_descriptor, uses_descriptor_buffer); + if (uses_descriptor_buffer) { + descriptor_buffer_layout = builder.MakeDescriptorBufferLayout(*descriptor_set_layout); + if (descriptor_buffer_layout.size > DescriptorBufferRing::MaxAllocationSize()) { + LOG_WARNING(Render_Vulkan, + "Graphics pipeline {:016X} needs {} descriptor bytes, falling back to sets", + key.Hash(), descriptor_buffer_layout.size); + uses_descriptor_buffer = false; + descriptor_buffer_layout = {}; + descriptor_set_layout = builder.CreateDescriptorSetLayout(uses_push_descriptor); } + } - const VkDescriptorSetLayout set_layout{*descriptor_set_layout}; - pipeline_layout = builder.CreatePipelineLayout(set_layout); + const VkDescriptorSetLayout set_layout{*descriptor_set_layout}; + pipeline_layout = builder.CreatePipelineLayout(set_layout); + if (!uses_descriptor_buffer) { descriptor_update_template = builder.CreateTemplate(set_layout, *pipeline_layout, uses_push_descriptor); + if (!uses_push_descriptor) { + descriptor_allocator = + descriptor_pool.Allocator(device, scheduler, set_layout, stage_infos); + } + } - const VkRenderPass render_pass{render_pass_cache.Get(MakeRenderPassKey(key.state, device))}; + auto func{[this, shader_notify, &render_pass_cache, pipeline_statistics] { + VkRenderPass render_pass{}; + if (!device.IsKhrDynamicRenderingSupported()) { + render_pass = render_pass_cache.Get(MakeRenderPassKey(key.state, device)); + } Validate(); try { MakePipeline(render_pass); @@ -496,7 +518,7 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { buffer_cache.UpdateGraphicsBuffers(is_indexed); buffer_cache.BindHostGeometryBuffers(is_indexed); - guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries); + guest_descriptor_queue.Acquire(scheduler, num_descriptor_entries, uses_descriptor_buffer); RescalingPushConstant rescaling; RenderAreaPushConstant render_area; @@ -565,8 +587,33 @@ void GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, } const void* const descriptor_data{guest_descriptor_queue.UpdateData()}; - scheduler.Record([this, descriptor_data, bind_pipeline, rescaling_data = rescaling.Data(), - is_rescaling, update_rescaling, + bool update_descriptors = true; + if (descriptor_set_layout && !uses_push_descriptor && !uses_descriptor_buffer) { + const auto* const entries = static_cast(descriptor_data); + update_descriptors = + bind_pipeline || last_descriptor_payload.size() != num_descriptor_entries || + std::memcmp(last_descriptor_payload.data(), entries, + num_descriptor_entries * sizeof(DescriptorUpdateEntry)) != 0; + if (update_descriptors) { + last_descriptor_payload.assign(entries, entries + num_descriptor_entries); + } + } + VkDeviceSize descriptor_buffer_offset{}; + bool descriptor_buffer_ready{false}; + if (descriptor_set_layout && uses_descriptor_buffer) { + const DescriptorBufferRing::Allocation alloc{ + descriptor_buffer_ring.Allocate(scheduler, descriptor_buffer_layout.size)}; + if (alloc.host) { + WriteDescriptorBuffer(device, descriptor_buffer_layout, + static_cast(descriptor_data), + alloc.host); + descriptor_buffer_offset = alloc.offset; + descriptor_buffer_ready = true; + } + } + scheduler.Record([this, descriptor_data, bind_pipeline, update_descriptors, + descriptor_buffer_offset, descriptor_buffer_ready, + rescaling_data = rescaling.Data(), is_rescaling, update_rescaling, uses_render_area = render_area.uses_render_area, render_area_data = render_area.words](vk::CommandBuffer cmdbuf) { if (bind_pipeline) { @@ -593,10 +640,20 @@ void GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, if (!descriptor_set_layout) { return; } - if (uses_push_descriptor) { + if (uses_descriptor_buffer) { + if (!descriptor_buffer_ready) { + return; + } + const VkDescriptorBufferBindingInfoEXT binding_info{ + descriptor_buffer_ring.BindingInfo()}; + cmdbuf.BindDescriptorBuffersEXT(binding_info); + const u32 buffer_index{}; + cmdbuf.SetDescriptorBufferOffsetsEXT(VK_PIPELINE_BIND_POINT_GRAPHICS, *pipeline_layout, + 0, buffer_index, descriptor_buffer_offset); + } else if (uses_push_descriptor) { cmdbuf.PushDescriptorSetWithTemplateKHR(*descriptor_update_template, *pipeline_layout, 0, descriptor_data); - } else { + } else if (update_descriptors) { const VkDescriptorSet descriptor_set{descriptor_allocator.Commit()}; const vk::Device& dev{device.GetLogical()}; dev.UpdateDescriptorSet(descriptor_set, *descriptor_update_template, descriptor_data); @@ -757,16 +814,13 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .lineWidth = 1.0f, // TODO(alekpop): Transfer from regs }; - const bool smooth_lines_supported = - device.IsExtLineRasterizationSupported() && device.SupportsSmoothLines(); - const bool stippled_lines_supported = - device.IsExtLineRasterizationSupported() && device.SupportsStippledRectangularLines(); + const VkLineRasterizationModeEXT line_raster_mode = + device.GetLineRasterizationMode(key.state.smooth_lines != 0); + const bool stippled_lines_supported = device.SupportsStippleForMode(line_raster_mode); VkPipelineRasterizationLineStateCreateInfoEXT line_state{ .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_LINE_STATE_CREATE_INFO_EXT, .pNext = nullptr, - .lineRasterizationMode = key.state.smooth_lines != 0 && smooth_lines_supported - ? VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT - : VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT, + .lineRasterizationMode = line_raster_mode, .stippledLineEnable = (dynamic.line_stipple_enable && stippled_lines_supported) ? VK_TRUE : VK_FALSE, .lineStippleFactor = key.state.line_stipple_factor, @@ -805,6 +859,16 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { if (device.IsExtProvokingVertexSupported()) { provoking_vertex.pNext = std::exchange(rasterization_ci.pNext, &provoking_vertex); } + VkPipelineRasterizationDepthClipStateCreateInfoEXT depth_clip_state{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_DEPTH_CLIP_STATE_CREATE_INFO_EXT, + .pNext = nullptr, + .flags = 0, + .depthClipEnable = static_cast(dynamic.depth_clip_disabled == 0 ? VK_TRUE + : VK_FALSE), + }; + if (device.IsExtDepthClipEnableSupported()) { + depth_clip_state.pNext = std::exchange(rasterization_ci.pNext, &depth_clip_state); + } const bool supports_alpha_output = fragment_has_color0_output; const bool alpha_to_one_supported = device.SupportsAlphaToOne(); @@ -995,10 +1059,65 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { if (device.IsKhrPipelineExecutablePropertiesEnabled() && Settings::values.renderer_debug.GetValue()) { flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR; } + if (uses_descriptor_buffer) { + flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; + } + + const RenderPassKey renderpass_key{MakeRenderPassKey(key.state, device)}; + std::array color_attachment_formats{}; + for (size_t index = 0; index < renderpass_key.color_formats.size(); ++index) { + const PixelFormat pixel_format{renderpass_key.color_formats[index]}; + if (pixel_format == PixelFormat::Invalid) { + color_attachment_formats[index] = VK_FORMAT_UNDEFINED; + continue; + } + color_attachment_formats[index] = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, pixel_format).format; + } + VkFormat depth_attachment_format{VK_FORMAT_UNDEFINED}; + VkFormat stencil_attachment_format{VK_FORMAT_UNDEFINED}; + if (renderpass_key.depth_format != PixelFormat::Invalid) { + const VkFormat format{ + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, + renderpass_key.depth_format) + .format}; + const auto surface_type{VideoCore::Surface::GetFormatType(renderpass_key.depth_format)}; + if (surface_type == VideoCore::Surface::SurfaceType::Depth || + surface_type == VideoCore::Surface::SurfaceType::DepthStencil) { + depth_attachment_format = format; + } + if (surface_type == VideoCore::Surface::SurfaceType::Stencil || + surface_type == VideoCore::Surface::SurfaceType::DepthStencil) { + stencil_attachment_format = format; + } + } + const VkPipelineRenderingCreateInfo rendering_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, + .pNext = nullptr, + .viewMask = 0, + .colorAttachmentCount = static_cast(NumAttachments(key.state)), + .pColorAttachmentFormats = color_attachment_formats.data(), + .depthAttachmentFormat = depth_attachment_format, + .stencilAttachmentFormat = stencil_attachment_format, + }; + + VkPipelineCreationFeedback creation_feedback{}; + const VkPipelineCreationFeedbackCreateInfo feedback_ci{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CREATION_FEEDBACK_CREATE_INFO, + .pNext = device.IsKhrDynamicRenderingSupported() ? &rendering_ci : nullptr, + .pPipelineCreationFeedback = &creation_feedback, + .pipelineStageCreationFeedbackCount = 0, + .pPipelineStageCreationFeedbacks = nullptr, + }; + const void* const create_next = + device.IsExtPipelineCreationFeedbackSupported() + ? static_cast(&feedback_ci) + : (device.IsKhrDynamicRenderingSupported() ? static_cast(&rendering_ci) + : nullptr); pipeline = device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = create_next, .flags = flags, .stageCount = static_cast(shader_stages.size()), .pStages = shader_stages.data(), @@ -1018,6 +1137,14 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .basePipelineIndex = 0, }, *pipeline_cache); + if ((creation_feedback.flags & VK_PIPELINE_CREATION_FEEDBACK_VALID_BIT) != 0) { + const bool cache_hit = + (creation_feedback.flags & + VK_PIPELINE_CREATION_FEEDBACK_APPLICATION_PIPELINE_CACHE_HIT_BIT) != 0; + LOG_DEBUG(Render_Vulkan, "Graphics pipeline {:016X} cache_hit={} duration={}us", + key.Hash(), cache_hit, creation_feedback.duration / 1000); + } + // Log graphics pipeline creation if (GPU::Logging::IsActive()) { const std::string pipeline_info = fmt::format( diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index d1caeaee8e..6e3481d609 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -12,14 +12,18 @@ #include #include #include +#include #include "common/thread_worker.h" #include "shader_recompiler/shader_info.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/fixed_pipeline_state.h" +#include "video_core/renderer_vulkan/pipeline_helper.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" +#include "video_core/renderer_vulkan/vk_update_descriptor.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace VideoCore { @@ -76,7 +80,8 @@ public: Scheduler& scheduler, BufferCache& buffer_cache, TextureCache& texture_cache, vk::PipelineCache& pipeline_cache, VideoCore::ShaderNotify* shader_notify, const Device& device, DescriptorPool& descriptor_pool, - GuestDescriptorQueue& guest_descriptor_queue, Common::ThreadWorker* worker_thread, + GuestDescriptorQueue& guest_descriptor_queue, + DescriptorBufferRing& descriptor_buffer_ring, Common::ThreadWorker* worker_thread, PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache, const GraphicsPipelineCacheKey& key, std::array stages, const std::array& infos); @@ -148,6 +153,7 @@ private: vk::PipelineCache& pipeline_cache; Scheduler& scheduler; GuestDescriptorQueue& guest_descriptor_queue; + DescriptorBufferRing& descriptor_buffer_ring; bool (*configure_func)(GraphicsPipeline*, bool){}; @@ -170,10 +176,14 @@ private: vk::DescriptorUpdateTemplate descriptor_update_template; vk::Pipeline pipeline; + DescriptorBufferLayout descriptor_buffer_layout; + std::vector last_descriptor_payload; + std::condition_variable build_condvar; std::mutex build_mutex; std::atomic_bool is_built{false}; bool uses_push_descriptor{false}; + bool uses_descriptor_buffer{false}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 0ff4e52c54..8657e95401 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -17,6 +17,7 @@ #include "common/cityhash.h" #include "common/fs/fs.h" #include "common/fs/path_util.h" +#include "common/settings.h" #include "common/thread_worker.h" #include "core/core.h" #include "shader_recompiler/backend/spirv/emit_spirv.h" @@ -45,10 +46,6 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" #include "video_core/gpu_logging/gpu_logging.h" -#ifdef __ANDROID__ -#include "../../android/app/src/main/jni/android_settings.h" -#endif - namespace Vulkan { namespace { @@ -63,6 +60,8 @@ using VideoCommon::GenericEnvironment; using VideoCommon::GraphicsEnvironment; constexpr u32 CACHE_VERSION = 18; +constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128; +constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30; constexpr std::array VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'}; template @@ -304,12 +303,8 @@ size_t GetTotalPipelineWorkers() { const size_t max_core_threads = std::max(static_cast(std::thread::hardware_concurrency()), 2ULL) - 1ULL; #ifdef __ANDROID__ - const int configured = AndroidSettings::values.pipeline_worker_count.GetValue(); - const int clamped = std::clamp(configured, 4, 8); - const size_t desired = static_cast(clamped); - if (desired == 0) { - return 1ULL; - } + const s32 configured = Settings::values.pipeline_worker_count.GetValue(); + const size_t desired = static_cast(std::clamp(configured, 2, 8)); return std::min(max_core_threads, desired); #else return max_core_threads; @@ -340,17 +335,20 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, GuestDescriptorQueue& guest_descriptor_queue_, + DescriptorBufferRing& descriptor_buffer_ring_, RenderPassCache& render_pass_cache_, BufferCache& buffer_cache_, TextureCache& texture_cache_, VideoCore::ShaderNotify& shader_notify_) : VideoCommon::ShaderCache{device_memory_}, device{device_}, scheduler{scheduler_}, descriptor_pool{descriptor_pool_}, guest_descriptor_queue{guest_descriptor_queue_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_}, texture_cache{texture_cache_}, shader_notify{shader_notify_}, use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()}, use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()}, workers(device.HasBrokenParallelShaderCompiling() ? 1ULL : GetTotalPipelineWorkers(), - "VkPipelineBuilder"), - serialization_thread(1, "VkPipelineSerialization") { + "VkPipelineBuilder", {}, Common::ThreadPlacement::Background), + serialization_thread(1, "VkPipelineSerialization", {}, + Common::ThreadPlacement::Background) { const auto& float_control{device.FloatControlProperties()}; const VkDriverId driver_id{device.GetDriverID()}; const VkShaderStageFlags subgroup_stages{device.GetSubgroupSupportedStages()}; @@ -401,6 +399,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.IsWorkgroupMemoryExplicitLayout8BitAccessSupported(), .support_workgroup_layout_16bit_access = device.IsWorkgroupMemoryExplicitLayout16BitAccessSupported(), + .support_shader_quad_control = device.IsKhrShaderQuadControlSupported(), + .support_quad_shuffles = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_QUAD_BIT), .support_vote = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_VOTE_BIT), .supported_subgroup_stages = supported_subgroup_stages, .support_viewport_index_layer_non_geometry = @@ -514,6 +514,11 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, dynamic_features.has_dynamic_state3_depth_clamp_enable = dynamic_features.has_extended_dynamic_state_3_enables && device.SupportsDynamicState3DepthClampEnable(); + dynamic_features.has_dynamic_state3_conservative_raster_mode = + dynamic_features.has_extended_dynamic_state_3_enables && + device.SupportsDynamicState3ConservativeRasterizationMode(); + dynamic_features.has_depth_clip_enable = + device.IsExtDepthClipEnableSupported(); dynamic_features.has_dynamic_state3_logic_op_enable = dynamic_features.has_extended_dynamic_state_3_enables && device.SupportsDynamicState3LogicOpEnable(); @@ -526,7 +531,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.IsExtVertexInputDynamicStateSupported() && Settings::values.vertex_input_dynamic_state.GetValue(); - dynamic_features.has_provoking_vertex = device.IsExtProvokingVertexSupported(); + dynamic_features.has_provoking_vertex = + device.IsExtProvokingVertexSupported(); dynamic_features.has_provoking_vertex_first_mode = device.SupportsProvokingVertexFirstMode(); dynamic_features.has_provoking_vertex_last_mode = @@ -696,6 +702,10 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading if (use_vulkan_pipeline_cache) { SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache, CACHE_VERSION); + size_t size = 0; + vulkan_pipeline_cache.Read(&size, nullptr); + last_cache_size.store(size, std::memory_order_relaxed); + last_flush = std::chrono::steady_clock::now(); } if (state.statistics) { @@ -703,6 +713,35 @@ void PipelineCache::LoadDiskResources(u64 title_id, std::stop_token stop_loading } } +void PipelineCache::QueueVulkanPipelineCacheFlush() { + if (!use_vulkan_pipeline_cache || vulkan_pipeline_cache_filename.empty()) { + return; + } + if (++pipelines_since_flush < VULKAN_CACHE_FLUSH_PIPELINES) { + return; + } + const auto now = std::chrono::steady_clock::now(); + const auto megabytes = last_cache_size.load(std::memory_order_relaxed) / (1024 * 1024); + const std::chrono::seconds interval{ + std::max(VULKAN_CACHE_FLUSH_MIN_SECONDS, megabytes)}; + if (last_flush.time_since_epoch().count() != 0 && now - last_flush < interval) { + return; + } + if (flush_in_flight.exchange(true, std::memory_order_acq_rel)) { + return; + } + pipelines_since_flush = 0; + last_flush = now; + serialization_thread.QueueWork([this] { + SerializeVulkanPipelineCache(vulkan_pipeline_cache_filename, vulkan_pipeline_cache, + CACHE_VERSION); + size_t size = 0; + vulkan_pipeline_cache.Read(&size, nullptr); + last_cache_size.store(size, std::memory_order_relaxed); + flush_in_flight.store(false, std::memory_order_release); + }); +} + GraphicsPipeline* PipelineCache::CurrentGraphicsPipelineSlowPath() { const auto [pair, is_new]{graphics_cache.try_emplace(graphics_key)}; auto& pipeline{pair->second}; @@ -741,7 +780,7 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline( std::span envs, PipelineStatistics* statistics, bool build_in_parallel) try { auto hash = key.Hash(); - LOG_INFO(Render_Vulkan, "0x{:016x}", hash); + LOG_DEBUG(Render_Vulkan, "{:#016x}", hash); size_t env_index{0}; std::array programs; const bool uses_vertex_a{key.unique_hashes[0] != 0}; @@ -836,8 +875,8 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline( Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr}; return std::make_unique( scheduler, buffer_cache, texture_cache, vulkan_pipeline_cache, &shader_notify, device, - descriptor_pool, guest_descriptor_queue, thread_worker, statistics, render_pass_cache, key, - std::move(modules), infos); + descriptor_pool, guest_descriptor_queue, descriptor_buffer_ring, thread_worker, statistics, + render_pass_cache, key, std::move(modules), infos); } catch (const Shader::Exception& exception) { auto hash = key.Hash(); @@ -877,6 +916,7 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline() { } SerializePipeline(key, env_ptrs, pipeline_cache_filename, CACHE_VERSION); }); + QueueVulkanPipelineCacheFlush(); return pipeline; } @@ -896,6 +936,7 @@ std::unique_ptr PipelineCache::CreateComputePipeline( SerializePipeline(key, std::array{&env_}, pipeline_cache_filename, CACHE_VERSION); }); + QueueVulkanPipelineCacheFlush(); return pipeline; } @@ -904,11 +945,11 @@ std::unique_ptr PipelineCache::CreateComputePipeline( PipelineStatistics* statistics, bool build_in_parallel) try { auto hash = key.Hash(); if (device.HasBrokenCompute()) { - LOG_ERROR(Render_Vulkan, "Skipping 0x{:016x}", hash); + LOG_ERROR(Render_Vulkan, "Skipping {:#016x}", hash); return nullptr; } - LOG_INFO(Render_Vulkan, "0x{:016x}", hash); + LOG_DEBUG(Render_Vulkan, "{:#016x}", hash); Shader::Maxwell::Flow::CFG cfg{env, pools.flow_block, env.StartAddress()}; @@ -925,7 +966,7 @@ std::unique_ptr PipelineCache::CreateComputePipeline( const u32 max_shared_memory = device.GetMaxComputeSharedMemorySize(); if (needs_shared_mem_clamp && program.shared_memory_size > max_shared_memory) { LOG_WARNING(Render_Vulkan, - "Compute shader 0x{:016x} requests {}KB shared memory but device max is {}KB - clamping", + "Compute shader {:#016x} requests {}KB shared memory but device max is {}KB - clamping", key.unique_hash, program.shared_memory_size / 1024, max_shared_memory / 1024); @@ -957,7 +998,8 @@ std::unique_ptr PipelineCache::CreateComputePipeline( } Common::ThreadWorker* const thread_worker{build_in_parallel ? &workers : nullptr}; return std::make_unique(device, scheduler, vulkan_pipeline_cache, descriptor_pool, - guest_descriptor_queue, thread_worker, statistics, + guest_descriptor_queue, descriptor_buffer_ring, + thread_worker, statistics, &shader_notify, program.info, std::move(spv_module), key.unique_hash); diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h index 490f9da81a..9d66b663f5 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h @@ -7,6 +7,8 @@ #pragma once #include +#include +#include #include #include #include @@ -105,6 +107,7 @@ public: explicit PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue, + DescriptorBufferRing& descriptor_buffer_ring, RenderPassCache& render_pass_cache, BufferCache& buffer_cache, TextureCache& texture_cache, VideoCore::ShaderNotify& shader_notify_); ~PipelineCache(); @@ -143,10 +146,13 @@ private: vk::PipelineCache LoadVulkanPipelineCache(const std::filesystem::path& filename, u32 expected_cache_version); + void QueueVulkanPipelineCacheFlush(); + const Device& device; Scheduler& scheduler; DescriptorPool& descriptor_pool; GuestDescriptorQueue& guest_descriptor_queue; + DescriptorBufferRing& descriptor_buffer_ring; RenderPassCache& render_pass_cache; BufferCache& buffer_cache; TextureCache& texture_cache; @@ -169,6 +175,10 @@ private: std::filesystem::path vulkan_pipeline_cache_filename; vk::PipelineCache vulkan_pipeline_cache; + size_t pipelines_since_flush{}; + std::chrono::steady_clock::time_point last_flush{}; + std::atomic last_cache_size{}; + std::atomic_bool flush_in_flight{}; Common::ThreadWorker workers; Common::ThreadWorker serialization_thread; diff --git a/src/video_core/renderer_vulkan/vk_present_manager.cpp b/src/video_core/renderer_vulkan/vk_present_manager.cpp index 53c3f6759f..e3fd2301c1 100644 --- a/src/video_core/renderer_vulkan/vk_present_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_present_manager.cpp @@ -266,6 +266,7 @@ void PresentManager::WaitPresent() { void PresentManager::PresentThread(std::stop_token token) { Common::SetCurrentThreadName("VulkanPresent"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::High); while (!token.stop_requested()) { std::unique_lock lock{queue_mutex}; // Wait for presentation frames @@ -296,9 +297,6 @@ void PresentManager::RecreateSwapchain(Frame* frame) { } void PresentManager::SetImageCount() { - // We cannot have more than 7 images in flight at any given time. - // FRAMES_IN_FLIGHT is 8, and the cache TICKS_TO_DESTROY is 8. - // Mali drivers will give us 6. image_count = std::min(swapchain.GetImageCount(), 7); } diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 709f65dc25..6d8e2309cd 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -235,11 +235,23 @@ public: return; } PauseCounter(); - const auto driver_id = device.GetDriverID(); - if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || - driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { - pending_sync.clear(); + if (!CanResolveHostQueries()) { sync_values_stash.clear(); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([buffer = *accumulation_buffer](vk::CommandBuffer cmdbuf) { + cmdbuf.FillBuffer(buffer, 0, 8, 0); + }); + std::function func([this] { + amend_value = 0; + accumulation_value = 0; + }); + rasterizer->SyncOperation(std::move(func)); + AbandonCurrentQuery(); + num_slots_used = 0; + first_accumulation_checkpoint = (std::numeric_limits::max)(); + last_accumulation_checkpoint = 0; + accumulation_since_last_sync = false; + pending_sync.clear(); return; } sync_values_stash.clear(); @@ -406,6 +418,13 @@ public: } private: + bool CanResolveHostQueries() const { + const auto driver_id = device.GetDriverID(); + return driver_id != VK_DRIVER_ID_QUALCOMM_PROPRIETARY && + driver_id != VK_DRIVER_ID_ARM_PROPRIETARY && + driver_id != VK_DRIVER_ID_MESA_TURNIP; + } + template void ApplyBankOp(VideoCommon::HostQueryBase* query, Func&& func) { size_t size_slots = query->size_slots; @@ -919,7 +938,7 @@ private: return; } has_flushed_end_pending = true; - // Refresh buffers state before beginning transform feedback so counters are up-to-date + scheduler.MarkTransformFeedbackUsed(); UpdateBuffers(); if (!has_started || buffers_count == 0) { // No counter buffers available: begin without counters @@ -1423,18 +1442,12 @@ void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::Lo return; } } - bool was_running = impl->is_hcr_running; - if (was_running) { - PauseHostConditionalRendering(); - } + PauseHostConditionalRendering(); impl->hcr_setup.buffer = impl->hcr_buffer; impl->hcr_setup.offset = impl->hcr_offset; impl->hcr_setup.flags = is_equal ? VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT : 0; impl->hcr_is_set = true; impl->is_hcr_running = false; - if (was_running) { - ResumeHostConditionalRendering(); - } } void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, bool is_equal, @@ -1451,10 +1464,7 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo to_resolve = buffer->Handle(); to_resolve_offset = static_cast(offset); } - bool was_running = impl->is_hcr_running; - if (was_running) { - PauseHostConditionalRendering(); - } + PauseHostConditionalRendering(); impl->conditional_resolve_pass->Resolve(*impl->hcr_resolve_buffer, to_resolve, to_resolve_offset, compare_to_zero); impl->hcr_setup.buffer = *impl->hcr_resolve_buffer; @@ -1462,9 +1472,6 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo impl->hcr_setup.flags = is_equal ? 0 : VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT; impl->hcr_is_set = true; impl->is_hcr_running = false; - if (was_running) { - ResumeHostConditionalRendering(); - } } bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::LookupData object_1, @@ -1472,6 +1479,12 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::Lookup if (!impl->device.IsExtConditionalRendering()) { return false; } + const auto driver_id = impl->device.GetDriverID(); + if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || + driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { + EndHostConditionalRendering(); + return true; + } HostConditionalRenderingCompareBCImpl(object_1.address, true, true); return true; } @@ -1518,10 +1531,12 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValues(VideoCommon::Looku return false; } - auto driver_id = impl->device.GetDriverID(); + const auto driver_id = impl->device.GetDriverID(); const bool is_gpu_high = Settings::IsGPULevelHigh(); - if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { + if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || + driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || + driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { EndHostConditionalRendering(); return true; } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index 720f3b868c..d6a60e8439 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include @@ -102,8 +104,20 @@ VkViewport GetViewportState(const Device& device, const Maxwell& regs, size_t in .maxDepth = src.translate_z + src.scale_z, }; if (!device.IsExtDepthRangeUnrestrictedSupported()) { + const float unclamped_min = viewport.minDepth; + const float unclamped_max = viewport.maxDepth; viewport.minDepth = std::clamp(viewport.minDepth, 0.0f, 1.0f); viewport.maxDepth = std::clamp(viewport.maxDepth, 0.0f, 1.0f); + if (viewport.minDepth != unclamped_min || viewport.maxDepth != unclamped_max) { + static std::atomic reported{0}; + if (reported.fetch_add(1, std::memory_order_relaxed) < 32) { + LOG_WARNING(Render_Vulkan, + "Depth range clamped: viewport={} mode={} translate_z={} scale_z={} " + "range=[{}, {}] -> [{}, {}]", + index, static_cast(regs.depth_mode), src.translate_z, src.scale_z, + unclamped_min, unclamped_max, viewport.minDepth, viewport.maxDepth); + } + } } return viewport; } @@ -203,7 +217,10 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra : gpu{gpu_}, device_memory{device_memory_}, device{device_}, memory_allocator{memory_allocator_}, state_tracker{state_tracker_}, scheduler{scheduler_}, staging_pool(device, memory_allocator, scheduler), descriptor_pool(device, scheduler), - guest_descriptor_queue(device), compute_pass_descriptor_queue(device), + guest_descriptor_queue(device, UpdateDescriptorQueue::GUEST_FRAME_PAYLOAD_SIZE, + device.IsExtDescriptorBufferSupported()), + compute_pass_descriptor_queue(device, UpdateDescriptorQueue::COMPUTE_FRAME_PAYLOAD_SIZE), + descriptor_buffer_ring(device, memory_allocator), blit_image(device, scheduler, state_tracker, descriptor_pool), render_pass_cache(device), texture_cache_runtime{ device, scheduler, memory_allocator, staging_pool, @@ -216,14 +233,51 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra staging_pool, compute_pass_descriptor_queue, descriptor_pool, texture_cache), query_cache(gpu, *this, device_memory, query_cache_runtime), pipeline_cache(device_memory, device, scheduler, descriptor_pool, guest_descriptor_queue, - render_pass_cache, buffer_cache, texture_cache, gpu.ShaderNotify()), + descriptor_buffer_ring, render_pass_cache, buffer_cache, texture_cache, + gpu.ShaderNotify()), accelerate_dma(buffer_cache, texture_cache, scheduler), fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler), wfi_event(device.GetLogical().CreateEvent()) { scheduler.SetQueryCache(query_cache); + if (Settings::values.use_unified_memory.GetValue() && device_memory.IsBackingShared()) { + buffer_cache_runtime.TryEnableUnifiedMemory( + device_memory.GetPhysicalBase(), device_memory.GetPhysicalSize(), + device_memory.GetBackingHardwareBuffers(), + device_memory.GetBackingHardwareBufferWindowSize(), + device_memory.GetBackingHardwareBufferBase()); + } + memory_allocator.SetReclaimCallback([this](u64 bytes) -> u64 { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + const u64 usage_before = device.GetMemoryBudgetInfo().allocation_bytes; + master_semaphore.Refresh(); + const u64 completed = master_semaphore.KnownGpuTick(); + texture_cache.ReclaimDeferredResources(completed); + buffer_cache.ReclaimDeferredResources(completed); + vk::TickDeletionQueue(completed); + const u64 usage_after = device.GetMemoryBudgetInfo().allocation_bytes; + const u64 drained = usage_before > usage_after ? usage_before - usage_after : 0; + if (drained >= bytes) { + return drained; + } + const u64 remaining = bytes - drained; + u64 evicted = staging_pool.ReclaimMemory(remaining); + if (evicted < remaining) { + evicted += texture_cache.ReclaimMemory(remaining - evicted, false); + } + if (evicted < remaining) { + evicted += buffer_cache.ReclaimMemory(remaining - evicted, false); + } + master_semaphore.Refresh(); + const u64 completed_after = master_semaphore.KnownGpuTick(); + texture_cache.ReclaimDeferredResources(completed_after); + buffer_cache.ReclaimDeferredResources(completed_after); + vk::TickDeletionQueue(completed_after); + return drained + evicted; + }); } RasterizerVulkan::~RasterizerVulkan() { + memory_allocator.SetReclaimCallback(nullptr); scheduler.WaitWorker(); scheduler.Finish(); } @@ -241,11 +295,13 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { if (!pipeline) { return; } - std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; - // update engine as channel may be different. - pipeline->SetEngine(maxwell3d, gpu_memory); - if (!pipeline->Configure(is_indexed)) - return; + { + std::scoped_lock lock{buffer_cache.mutex, texture_cache.mutex}; + pipeline->SetEngine(maxwell3d, gpu_memory); + if (!pipeline->Configure(is_indexed)) { + return; + } + } UpdateDynamicStates(); @@ -359,7 +415,6 @@ void RasterizerVulkan::DrawTexture() { UpdateDynamicStates(); - query_cache.NotifySegment(true); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); const auto& draw_texture_state = maxwell3d->draw_manager.draw_texture_state; const auto& sampler = texture_cache.GetSampler(draw_texture_state.src_sampler, false); @@ -420,15 +475,21 @@ void RasterizerVulkan::Clear(u32 layer_count) { const bool ds_deferrable = !ds_used || ((!framebuffer->HasAspectDepthBit() || use_depth) && (!framebuffer->HasAspectStencilBit() || use_stencil) && !stencil_partial); - const bool can_defer_clear = ENABLE_DEFERRED_CLEAR && !regs.clear_control.use_scissor && - regs.clear_surface.layer == 0 && - !scheduler.IsRenderPassActive() && - (!use_color || color_full_channels) && ds_deferrable; + const bool clear_shape_deferrable = ENABLE_DEFERRED_CLEAR && + !regs.clear_control.use_scissor && + regs.clear_surface.layer == 0 && + (!use_color || color_full_channels) && ds_deferrable; + // An open pass normally blocks deferral, which drops the clear to ClearAttachments inside the + // pass and also loses the MSAA store discard, since that only applies when a clear is folded + // into the begin. A pass whose BeginRendering has not been recorded yet can be retracted for + // free, so the clear becomes a load op after all. Only retract when it will actually be used. + const bool can_defer_clear = + clear_shape_deferrable && + (!scheduler.IsRenderPassActive() || scheduler.RetractUnrecordedRenderPass()); if (!can_defer_clear) { scheduler.RequestRenderpass(framebuffer); } - query_cache.NotifySegment(true); query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); u32 up_scale = 1; u32 down_shift = 0; @@ -673,7 +734,6 @@ void RasterizerVulkan::FlushRegion(DAddr addr, u64 size, VideoCommon::CacheType texture_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::BufferCache))) { - std::scoped_lock lock{buffer_cache.mutex}; buffer_cache.DownloadMemory(addr, size); } if ((True(which & VideoCommon::CacheType::QueryCache))) { @@ -771,12 +831,15 @@ bool RasterizerVulkan::OnCPUWrite(DAddr addr, u64 size) { return false; } +static constexpr bool ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP = true; + void RasterizerVulkan::OnCacheInvalidation(DAddr addr, u64 size) { if (addr == 0 || size == 0) { return; } - { + if (!ENABLE_TEXTURE_CACHE_INVALIDATION_SKIP || + device_memory.IsRegionTextureCached(addr, size)) { std::scoped_lock lock{texture_cache.mutex}; texture_cache.WriteMemory(addr, size); } @@ -880,8 +943,12 @@ void RasterizerVulkan::FlushCommands() { void RasterizerVulkan::TickFrame() { draw_counter = 0; + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + vk::TickDeletionQueue(master_semaphore.KnownGpuTick()); guest_descriptor_queue.TickFrame(); compute_pass_descriptor_queue.TickFrame(); + descriptor_buffer_ring.TickFrame(); fence_manager.TickFrame(); staging_pool.TickFrame(); { @@ -969,11 +1036,11 @@ void RasterizerVulkan::LoadDiskResources(u64 title_id, std::stop_token stop_load void RasterizerVulkan::FlushWork() { #ifdef __ANDROID__ - static constexpr u32 DRAWS_TO_DISPATCH = 512; - static constexpr u32 CHECK_MASK = 3; + static constexpr u32 DRAWS_TO_DISPATCH = 1024; + static constexpr u32 CHECK_MASK = 63; #else static constexpr u32 DRAWS_TO_DISPATCH = 4096; - static constexpr u32 CHECK_MASK = 7; + static constexpr u32 CHECK_MASK = 31; #endif // __ANDROID__ static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0); @@ -1279,6 +1346,13 @@ void RasterizerVulkan::UpdateDepthBias(Tegra::Engines::Maxwell3D::Regs& regs) { regs.zeta.format == Tegra::DepthFormat::S8Z24_UNORM || regs.zeta.format == Tegra::DepthFormat::V8Z24_UNORM; + const bool is_float_depth = regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT || + regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT_X24S8_UINT; + + if (is_float_depth && !device.IsExtDepthBiasControlSupported()) { + units /= static_cast(1ULL << (32 - 24)); + } + if (is_d24 && !device.SupportsD24DepthBuffer()) { static constexpr const size_t length = sizeof(NEEDS_D24) / sizeof(NEEDS_D24[0]); @@ -1451,7 +1525,10 @@ void RasterizerVulkan::UpdateLineWidth(Tegra::Engines::Maxwell3D::Regs& regs) { } const float width = regs.line_anti_alias_enable ? regs.line_width_smooth : regs.line_width_aliased; - scheduler.Record([width](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineWidth(width); }); + const float clamped_width = device.ClampLineWidth(width); + scheduler.Record([clamped_width](vk::CommandBuffer cmdbuf) { + cmdbuf.SetLineWidth(clamped_width); + }); } void RasterizerVulkan::UpdateCullMode(Tegra::Engines::Maxwell3D::Regs& regs) { @@ -1548,7 +1625,10 @@ void RasterizerVulkan::UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs& return; } - scheduler.Record([enable = regs.line_stipple_enable](vk::CommandBuffer cmdbuf) { + const VkLineRasterizationModeEXT mode = + device.GetLineRasterizationMode(regs.line_anti_alias_enable != 0); + const bool enable = regs.line_stipple_enable != 0 && device.SupportsStippleForMode(mode); + scheduler.Record([enable](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineStippleEnableEXT(enable); }); } @@ -1562,28 +1642,24 @@ void RasterizerVulkan::UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Re } if (!device.SupportsDynamicState3LineRasterizationMode()) { - static std::once_flag warn_missing_rect; - std::call_once(warn_missing_rect, [] { + static std::once_flag warn_missing_dynamic_state; + std::call_once(warn_missing_dynamic_state, [] { LOG_WARNING(Render_Vulkan, - "Driver lacks rectangular line rasterization support; skipping dynamic " - "line state updates"); + "Driver lacks dynamic line rasterization mode; the pipeline static value " + "is used instead"); }); return; } const bool wants_smooth = regs.line_anti_alias_enable != 0; - VkLineRasterizationModeEXT mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT; - if (wants_smooth) { - if (device.SupportsSmoothLines()) { - mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT; - } else { - static std::once_flag warn_missing_smooth; - std::call_once(warn_missing_smooth, [] { - LOG_WARNING(Render_Vulkan, - "Line anti-aliasing requested but smoothLines feature unavailable; " - "using rectangular rasterization"); - }); - } + const VkLineRasterizationModeEXT mode = device.GetLineRasterizationMode(wants_smooth); + if (wants_smooth && mode != VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT) { + static std::once_flag warn_missing_smooth; + std::call_once(warn_missing_smooth, [] { + LOG_WARNING(Render_Vulkan, + "Line anti-aliasing requested but smoothLines feature unavailable; " + "falling back to the closest supported mode"); + }); } scheduler.Record([mode](vk::CommandBuffer cmdbuf) { cmdbuf.SetLineRasterizationModeEXT(mode); @@ -1643,12 +1719,7 @@ void RasterizerVulkan::UpdateDepthClampEnable(Tegra::Engines::Maxwell3D::Regs& r if (!device.SupportsDynamicState3DepthClampEnable()) { return; } - bool is_enabled = !(regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::Passthrough || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumXYZ || - regs.viewport_clip_control.geometry_clip == - Maxwell::ViewportClipControl::GeometryClip::FrustumZ); + const bool is_enabled = IsDepthClampEnabled(regs, device.IsExtDepthClipEnableSupported()); scheduler.Record( [is_enabled](vk::CommandBuffer cmdbuf) { cmdbuf.SetDepthClampEnableEXT(is_enabled); }); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 206bf1043a..5849eff833 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -17,6 +17,7 @@ #include "video_core/rasterizer_interface.h" #include "video_core/renderer_vulkan/blit_image.h" #include "video_core/renderer_vulkan/vk_buffer_cache.h" +#include "video_core/renderer_vulkan/vk_descriptor_buffer.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" #include "video_core/renderer_vulkan/vk_fence_manager.h" #include "video_core/renderer_vulkan/vk_pipeline_cache.h" @@ -179,7 +180,6 @@ private: void UpdateRasterizerDiscardEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateConservativeRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs& regs); - void UpdateLineStipple(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateDepthBiasEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLogicOpEnable(Tegra::Engines::Maxwell3D::Regs& regs); @@ -207,6 +207,7 @@ private: DescriptorPool descriptor_pool; GuestDescriptorQueue guest_descriptor_queue; ComputePassDescriptorQueue compute_pass_descriptor_queue; + DescriptorBufferRing descriptor_buffer_ring; BlitImageHelper blit_image; RenderPassCache render_pass_cache; diff --git a/src/video_core/renderer_vulkan/vk_resource_pool.cpp b/src/video_core/renderer_vulkan/vk_resource_pool.cpp index 6572f82ba9..9a922e6c85 100644 --- a/src/video_core/renderer_vulkan/vk_resource_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_resource_pool.cpp @@ -12,10 +12,7 @@ ResourcePool::ResourcePool(MasterSemaphore& master_semaphore_, size_t grow_step_ : master_semaphore{&master_semaphore_}, grow_step{grow_step_} {} size_t ResourcePool::CommitResource() { - // Refresh semaphore to query updated results - master_semaphore->Refresh(); - const u64 gpu_tick = master_semaphore->KnownGpuTick(); - const auto search = [this, gpu_tick](size_t begin, size_t end) -> std::optional { + const auto search = [this](size_t begin, size_t end, u64 gpu_tick) -> std::optional { for (size_t iterator = begin; iterator < end; ++iterator) { if (gpu_tick >= ticks[iterator]) { ticks[iterator] = master_semaphore->CurrentTick(); @@ -24,11 +21,17 @@ size_t ResourcePool::CommitResource() { } return std::nullopt; }; - // Try to find a free resource from the hinted position to the end. - std::optional found = search(hint_iterator, ticks.size()); + const auto find_free = [&](u64 gpu_tick) -> std::optional { + std::optional result = search(hint_iterator, ticks.size(), gpu_tick); + if (!result) { + result = search(0, hint_iterator, gpu_tick); + } + return result; + }; + std::optional found = find_free(master_semaphore->KnownGpuTick()); if (!found) { - // Search from beginning to the hinted position. - found = search(0, hint_iterator); + master_semaphore->Refresh(); + found = find_free(master_semaphore->KnownGpuTick()); if (!found) { // Both searches failed, the pool is full; handle it. const size_t free_resource = ManageOverflow(); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 6234b41978..4cd932bafb 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -47,6 +47,7 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) master_semaphore{std::make_unique(device)}, command_pool{std::make_unique(*master_semaphore, device)} { + vk::SetDeletionTimeline(master_semaphore->CurrentTick()); AcquireNewChunk(); AllocateWorkerCommandBuffer(); worker_thread = std::jthread([this](std::stop_token token) { WorkerThread(token); }); @@ -93,13 +94,87 @@ void Scheduler::DispatchWork() { } } +void Scheduler::BeginDynamicRendering(const Framebuffer* framebuffer, const DeferredClear* clear) { + const VkExtent2D render_area = framebuffer->RenderArea(); + std::array attachment_views{}; + const auto& color_views = framebuffer->ColorAttachments(); + for (size_t index = 0; index < color_views.size(); ++index) { + attachment_views[index] = color_views[index]; + } + attachment_views[8] = framebuffer->DepthAttachment(); + state.renderpass = VkRenderPass{}; + state.framebuffer = VkFramebuffer{}; + state.attachment_views = attachment_views; + state.color_resolve_views = framebuffer->ColorResolveAttachments(); + state.color_resolve_modes = framebuffer->ColorResolveModes(); + state.depth_resolve_view = framebuffer->DepthResolveAttachment(); + state.depth_resolve_mode = framebuffer->DepthResolveMode(); + state.stencil_resolve_mode = framebuffer->StencilResolveMode(); + state.discards_msaa_color = framebuffer->DiscardsMsaaColor(); + state.discards_msaa_depth = framebuffer->DiscardsMsaaDepth(); + state.render_area = render_area; + state.num_color = framebuffer->NumColorAttachments(); + state.has_depth = framebuffer->HasAspectDepthBit(); + state.has_stencil = framebuffer->HasAspectStencilBit(); + state.layer_count = framebuffer->NumLayers(); + state.rendering = true; + + if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { + const std::string render_pass_info = + fmt::format("renderArea={}x{}, numImages={}", render_area.width, render_area.height, + framebuffer->NumImages()); + GPU::Logging::GPULogger::GetInstance().LogRenderPassBegin(render_pass_info); + } + + pending_begin = true; + has_pending_begin_clear = clear != nullptr; + pending_begin_clear = clear != nullptr ? *clear : DeferredClear{}; + num_renderpass_images = framebuffer->NumImages(); + renderpass_images = framebuffer->Images(); + renderpass_image_ranges = framebuffer->ImageRanges(); +} + +void Scheduler::FlushPendingRenderPass() { + if (!pending_begin) { + return; + } + + pending_begin = false; + const bool had_clear = has_pending_begin_clear; + has_pending_begin_clear = false; + RecordDynamicBegin(had_clear ? &pending_begin_clear : nullptr); +} + +bool Scheduler::RetractUnrecordedRenderPass() { + if (!pending_begin) { + return false; + } + pending_begin = false; + if (has_pending_begin_clear) { + deferred_clear = pending_begin_clear; + has_pending_begin_clear = false; + } + state.renderpass = VkRenderPass{}; + state.framebuffer = VkFramebuffer{}; + state.attachment_views = {}; + state.rendering = false; + state.uses_transform_feedback = false; + num_renderpass_images = 0; + return true; +} + void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, const VkClearValue* clear_values, u32 clear_value_count) { - const VkFramebuffer framebuffer_handle = framebuffer->Handle(); + if (device.IsKhrDynamicRenderingSupported()) { + BeginDynamicRendering(framebuffer, nullptr); + return; + } const VkExtent2D render_area = framebuffer->RenderArea(); + const VkFramebuffer framebuffer_handle = framebuffer->Handle(); state.renderpass = renderpass; state.framebuffer = framebuffer_handle; state.render_area = render_area; + state.rendering = true; if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { const std::string render_pass_info = @@ -141,6 +216,12 @@ void Scheduler::RealizeDeferredClear() { const DeferredClear dc = deferred_clear; deferred_clear = {}; + if (device.IsKhrDynamicRenderingSupported()) { + EndRenderPass(); + BeginDynamicRendering(dc.framebuffer, &dc); + return; + } + std::array clear_values{}; u32 count = 0; const RenderPassKey& base = dc.framebuffer->RenderPassKeyBase(); @@ -195,9 +276,25 @@ void Scheduler::RequestRenderpass(const Framebuffer* framebuffer) { RealizeDeferredClear(); return; } + const VkExtent2D render_area = framebuffer->RenderArea(); + if (device.IsKhrDynamicRenderingSupported()) { + std::array attachment_views{}; + const auto& color_views = framebuffer->ColorAttachments(); + for (size_t index = 0; index < color_views.size(); ++index) { + attachment_views[index] = color_views[index]; + } + attachment_views[8] = framebuffer->DepthAttachment(); + if (state.rendering && attachment_views == state.attachment_views && + render_area.width == state.render_area.width && + render_area.height == state.render_area.height) { + return; + } + EndRenderPass(); + BeginDynamicRendering(framebuffer, nullptr); + return; + } const VkRenderPass renderpass = framebuffer->RenderPass(); const VkFramebuffer framebuffer_handle = framebuffer->Handle(); - const VkExtent2D render_area = framebuffer->RenderArea(); if (renderpass == state.renderpass && framebuffer_handle == state.framebuffer && render_area.width == state.render_area.width && render_area.height == state.render_area.height) { @@ -249,6 +346,7 @@ bool Scheduler::UpdateRescaling(bool is_rescaling) { void Scheduler::WorkerThread(std::stop_token stop_token) { Common::SetCurrentThreadName("VulkanWorker"); + Common::SetCurrentThreadPriority(Common::ThreadPriority::VeryHigh); const auto TryPopQueue{[this](auto& work) -> bool { if (work_queue.empty()) { @@ -322,6 +420,7 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se InvalidateState(); const u64 signal_value = master_semaphore->NextTick(); + vk::SetDeletionTimeline(master_semaphore->CurrentTick()); RecordWithUploadBuffer([signal_semaphore, wait_semaphore, signal_value, this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { static constexpr VkMemoryBarrier WRITE_BARRIER{ @@ -372,6 +471,155 @@ void Scheduler::InvalidateState() { state_tracker.InvalidateCommandBufferState(); } +namespace { +/// The part of a dynamic rendering begin that every pass needs. +struct DynamicRenderingBase { + std::array views; + VkExtent2D render_area; + u32 num_color; + u32 layers; + VkImageView ds_resolve_view; + VkResolveModeFlagBits depth_resolve_mode; + VkResolveModeFlagBits stencil_resolve_mode; + bool has_depth; + bool has_stencil; + bool ds_discard; +}; + +/// The part only a pass that actually clears something needs. +struct DynamicRenderingClears { + std::array color_values; + VkClearValue ds_value; + u32 color_clear_mask; + u32 color_discard_mask; + bool ds_clear; +}; + +/// Builds the rendering info and begins the instance. `resolve_views`/`resolve_modes` and `clears` +/// are null for passes with no colour resolve targets and no clears, so those tables never have to +/// be copied into the recorded command. +void IssueBeginRendering(vk::CommandBuffer cmdbuf, const DynamicRenderingBase& base, + const std::array* resolve_views, + const std::array* resolve_modes, + const DynamicRenderingClears* clears) { + std::array color_infos{}; + for (u32 index = 0; index < base.num_color; ++index) { + const bool clear_slot = + clears != nullptr && ((clears->color_clear_mask >> index) & 1u) != 0; + const VkImageView resolve_view = + resolve_views != nullptr ? (*resolve_views)[index] : VK_NULL_HANDLE; + const bool has_resolve = resolve_view != VK_NULL_HANDLE; + const bool discard_slot = + has_resolve && clears != nullptr && ((clears->color_discard_mask >> index) & 1u) != 0; + color_infos[index] = VkRenderingAttachmentInfo{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = base.views[index], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = has_resolve ? (*resolve_modes)[index] : VK_RESOLVE_MODE_NONE, + .resolveImageView = resolve_view, + .resolveImageLayout = + has_resolve ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = clear_slot ? VK_ATTACHMENT_LOAD_OP_CLEAR : VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = discard_slot ? VK_ATTACHMENT_STORE_OP_DONT_CARE + : VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = clear_slot ? clears->color_values[index] : VkClearValue{}, + }; + } + const bool ds_clear = clears != nullptr && clears->ds_clear; + const VkClearValue ds_clear_value = clears != nullptr ? clears->ds_value : VkClearValue{}; + const bool has_ds_resolve = base.ds_resolve_view != VK_NULL_HANDLE; + const VkAttachmentLoadOp ds_load_op = ds_clear ? VK_ATTACHMENT_LOAD_OP_CLEAR + : base.ds_discard ? VK_ATTACHMENT_LOAD_OP_DONT_CARE + : VK_ATTACHMENT_LOAD_OP_LOAD; + const VkAttachmentStoreOp ds_store_op = + base.ds_discard ? VK_ATTACHMENT_STORE_OP_DONT_CARE : VK_ATTACHMENT_STORE_OP_STORE; + const VkRenderingAttachmentInfo depth_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = base.views[8], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = has_ds_resolve ? base.depth_resolve_mode : VK_RESOLVE_MODE_NONE, + .resolveImageView = has_ds_resolve ? base.ds_resolve_view : VK_NULL_HANDLE, + .resolveImageLayout = has_ds_resolve ? VK_IMAGE_LAYOUT_GENERAL + : VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = ds_load_op, + .storeOp = ds_store_op, + .clearValue = ds_clear ? ds_clear_value : VkClearValue{}, + }; + // Stencil gets its own struct because its resolve mode may differ from depth's. + const VkRenderingAttachmentInfo stencil_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = base.views[8], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = has_ds_resolve ? base.stencil_resolve_mode : VK_RESOLVE_MODE_NONE, + .resolveImageView = has_ds_resolve ? base.ds_resolve_view : VK_NULL_HANDLE, + .resolveImageLayout = has_ds_resolve ? VK_IMAGE_LAYOUT_GENERAL + : VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = ds_load_op, + .storeOp = ds_store_op, + .clearValue = ds_clear ? ds_clear_value : VkClearValue{}, + }; + const VkRenderingInfo rendering_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, + .pNext = nullptr, + .flags = 0, + .renderArea = + { + .offset = {.x = 0, .y = 0}, + .extent = base.render_area, + }, + .layerCount = base.layers, + .viewMask = 0, + .colorAttachmentCount = base.num_color, + .pColorAttachments = color_infos.data(), + .pDepthAttachment = base.has_depth ? &depth_info : nullptr, + .pStencilAttachment = base.has_stencil ? &stencil_info : nullptr, + }; + cmdbuf.BeginRendering(rendering_info); +} +} // Anonymous namespace + +void Scheduler::RecordDynamicBegin(const DeferredClear* clear) { + const DynamicRenderingBase base{ + .views = state.attachment_views, + .render_area = state.render_area, + .num_color = state.num_color, + .layers = state.layer_count, + .ds_resolve_view = state.depth_resolve_view, + .depth_resolve_mode = state.depth_resolve_mode, + .stencil_resolve_mode = state.stencil_resolve_mode, + .has_depth = state.has_depth, + .has_stencil = state.has_stencil, + .ds_discard = state.discards_msaa_depth, + }; + bool has_color_resolve = false; + for (const VkImageView resolve_view : state.color_resolve_views) { + has_color_resolve = has_color_resolve || resolve_view != VK_NULL_HANDLE; + } + if (clear == nullptr && !has_color_resolve) { + // Common case. Every byte captured here is copied into the command chunk on every single + // pass begin, and the clear values plus the two resolve tables are two thirds of them. + Record([base](vk::CommandBuffer cmdbuf) { + IssueBeginRendering(cmdbuf, base, nullptr, nullptr, nullptr); + }); + return; + } + const DynamicRenderingClears clears{ + .color_values = clear != nullptr ? clear->color_values : std::array{}, + .ds_value = clear != nullptr ? clear->depth_stencil_value : VkClearValue{}, + .color_clear_mask = clear != nullptr ? clear->color_clear_mask : 0u, + .color_discard_mask = + clear != nullptr && state.discards_msaa_color ? clear->color_clear_mask : 0u, + .ds_clear = clear != nullptr && clear->depth_stencil, + }; + Record([base, resolve_views = state.color_resolve_views, + resolve_modes = state.color_resolve_modes, clears](vk::CommandBuffer cmdbuf) { + IssueBeginRendering(cmdbuf, base, &resolve_views, &resolve_modes, &clears); + }); +} + void Scheduler::EndPendingOperations() { query_cache->CounterReset(VideoCommon::QueryType::ZPassPixelCount64); EndRenderPass(); @@ -380,7 +628,10 @@ void Scheduler::EndPendingOperations() { void Scheduler::EndRenderPass() { RealizeDeferredClear(); - if (!state.renderpass) { + if (query_cache) { + query_cache->NotifySegment(false); + } + if (!state.rendering) { return; } @@ -393,14 +644,24 @@ void Scheduler::EndRenderPass() } query_cache->CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, false); - query_cache->NotifySegment(false); + + if (pending_begin) { + if (!has_pending_begin_clear) { + RetractUnrecordedRenderPass(); + return; + } + FlushPendingRenderPass(); + } Record([num_images = num_renderpass_images, images = renderpass_images, ranges = renderpass_image_ranges, - has_transform_feedback = device.IsExtTransformFeedbackSupported()]( + has_transform_feedback = state.uses_transform_feedback, + dynamic_rendering = device.IsKhrDynamicRenderingSupported()]( vk::CommandBuffer cmdbuf) { + const bool use_sync2 = cmdbuf.HasPipelineBarrier2(); std::array barriers; + std::array barriers2; for (size_t i = 0; i < num_images; ++i) { const VkImageSubresourceRange& range = ranges[i]; const bool is_color = (range.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) != 0; @@ -418,15 +679,57 @@ void Scheduler::EndRenderPass() src_access |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - barriers[i] = VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + if (!use_sync2) { + barriers[i] = VkImageMemoryBarrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = src_access, + .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT + | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT + | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT + | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = images[i], + .subresourceRange = range, + }; + continue; + } + + VkPipelineStageFlags2 src_stage = 0; + VkAccessFlags2 dst_access = + VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT; + if (is_color) { + src_stage |= VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT; + dst_access |= VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT; + } + if (is_depth_stencil) { + src_stage |= VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT + | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT; + dst_access |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT + | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + } + if (!is_color && !is_depth_stencil) { + src_stage = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT + | VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT + | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT; + dst_access |= VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT + | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT + | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT + | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + } + barriers2[i] = VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = src_access, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT - | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT - | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, + .srcStageMask = src_stage, + .srcAccessMask = static_cast(src_access), + .dstStageMask = VK_PIPELINE_STAGE_2_ALL_GRAPHICS_BIT + | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = dst_access, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -435,10 +738,19 @@ void Scheduler::EndRenderPass() .subresourceRange = range, }; } - cmdbuf.EndRenderPass(); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, nullptr, nullptr, vk::Span(barriers.data(), num_images)); + if (dynamic_rendering) { + cmdbuf.EndRendering(); + } else { + cmdbuf.EndRenderPass(); + } + if (use_sync2) { + cmdbuf.PipelineBarrier2(0, nullptr, nullptr, + vk::Span(barriers2.data(), num_images)); + } else { + cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, + 0, nullptr, nullptr, vk::Span(barriers.data(), num_images)); + } if (has_transform_feedback) { static constexpr VkMemoryBarrier XFB_OUTPUT_BARRIER{ .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, @@ -453,6 +765,10 @@ void Scheduler::EndRenderPass() }); state.renderpass = VkRenderPass{}; + state.framebuffer = VkFramebuffer{}; + state.attachment_views = {}; + state.rendering = false; + state.uses_transform_feedback = false; num_renderpass_images = 0; } diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index 5c4af8c146..5a48b50a11 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -71,7 +71,12 @@ public: /// Returns true when a render pass is currently active in the scheduler state. bool IsRenderPassActive() const { - return state.renderpass != VK_NULL_HANDLE; + return state.rendering; + } + + /// Flags that transform feedback writes have been recorded since the last render pass end. + void MarkTransformFeedbackUsed() noexcept { + state.uses_transform_feedback = true; } /// Update the pipeline to the current execution context. @@ -107,12 +112,17 @@ public: template requires std::is_invocable_v void Record(T&& c) { + if (pending_begin) { + FlushPendingRenderPass(); + } this->RecordWithUploadBuffer( [command = std::move(c)](vk::CommandBuffer cmdbuf, vk::CommandBuffer) { command(cmdbuf); }); } + bool RetractUnrecordedRenderPass(); + /// Returns the current command buffer tick. [[nodiscard]] u64 CurrentTick() const noexcept { return master_semaphore->CurrentTick(); @@ -131,33 +141,16 @@ public: } master_semaphore->Wait(tick); } - if (Settings::values.use_speed_limit.GetValue() && target_fps > 0.0) { - auto now = std::chrono::steady_clock::now(); - if (last_target_fps != target_fps) { - frame_interval = std::chrono::duration_cast(std::chrono::duration(1.0 / target_fps)); - max_frame_count = static_cast(0.1 * target_fps); - last_target_fps = target_fps; - frame_counter = 0; - start_time = now; - } - frame_counter++; - auto target_time = start_time + frame_interval * frame_counter; - if (target_time >= now) { - auto sleep_time = target_time - now; - if (sleep_time > std::chrono::milliseconds(15)) { - std::this_thread::sleep_for(sleep_time - std::chrono::milliseconds(1)); - } - while (std::chrono::steady_clock::now() < target_time) { - std::this_thread::yield(); - } - } else if (frame_counter > max_frame_count) { - frame_counter = 0; - start_time = now; - } - } + ApplyFramePacing(target_fps); + } + + void WaitSubmitted(u64 tick, double target_fps = 0.0) { + if (tick > 0 && tick < master_semaphore->CurrentTick()) { + master_semaphore->Wait(tick); + } + ApplyFramePacing(target_fps); } - /// Returns the master timeline semaphore. [[nodiscard]] MasterSemaphore& GetMasterSemaphore() const noexcept { return *master_semaphore; } @@ -165,6 +158,35 @@ public: std::mutex submit_mutex; private: + void ApplyFramePacing(double target_fps) { + if (!Settings::values.use_speed_limit.GetValue() || target_fps <= 0.0) { + return; + } + auto now = std::chrono::steady_clock::now(); + if (last_target_fps != target_fps) { + frame_interval = std::chrono::duration_cast( + std::chrono::duration(1.0 / target_fps)); + max_frame_count = static_cast(0.1 * target_fps); + last_target_fps = target_fps; + frame_counter = 0; + start_time = now; + } + frame_counter++; + auto target_time = start_time + frame_interval * frame_counter; + if (target_time >= now) { + auto sleep_time = target_time - now; + if (sleep_time > std::chrono::milliseconds(15)) { + std::this_thread::sleep_for(sleep_time - std::chrono::milliseconds(1)); + } + while (std::chrono::steady_clock::now() < target_time) { + std::this_thread::yield(); + } + } else if (frame_counter > max_frame_count) { + frame_counter = 0; + start_time = now; + } + } + class Command { public: virtual ~Command() = default; @@ -244,17 +266,31 @@ private: size_t command_offset = 0; bool submit = false; - alignas(std::max_align_t) std::array data{}; + alignas(std::max_align_t) std::array data{}; }; struct State { VkRenderPass renderpass{}; VkFramebuffer framebuffer{}; + std::array attachment_views{}; + std::array color_resolve_views{}; + std::array color_resolve_modes{}; + VkImageView depth_resolve_view{}; + VkResolveModeFlagBits depth_resolve_mode = VK_RESOLVE_MODE_NONE; + VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_NONE; VkExtent2D render_area = {0, 0}; GraphicsPipeline* graphics_pipeline = nullptr; + bool rendering = false; + bool discards_msaa_color = false; + bool discards_msaa_depth = false; + u32 num_color = 0; + bool has_depth = false; + bool has_stencil = false; + u32 layer_count = 1; bool is_rescaling = false; bool rescaling_defined = false; bool needs_state_enable_refresh = false; + bool uses_transform_feedback = false; }; struct DeferredClear { @@ -269,6 +305,9 @@ private: void BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, const VkClearValue* clear_values, u32 clear_value_count); + /// Begins a dynamic rendering pass, optionally realizing a deferred clear via load ops. + void BeginDynamicRendering(const Framebuffer* framebuffer, const DeferredClear* clear); + /// If a deferred clear is pending. void RealizeDeferredClear(); @@ -282,6 +321,10 @@ private: void EndPendingOperations(); + void RecordDynamicBegin(const DeferredClear* clear); + + void FlushPendingRenderPass(); + void EndRenderPass(); void AcquireNewChunk(); @@ -299,6 +342,10 @@ private: DeferredClear deferred_clear; + bool pending_begin = false; + bool has_pending_begin_clear = false; + DeferredClear pending_begin_clear; + std::unique_ptr chunk; std::function on_submit; diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index b03449f9f7..dfa374fe48 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -84,10 +84,16 @@ StagingBufferPool::StagingBufferPool(const Device& device_, MemoryAllocator& mem if (device.IsExtTransformFeedbackSupported()) { stream_ci.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + stream_ci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } stream_buffer = memory_allocator.CreateBuffer(stream_ci, MemoryUsage::Stream); if (device.HasDebuggingToolAttached()) { stream_buffer.SetObjectNameEXT("Stream Buffer"); } + if (device.IsBufferDeviceAddressSupported()) { + stream_buffer_address = device.GetLogical().GetBufferDeviceAddress(*stream_buffer); + } stream_pointer = stream_buffer.Mapped(); ASSERT_MSG(!stream_pointer.empty(), "Stream buffer must be host visible!"); } @@ -149,6 +155,7 @@ StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) { iterator = Common::AlignUp(iterator + size, MAX_ALIGNMENT); return StagingBufferRef{ .buffer = *stream_buffer, + .device_address = stream_buffer_address, .offset = static_cast(offset), .mapped_span = stream_pointer.subspan(offset, size), .usage{}, @@ -212,14 +219,22 @@ StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, MemoryUsage if (device.IsExtTransformFeedbackSupported()) { buffer_ci.usage |= VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_BUFFER_BIT_EXT; } + if (device.IsBufferDeviceAddressSupported()) { + buffer_ci.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + } vk::Buffer buffer = memory_allocator.CreateBuffer(buffer_ci, usage); if (device.HasDebuggingToolAttached()) { ++buffer_index; buffer.SetObjectNameEXT(fmt::format("Staging Buffer {}", buffer_index).c_str()); } const std::span mapped_span = buffer.Mapped(); + const VkDeviceAddress buffer_address = + device.IsBufferDeviceAddressSupported() + ? device.GetLogical().GetBufferDeviceAddress(*buffer) + : VkDeviceAddress{}; StagingBuffer& entry = GetCache(usage)[log2_size].entries.emplace_back(StagingBuffer{ .buffer = std::move(buffer), + .device_address = buffer_address, .mapped_span = mapped_span, .usage = usage, .log2_level = log2_size, @@ -252,25 +267,62 @@ void StagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, size_t log2) { constexpr size_t deletions_per_tick = 16; auto& staging = cache[log2]; auto& entries = staging.entries; - const size_t old_size = entries.size(); + if (entries.empty()) { + staging.delete_index = 0; + staging.iterate_index = 0; + return; + } const auto is_deletable = [this](const StagingBuffer& entry) { return scheduler.IsFree(entry.tick); }; - const size_t begin_offset = staging.delete_index; - const size_t end_offset = (std::min)(begin_offset + deletions_per_tick, old_size); + const size_t begin_offset = (std::min)(staging.delete_index, entries.size()); + const size_t end_offset = (std::min)(begin_offset + deletions_per_tick, entries.size()); const auto begin = entries.begin() + begin_offset; const auto end = entries.begin() + end_offset; - entries.erase(std::remove_if(begin, end, is_deletable), end); + const auto surviving_end = std::remove_if(begin, end, is_deletable); + const size_t removed = static_cast(std::distance(surviving_end, end)); + entries.erase(surviving_end, end); - const size_t new_size = entries.size(); - staging.delete_index += deletions_per_tick; - if (staging.delete_index >= new_size) { + staging.delete_index = end_offset - removed; + if (staging.delete_index >= entries.size()) { staging.delete_index = 0; } - if (staging.iterate_index > new_size) { + if (staging.iterate_index > entries.size()) { staging.iterate_index = 0; } } +u64 StagingBufferPool::ReclaimMemory(u64 target_bytes) { + u64 freed = 0; + const auto is_deletable = [this](const StagingBuffer& entry) { + return scheduler.IsFree(entry.tick); + }; + const auto reclaim_cache = [&](StagingBuffersCache& cache) { + for (size_t level = NUM_LEVELS; level-- > 0 && freed < target_bytes;) { + auto& staging = cache[level]; + auto& entries = staging.entries; + if (entries.empty()) { + continue; + } + const u64 entry_bytes = 1ULL << level; + auto it = entries.begin(); + while (it != entries.end() && freed < target_bytes) { + if (is_deletable(*it)) { + it = entries.erase(it); + freed += entry_bytes; + } else { + ++it; + } + } + staging.delete_index = 0; + staging.iterate_index = 0; + } + }; + reclaim_cache(device_local_cache); + reclaim_cache(upload_cache); + reclaim_cache(download_cache); + return freed; +} + } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h index f63a203272..9d05d1cca6 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later @@ -18,6 +21,7 @@ class Scheduler; struct StagingBufferRef { VkBuffer buffer; + VkDeviceAddress device_address; VkDeviceSize offset; std::span mapped_span; MemoryUsage usage; @@ -42,6 +46,8 @@ public: void TickFrame(); + u64 ReclaimMemory(u64 target_bytes); + private: struct StreamBufferCommit { size_t upper_bound; @@ -50,6 +56,7 @@ private: struct StagingBuffer { vk::Buffer buffer; + VkDeviceAddress device_address; std::span mapped_span; MemoryUsage usage; u32 log2_level; @@ -60,6 +67,7 @@ private: StagingBufferRef Ref() const noexcept { return { .buffer = *buffer, + .device_address = device_address, .offset = 0, .mapped_span = mapped_span, .usage = usage, @@ -103,6 +111,7 @@ private: Scheduler& scheduler; vk::Buffer stream_buffer; + VkDeviceAddress stream_buffer_address{}; std::span stream_pointer; VkDeviceSize stream_buffer_size; VkDeviceSize region_size; diff --git a/src/video_core/renderer_vulkan/vk_swapchain.cpp b/src/video_core/renderer_vulkan/vk_swapchain.cpp index fdbad16721..507c62b8f0 100644 --- a/src/video_core/renderer_vulkan/vk_swapchain.cpp +++ b/src/video_core/renderer_vulkan/vk_swapchain.cpp @@ -9,10 +9,6 @@ #include #include -#ifdef __ANDROID__ -#include -#endif - #include "common/logging.h" #include "common/settings.h" #include "common/settings_enums.h" @@ -176,34 +172,26 @@ bool Swapchain::AcquireNextImage() { break; } - const auto wait_with_frame_pacing = [this] { +#ifdef __ANDROID__ + scheduler.WaitSubmitted(resource_ticks[image_index]); +#else switch (Settings::values.frame_pacing_mode.GetValue()) { case Settings::FramePacingMode::Target_Auto: - scheduler.Wait(resource_ticks[image_index]); + scheduler.WaitSubmitted(resource_ticks[image_index]); break; case Settings::FramePacingMode::Target_30: - scheduler.Wait(resource_ticks[image_index], 30.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 30.0); break; case Settings::FramePacingMode::Target_60: - scheduler.Wait(resource_ticks[image_index], 60.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 60.0); break; case Settings::FramePacingMode::Target_90: - scheduler.Wait(resource_ticks[image_index], 90.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 90.0); break; case Settings::FramePacingMode::Target_120: - scheduler.Wait(resource_ticks[image_index], 120.0); + scheduler.WaitSubmitted(resource_ticks[image_index], 120.0); break; } - }; - -#ifdef __ANDROID__ - if (android_get_device_api_level() >= 30) { - scheduler.Wait(resource_ticks[image_index]); - } else { - wait_with_frame_pacing(); - } -#else - wait_with_frame_pacing(); #endif resource_ticks[image_index] = scheduler.CurrentTick(); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index 44a4227a19..eac67335f0 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -54,6 +54,8 @@ using VideoCore::Surface::SurfaceType; namespace { constexpr bool ENABLE_MSAA_RESOLVE_CONSUME = true; constexpr bool ENABLE_MSAA_COLOR_DISCARD = true; +constexpr bool ENABLE_MSAA_DEPTH_DISCARD = true; +constexpr bool ENABLE_MSAA_DEPTH_RESOLVE = false; constexpr VkBorderColor ConvertBorderColor(const std::array& color) { if (color == std::array{0, 0, 0, 0}) { @@ -144,11 +146,6 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { info.size.depth == 1; } -[[nodiscard]] bool WillUseWidenedAstcFormat(const Device& device, const ImageInfo& info) { - return WillUseAcceleratedAstcDecode(device, info) && - !VideoCore::Surface::IsPixelFormatSRGB(info.format); -} - [[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info, std::optional format_override = {}) { auto format_info = @@ -212,7 +209,11 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { return device.IsFormatSupported(view_format, VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT, FormatType::Optimal); }); - if (has_storage_compatible_view) { + const bool storage_allowed_for_samples = + image_ci.samples == VK_SAMPLE_COUNT_1_BIT || + (device.GetStorageImageSampleCounts() & + static_cast(image_ci.samples)) != 0; + if (has_storage_compatible_view && storage_allowed_for_samples) { image_ci.usage |= VK_IMAGE_USAGE_STORAGE_BIT; } @@ -269,6 +270,10 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { } } +[[nodiscard]] bool IsLdrAstcFormat(VkFormat format) { + return format >= VK_FORMAT_ASTC_4x4_UNORM_BLOCK && format <= VK_FORMAT_ASTC_12x12_SRGB_BLOCK; +} + [[nodiscard]] VkImageAspectFlags ImageViewAspectMask(const VideoCommon::ImageViewInfo& info) { if (info.IsRenderTarget()) { return ImageAspectMask(info.format); @@ -896,6 +901,15 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const 0, nullptr, nullptr, write_barriers); }); } + +[[nodiscard]] bool CanBlitNatively(const Device& device, PixelFormat format) { + static constexpr auto OPTIMAL_FORMAT = FormatType::Optimal; + static constexpr VkFormatFeatureFlags BLIT_USAGE = + VK_FORMAT_FEATURE_BLIT_SRC_BIT | VK_FORMAT_FEATURE_BLIT_DST_BIT; + const VkFormat vk_format = + MaxwellToVK::SurfaceFormat(device, OPTIMAL_FORMAT, false, format).format; + return device.IsFormatSupported(vk_format, BLIT_USAGE, OPTIMAL_FORMAT); +} } // Anonymous namespace TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& scheduler_, @@ -934,6 +948,10 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched bl3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue); } + bl2d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); + bl3db_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); } void TextureCacheRuntime::Finish() { @@ -996,13 +1014,21 @@ VkBuffer TextureCacheRuntime::GetTemporaryBuffer(size_t needed_size) { } VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, VkFormat format, - VkExtent2D extent, u32 layers) { + VkExtent2D extent, u32 layers, + VkImageAspectFlags aspect) { + const bool is_depth_stencil = + (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0; ResolveShadow& shadow = resolve_shadows[msaa_image]; if (shadow.image && shadow.format == format && shadow.extent.width == extent.width && shadow.extent.height == extent.height && shadow.layers == layers) { shadow.up_to_date = true; return *shadow.view; } + const VkImageUsageFlags shadow_usage = + static_cast(is_depth_stencil + ? VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT + : VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT) | + VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_SRC_BIT; shadow.image = memory_allocator.CreateImage(VkImageCreateInfo{ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .pNext = nullptr, @@ -1014,8 +1040,7 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk .arrayLayers = layers, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, - .usage = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | - VK_IMAGE_USAGE_TRANSFER_SRC_BIT, + .usage = shadow_usage, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -1030,7 +1055,7 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk .format = format, .components{}, .subresourceRange{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect, .baseMipLevel = 0, .levelCount = 1, .baseArrayLayer = 0, @@ -1041,6 +1066,38 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk shadow.extent = extent; shadow.layers = layers; shadow.up_to_date = true; + if (device.IsKhrDynamicRenderingSupported()) { + const VkAccessFlags dst_access = + is_depth_stencil ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + : VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + const VkPipelineStageFlags dst_stage = + is_depth_stencil ? VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT + : VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + scheduler.RecordWithUploadBuffer( + [image = *shadow.image, layers, aspect, dst_access, dst_stage]( + vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + const VkImageMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = dst_access, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = aspect, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = layers, + }, + }; + upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, dst_stage, 0, + barrier); + }); + } return *shadow.view; } @@ -1228,27 +1285,19 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst blit_image_helper.ResolveDepthStencil(dst_framebuffer, src, dst_region, src_region); return; } - if (aspect_mask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { - const auto format = src.format; - const auto can_blit_depth_stencil = [this, format] { - switch (format) { - case VideoCore::Surface::PixelFormat::D24_UNORM_S8_UINT: - case VideoCore::Surface::PixelFormat::S8_UINT_D24_UNORM: - return device.IsBlitDepth24Stencil8Supported(); - case VideoCore::Surface::PixelFormat::D32_FLOAT_S8_UINT: - return device.IsBlitDepth32Stencil8Supported(); - default: - UNREACHABLE(); - } - }(); - // Use shader-based depth/stencil blits if hardware doesn't support the format - // Note: MSAA resolves (MSAA->single) use vkCmdResolveImage which works fine - if (!can_blit_depth_stencil) { - UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa); - blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region, - filter, operation); + static constexpr VkImageAspectFlags DEPTH_STENCIL_ASPECTS = + VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT; + if ((aspect_mask & DEPTH_STENCIL_ASPECTS) != 0 && !CanBlitNatively(device, src.format)) { + if (aspect_mask != DEPTH_STENCIL_ASPECTS) { + UNIMPLEMENTED_MSG("Host cannot blit format {} and no helper path exists for aspect " + "mask 0x{:x}", + src.format, aspect_mask); return; } + UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa); + blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region, filter, + operation); + return; } ASSERT(!(is_dst_msaa && !is_src_msaa)); ASSERT(operation == Fermi2D::Operation::SrcCopy); @@ -1353,7 +1402,7 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst } void TextureCacheRuntime::ConvertImage(Framebuffer* dst, ImageView& dst_view, ImageView& src_view) { - if (!dst->RenderPass()) { + if (!dst->RenderPass() && !device.IsKhrDynamicRenderingSupported()) { return; } @@ -1641,11 +1690,6 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, std::span copies) { const bool msaa_to_non_msaa = src.info.num_samples > 1 && dst.info.num_samples == 1; const u32 num_samples = msaa_to_non_msaa ? src.info.num_samples : dst.info.num_samples; - if (dst.AspectMask() != VK_IMAGE_ASPECT_COLOR_BIT || - VideoCore::Surface::IsPixelFormatInteger(dst.info.format)) { - UNIMPLEMENTED_MSG("Copying images with different samples is not supported."); - return; - } if (ENABLE_MSAA_RESOLVE_CONSUME && msaa_to_non_msaa && copies.size() == 1 && src.info.format == dst.info.format) { const VideoCommon::ImageCopy& copy = copies.front(); @@ -1656,16 +1700,19 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, static_cast(copy.extent.height) <= shadow->extent.height) { const VkImage shadow_image = *shadow->image; const VkImage dst_image = dst.Handle(); + const VkImageAspectFlags aspect = dst.AspectMask(); + const bool is_depth_stencil = + (aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0; const VkImageCopy region{ .srcSubresource{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect, .mipLevel = 0, .baseArrayLayer = static_cast(copy.src_subresource.base_layer), .layerCount = static_cast(copy.src_subresource.num_layers), }, .srcOffset = {0, 0, 0}, .dstSubresource{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .aspectMask = aspect, .mipLevel = static_cast(copy.dst_subresource.base_level), .baseArrayLayer = static_cast(copy.dst_subresource.base_layer), .layerCount = static_cast(copy.dst_subresource.num_layers), @@ -1674,34 +1721,38 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, .extent = {copy.extent.width, copy.extent.height, 1}, }; scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([shadow_image, dst_image, region](vk::CommandBuffer cmdbuf) { + scheduler.Record([shadow_image, dst_image, region, aspect, + is_depth_stencil](vk::CommandBuffer cmdbuf) { + const VkAccessFlags attachment_write = + is_depth_stencil ? VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT + : VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; const std::array pre_barriers{ VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .srcAccessMask = attachment_write, .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = shadow_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, VkImageMemoryBarrier{ .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, + .srcAccessMask = + VK_ACCESS_SHADER_WRITE_BIT | attachment_write | + VK_ACCESS_TRANSFER_WRITE_BIT, .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, }; @@ -1716,7 +1767,7 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = shadow_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, VkImageMemoryBarrier{ @@ -1726,17 +1777,20 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = dst_image, - .subresourceRange{VK_IMAGE_ASPECT_COLOR_BIT, 0, VK_REMAINING_MIP_LEVELS, 0, + .subresourceRange{aspect, 0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS}, }, }; cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | + VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, nullptr, nullptr, pre_barriers); @@ -1749,10 +1803,36 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, return; } } + if (dst.AspectMask() != VK_IMAGE_ASPECT_COLOR_BIT || + VideoCore::Surface::IsPixelFormatInteger(dst.info.format)) { + const u64 key{(static_cast(dst.AspectMask()) << 32) | + static_cast(dst.info.format)}; + if (unsupported_msaa_resolves.insert(key).second) { + LOG_WARNING(Render_Vulkan, + "MSAA resolve unsupported: format={}, aspect={:#x}, samples {}->{}", + dst.info.format, dst.AspectMask(), src.info.num_samples, + dst.info.num_samples); + } + return; + } blit_image_helper.CopyMSAA(render_pass_cache, dst.Handle(), dst.info.format, src.Handle(), src.info.format, num_samples, copies, msaa_to_non_msaa); } +u64 TextureCacheRuntime::CurrentSyncPoint() const noexcept { + return scheduler.CurrentTick(); +} + +u64 TextureCacheRuntime::CompletedSyncPoint() const { + auto& master_semaphore = scheduler.GetMasterSemaphore(); + master_semaphore.Refresh(); + return master_semaphore.KnownGpuTick(); +} + +void TextureCacheRuntime::WaitSyncPoint(u64 sync_point) { + scheduler.Wait(sync_point); +} + u64 TextureCacheRuntime::GetDeviceLocalMemory() const { return device.GetDeviceLocalMemory(); } @@ -1761,6 +1841,10 @@ u64 TextureCacheRuntime::GetDeviceMemoryUsage() const { return device.GetDeviceMemoryUsage(); } +u64 TextureCacheRuntime::GetDeviceAllocationUsage() const { + return device.GetMemoryBudgetInfo().allocation_bytes; +} + bool TextureCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } @@ -1770,6 +1854,7 @@ std::optional TextureCacheRuntime::GetSamplerHeapBudget() const { } void TextureCacheRuntime::TickFrame() { + device.TickAllocatorFrame(); std::erase_if(pending_msaa_images, [this](const auto& pending) { return scheduler.IsFree(pending.first); }); @@ -1780,12 +1865,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, runtime{&runtime_}, original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info, - WillUseWidenedAstcFormat(runtime_.device, info) - ? std::span{} - : runtime->ViewFormats(info.format), - WillUseWidenedAstcFormat(runtime_.device, info) - ? std::make_optional(VK_FORMAT_R32G32B32A32_SFLOAT) - : std::nullopt)), + runtime->ViewFormats(info.format))), aspect_mask(ImageAspectMask(info.format)) { if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { switch (Settings::values.accelerate_astc.GetValue()) { @@ -1800,7 +1880,13 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu default: break; } - flags |= VideoCommon::ImageFlagBits::Converted; + } else if (runtime->bl2d_unswizzle_pass && + BlockLinearUnswizzle2DPass::IsSupported(info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + flags |= VideoCommon::ImageFlagBits::CostlyLoad; + } else if (runtime->bl3db_unswizzle_pass && + BlockLinearUnswizzle3DBufferPass::IsSupported(runtime->device, info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; flags |= VideoCommon::ImageFlagBits::CostlyLoad; } if (IsPixelFormatBCn(info.format) && !runtime->device.IsOptimalBcnSupported()) { @@ -1816,9 +1902,7 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Uncompressed) { const auto& device = runtime->device.GetLogical(); - const VkFormat storage_format = WillUseWidenedAstcFormat(runtime->device, info) - ? VK_FORMAT_R32G32B32A32_SFLOAT - : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; for (s32 level = 0; level < info.resources.levels; ++level) { storage_image_views[level] = MakeStorageView(device, level, *original_image, storage_format); @@ -1892,9 +1976,11 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, ScaleDown(true); } - const bool wants_msaa_upload = info.num_samples > 1 - && (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0 + const bool is_color_upload = (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0 && !VideoCore::Surface::IsPixelFormatInteger(info.format); + const bool is_depth_upload = (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0; + const bool wants_msaa_upload = + info.num_samples > 1 && (is_color_upload || is_depth_upload); if (wants_msaa_upload) { ImageInfo temp_info = info; @@ -1933,10 +2019,10 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, image_copies.push_back(image_copy); } + runtime->TransitionImageLayout(*this); runtime->blit_image_helper.CopyMSAA(runtime->render_pass_cache, Handle(), info.format, temp_vk_image, info.format, info.num_samples, image_copies, false); - initialized = true; runtime->pending_msaa_images.emplace_back(scheduler->CurrentTick(), std::move(temp_image)); if (is_rescaled) { @@ -1947,6 +2033,9 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, if (info.num_samples > 1) { LOG_WARNING(Render_Vulkan, "MSAA upload not implemented for format {}", info.format); + if (runtime != nullptr) { + runtime->TransitionImageLayout(*this); + } if (is_rescaled) { ScaleUp(); } @@ -2204,9 +2293,7 @@ VkImageView Image::StorageImageView(s32 level) noexcept { auto format_info = MaxwellToVK::SurfaceFormat(runtime->device, FormatType::Optimal, true, info.format); if (WillUseAcceleratedAstcDecode(runtime->device, info)) { - format_info.format = WillUseWidenedAstcFormat(runtime->device, info) - ? VK_FORMAT_R32G32B32A32_SFLOAT - : VK_FORMAT_A8B8G8R8_UNORM_PACK32; + format_info.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; } view = MakeStorageView(runtime->device.GetLogical(), level, *(this->*current_image), format_info.format); @@ -2382,11 +2469,7 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI SanitizeDepthStencilSwizzle(swizzle, device->SupportsDepthStencilSwizzleOne()); } } - uses_widened_astc_format = WillUseWidenedAstcFormat(*device, image.info); auto format_info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); - if (uses_widened_astc_format) { - format_info.format = VK_FORMAT_R32G32B32A32_SFLOAT; - } if (device->ApiVersion() >= VK_API_VERSION_1_3) { const VkFormatProperties3 properties3 = device->GetPhysical().GetFormatProperties3(format_info.format); @@ -2404,9 +2487,18 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI .pNext = nullptr, .usage = clamped_view_usage, }; + const VkImageViewASTCDecodeModeEXT astc_decode_mode{ + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_ASTC_DECODE_MODE_EXT, + .pNext = &image_view_usage, + .decodeMode = VK_FORMAT_R8G8B8A8_UNORM, + }; + const void* view_next = &image_view_usage; + if (device->IsExtAstcDecodeModeSupported() && IsLdrAstcFormat(format_info.format)) { + view_next = &astc_decode_mode; + } const VkImageViewCreateInfo create_info{ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, - .pNext = &image_view_usage, + .pNext = view_next, .flags = 0, .image = image.Handle(), .viewType = VkImageViewType{}, @@ -2529,14 +2621,12 @@ VkImageView ImageView::StorageView(Shader::TextureType texture_type, Shader::ImageFormat image_format) { if (image_handle) { if (image_format == Shader::ImageFormat::Typeless) { - if (!typeless_storage_view) { + auto& view{typeless_storage_views[static_cast(texture_type)]}; + if (!view) { auto info = MaxwellToVK::SurfaceFormat(*device, FormatType::Optimal, true, format); - if (uses_widened_astc_format) { - info.format = VK_FORMAT_R32G32B32A32_SFLOAT; - } - typeless_storage_view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type); + view = MakeView(info.format, VK_IMAGE_ASPECT_COLOR_BIT, texture_type); } - return *typeless_storage_view; + return *view; } const bool is_signed = image_format == Shader::ImageFormat::R8_SINT || image_format == Shader::ImageFormat::R16_SINT; @@ -2683,7 +2773,7 @@ Framebuffer::Framebuffer(TextureCacheRuntime& runtime, std::spanSamples(); + color_attachments[index] = color_buffer->RenderTarget(); + color_attachment_formats[index] = + MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, + color_buffer->format) + .format; + num_color_attachments = static_cast(index + 1); ++num_images; } const size_t num_colors = attachments.size(); @@ -2741,6 +2837,11 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, const VkImageSubresourceRange subresource_range = MakeSubresourceRange(depth_buffer); image_ranges[num_images] = subresource_range; samples = depth_buffer->Samples(); + depth_attachment = depth_buffer->RenderTarget(); + depth_attachment_format = + MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, + depth_buffer->format) + .format; ++num_images; has_depth = (subresource_range.aspectMask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0; has_stencil = (subresource_range.aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) != 0; @@ -2752,10 +2853,23 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, samples != VK_SAMPLE_COUNT_1_BIT && num_colors > 0 && runtime.device.IsTiler(); renderpass_key.resolve_color = do_resolve_color; + const VkResolveModeFlagBits depth_mode = runtime.device.GetDepthResolveMode(); + const VkResolveModeFlagBits stencil_mode = runtime.device.GetStencilResolveMode(); + const bool resolve_modes_compatible = + depth_mode != VK_RESOLVE_MODE_NONE && + (!has_stencil || (stencil_mode != VK_RESOLVE_MODE_NONE && + (runtime.device.IsIndependentResolveSupported() || + depth_mode == stencil_mode))); + const bool msaa_depth = + samples != VK_SAMPLE_COUNT_1_BIT && has_depth && runtime.device.IsTiler(); + const bool do_resolve_depth = ENABLE_MSAA_DEPTH_RESOLVE && msaa_depth && + runtime.device.IsKhrDynamicRenderingSupported() && + resolve_modes_compatible; + discard_msaa_color = ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_COLOR_DISCARD && do_resolve_color; + discard_msaa_depth = ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_DEPTH_DISCARD && msaa_depth; - renderpass = runtime.render_pass_cache.Get(renderpass_key); render_pass_key = renderpass_key; render_pass_cache = &runtime.render_pass_cache; render_area.width = (std::min)(render_area.width, width); @@ -2770,10 +2884,15 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, } const VkFormat vk_format = MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, format).format; + color_resolve_modes[index] = VideoCore::Surface::IsPixelFormatInteger(format) + ? VK_RESOLVE_MODE_SAMPLE_ZERO_BIT + : VK_RESOLVE_MODE_AVERAGE_BIT; if (ENABLE_MSAA_RESOLVE_CONSUME) { const VkImage msaa_image = images[rt_map[index]]; - attachments.push_back(runtime.GetOrCreateResolveShadow(msaa_image, vk_format, - render_area, layers)); + const VkImageView shadow_view = runtime.GetOrCreateResolveShadow( + msaa_image, vk_format, render_area, layers, VK_IMAGE_ASPECT_COLOR_BIT); + color_resolve_attachments[index] = shadow_view; + attachments.push_back(shadow_view); continue; } VkImageCreateInfo resolve_ci{ @@ -2812,13 +2931,61 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, .layerCount = layers, }, }); + if (runtime.device.IsKhrDynamicRenderingSupported()) { + runtime.scheduler.RecordWithUploadBuffer( + [image = *resolve_image, layers](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + const VkImageMemoryBarrier barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + .pNext = nullptr, + .srcAccessMask = 0, + .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .baseMipLevel = 0, + .levelCount = 1, + .baseArrayLayer = 0, + .layerCount = layers, + }, + }; + upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, barrier); + }); + } + color_resolve_attachments[index] = *resolve_view; attachments.push_back(*resolve_view); resolve_images.push_back(std::move(resolve_image)); resolve_image_views.push_back(std::move(resolve_view)); } } + if (do_resolve_depth && depth_buffer != nullptr) { + const u32 layers = static_cast((std::max)(num_layers, 1)); + VkImageAspectFlags aspect = 0; + if (has_depth) { + aspect |= VK_IMAGE_ASPECT_DEPTH_BIT; + } + if (has_stencil) { + aspect |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + depth_resolve_attachment = + runtime.GetOrCreateResolveShadow(depth_buffer->ImageHandle(), depth_attachment_format, + render_area, layers, aspect); + depth_resolve_mode = depth_mode; + stencil_resolve_mode = has_stencil ? stencil_mode : VK_RESOLVE_MODE_NONE; + } + num_color_buffers = static_cast(num_colors); + layer_count = static_cast((std::max)(num_layers, 1)); + if (runtime.device.IsKhrDynamicRenderingSupported()) { + return; + } + renderpass = runtime.render_pass_cache.Get(renderpass_key); framebuffer = runtime.device.GetLogical().CreateFramebuffer({ .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, .pNext = nullptr, @@ -2828,7 +2995,7 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, .pAttachments = attachments.data(), .width = render_area.width, .height = render_area.height, - .layers = static_cast((std::max)(num_layers, 1)), + .layers = layer_count, }); } @@ -2844,6 +3011,53 @@ VkRenderPass Framebuffer::RenderPassVariant(u32 color_clear_mask, bool depth_ste return render_pass_cache->Get(key); } +void Framebuffer::BeginRendering(vk::CommandBuffer cmdbuf) const { + std::array color_attachment_infos{}; + for (size_t index = 0; index < num_color_attachments; ++index) { + color_attachment_infos[index] = VkRenderingAttachmentInfo{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = color_attachments[index], + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = VK_RESOLVE_MODE_NONE, + .resolveImageView = VK_NULL_HANDLE, + .resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = {}, + }; + } + const VkRenderingAttachmentInfo depth_attachment_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = depth_attachment, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = VK_RESOLVE_MODE_NONE, + .resolveImageView = VK_NULL_HANDLE, + .resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = {}, + }; + const VkRenderingInfo rendering_info{ + .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, + .pNext = nullptr, + .flags = 0, + .renderArea = + { + .offset = {.x = 0, .y = 0}, + .extent = render_area, + }, + .layerCount = layer_count, + .viewMask = 0, + .colorAttachmentCount = num_color_attachments, + .pColorAttachments = color_attachment_infos.data(), + .pDepthAttachment = has_depth ? &depth_attachment_info : nullptr, + .pStencilAttachment = has_stencil ? &depth_attachment_info : nullptr, + }; + cmdbuf.BeginRendering(rendering_info); +} + void TextureCacheRuntime::AccelerateImageUpload( Image& image, const StagingBufferRef& map, std::span swizzles, @@ -2853,6 +3067,15 @@ void TextureCacheRuntime::AccelerateImageUpload( return astc_decoder_pass->Assemble(image, map, swizzles); } + if (bl2d_unswizzle_pass && BlockLinearUnswizzle2DPass::IsSupported(image.info)) { + return bl2d_unswizzle_pass->Unswizzle(image, map, swizzles); + } + + if (bl3db_unswizzle_pass && + BlockLinearUnswizzle3DBufferPass::IsSupported(device, image.info)) { + return bl3db_unswizzle_pass->Unswizzle(image, map, swizzles); + } + if (!Settings::values.gpu_unswizzle_enabled.GetValue() || !bl3d_unswizzle_pass) { if (IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D) { ASSERT(false && "GPU unswizzle is disabled for BCn 3D texture"); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 99f2978cca..fa7dc39313 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -60,10 +60,18 @@ public: void TickFrame(); + u64 CurrentSyncPoint() const noexcept; + + u64 CompletedSyncPoint() const; + + void WaitSyncPoint(u64 sync_point); + u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; + u64 GetDeviceAllocationUsage() const; + bool CanReportMemoryUsage() const; std::optional GetSamplerHeapBudget() const; @@ -121,7 +129,8 @@ public: }; [[nodiscard]] VkImageView GetOrCreateResolveShadow(VkImage msaa_image, VkFormat format, - VkExtent2D extent, u32 layers); + VkExtent2D extent, u32 layers, + VkImageAspectFlags aspect); [[nodiscard]] const ResolveShadow* GetValidResolveShadow(VkImage msaa_image) const; @@ -149,6 +158,8 @@ public: std::optional astc_decoder_pass; std::optional bl3d_unswizzle_pass; + std::optional bl2d_unswizzle_pass; + std::optional bl3db_unswizzle_pass; const Settings::ResolutionScalingInfo& resolution; std::array, VideoCore::Surface::MaxPixelFormat> view_formats; @@ -156,6 +167,7 @@ public: std::array buffers{}; std::vector> pending_msaa_images; ankerl::unordered_dense::map resolve_shadows; + ankerl::unordered_dense::set unsupported_msaa_resolves; }; class Framebuffer { @@ -178,6 +190,8 @@ public: std::span color_buffers, ImageView* depth_buffer, bool is_rescaled = false); + void BeginRendering(vk::CommandBuffer cmdbuf) const; + [[nodiscard]] VkFramebuffer Handle() const noexcept { return *framebuffer; } @@ -193,6 +207,51 @@ public: [[nodiscard]] VkRenderPass RenderPassVariant(u32 color_clear_mask, bool depth_stencil_clear, u32 color_discard_mask) const; + [[nodiscard]] u32 NumColorAttachments() const noexcept { + return num_color_attachments; + } + + [[nodiscard]] const std::array& ColorAttachments() const noexcept { + return color_attachments; + } + + [[nodiscard]] const std::array& ColorAttachmentFormats() const noexcept { + return color_attachment_formats; + } + + [[nodiscard]] const std::array& ColorResolveAttachments() const noexcept { + return color_resolve_attachments; + } + + [[nodiscard]] const std::array& ColorResolveModes() + const noexcept { + return color_resolve_modes; + } + + [[nodiscard]] VkImageView DepthAttachment() const noexcept { + return depth_attachment; + } + + [[nodiscard]] VkImageView DepthResolveAttachment() const noexcept { + return depth_resolve_attachment; + } + + [[nodiscard]] VkResolveModeFlagBits DepthResolveMode() const noexcept { + return depth_resolve_mode; + } + + [[nodiscard]] VkResolveModeFlagBits StencilResolveMode() const noexcept { + return stencil_resolve_mode; + } + + [[nodiscard]] VkFormat DepthAttachmentFormat() const noexcept { + return depth_attachment_format; + } + + [[nodiscard]] u32 NumLayers() const noexcept { + return layer_count; + } + [[nodiscard]] VkExtent2D RenderArea() const noexcept { return render_area; } @@ -245,6 +304,10 @@ public: return discard_msaa_color; } + [[nodiscard]] bool DiscardsMsaaDepth() const noexcept { + return discard_msaa_depth; + } + private: vk::Framebuffer framebuffer; VkRenderPass renderpass{}; @@ -255,6 +318,17 @@ private: std::array images{}; std::array image_ranges{}; std::array rt_map{}; + std::array color_attachments{}; + std::array color_attachment_formats{}; + std::array color_resolve_attachments{}; + std::array color_resolve_modes{}; + VkImageView depth_attachment{}; + VkImageView depth_resolve_attachment{}; + VkResolveModeFlagBits depth_resolve_mode = VK_RESOLVE_MODE_NONE; + VkResolveModeFlagBits stencil_resolve_mode = VK_RESOLVE_MODE_NONE; + VkFormat depth_attachment_format = VK_FORMAT_UNDEFINED; + u32 num_color_attachments = 0; + u32 layer_count = 1; bool has_depth{}; bool has_stencil{}; bool is_rescaled{}; @@ -263,6 +337,7 @@ private: RenderPassKey render_pass_key{}; RenderPassCache* render_pass_cache{nullptr}; bool discard_msaa_color{}; + bool discard_msaa_depth{}; }; class Image : public VideoCommon::ImageBase { @@ -426,7 +501,7 @@ private: std::array image_views; std::optional storage_views; - vk::ImageView typeless_storage_view; + std::array typeless_storage_views; vk::ImageView depth_view; vk::ImageView stencil_view; vk::ImageView color_view; @@ -436,7 +511,6 @@ private: VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT; u32 buffer_size = 0; - bool uses_widened_astc_format = false; bool supports_depth_comparison = false; }; @@ -487,6 +561,7 @@ struct TextureCacheParams { static constexpr bool HAS_EMULATED_COPIES = false; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; static constexpr bool IMPLEMENTS_ASYNC_DOWNLOADS = true; + static constexpr bool HAS_TIMELINE_SYNC_POINTS = true; using Runtime = Vulkan::TextureCacheRuntime; using Image = Vulkan::Image; diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp index b0227ac908..742a8d9969 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.cpp +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.cpp @@ -16,8 +16,11 @@ namespace Vulkan { -UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_) - : device{device_} +UpdateDescriptorQueue::UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_, + bool supports_descriptor_buffer_) + : device{device_}, frame_payload_size{frame_payload_size_}, + supports_descriptor_buffer{supports_descriptor_buffer_}, + payload(frame_payload_size_ * FRAMES_IN_FLIGHT) { payload_start = payload.data(); payload_cursor = payload.data(); @@ -29,19 +32,21 @@ void UpdateDescriptorQueue::TickFrame() { if (++frame_index >= FRAMES_IN_FLIGHT) { frame_index = 0; } - payload_start = payload.data() + frame_index * FRAME_PAYLOAD_SIZE; + payload_start = payload.data() + frame_index * frame_payload_size; payload_cursor = payload_start; } -void UpdateDescriptorQueue::Acquire(Scheduler& scheduler, size_t required_entries) { +void UpdateDescriptorQueue::Acquire(Scheduler& scheduler, size_t required_entries, + bool use_descriptor_buffer_) { + use_descriptor_buffer = supports_descriptor_buffer && use_descriptor_buffer_; static constexpr size_t DEFAULT_REQUIRED_ENTRIES = 0x400; const size_t reserve = required_entries > 0 ? required_entries : DEFAULT_REQUIRED_ENTRIES; - ASSERT_MSG(reserve < FRAME_PAYLOAD_SIZE, "Descriptor reservation {} >= frame capacity {}", - reserve, FRAME_PAYLOAD_SIZE); + ASSERT_MSG(reserve < frame_payload_size, "Descriptor reservation {} >= frame capacity {}", + reserve, frame_payload_size); const size_t used = static_cast(std::distance(payload_start, payload_cursor)); - if (used + reserve >= FRAME_PAYLOAD_SIZE) { + if (used + reserve >= frame_payload_size) { LOG_WARNING(Render_Vulkan, "Payload overflow (used={}, reserve={}, capacity={})", - used, reserve, FRAME_PAYLOAD_SIZE); + used, reserve, frame_payload_size); scheduler.WaitWorker(); payload_cursor = payload_start; } diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 39b2b5a688..9f30e65dbd 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -6,8 +6,8 @@ #pragma once -#include #include +#include #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { @@ -15,30 +15,45 @@ namespace Vulkan { class Device; class Scheduler; +struct DescriptorAddress { + VkDeviceAddress address; + VkDeviceSize range; + VkFormat format; +}; + union DescriptorUpdateEntry { DescriptorUpdateEntry() = default; DescriptorUpdateEntry(VkDescriptorImageInfo image_) : image{image_} {} DescriptorUpdateEntry(VkDescriptorBufferInfo buffer_) : buffer{buffer_} {} DescriptorUpdateEntry(VkBufferView texel_buffer_) : texel_buffer{texel_buffer_} {} + DescriptorUpdateEntry(DescriptorAddress address_) : address{address_} {} std::monostate empty{}; VkDescriptorImageInfo image; VkDescriptorBufferInfo buffer; VkBufferView texel_buffer; + DescriptorAddress address; }; class UpdateDescriptorQueue final { // This should be plenty for the vast majority of cases. Most desktop platforms only // provide up to 3 swapchain images. static constexpr size_t FRAMES_IN_FLIGHT = 8; - static constexpr size_t FRAME_PAYLOAD_SIZE = 0x20000; - static constexpr size_t PAYLOAD_SIZE = FRAME_PAYLOAD_SIZE * FRAMES_IN_FLIGHT; public: - explicit UpdateDescriptorQueue(const Device& device_); + static constexpr size_t GUEST_FRAME_PAYLOAD_SIZE = 0x80000; + static constexpr size_t COMPUTE_FRAME_PAYLOAD_SIZE = 0x20000; + + explicit UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_, + bool supports_descriptor_buffer_ = false); ~UpdateDescriptorQueue(); + [[nodiscard]] bool UsesDescriptorBuffer() const noexcept { + return use_descriptor_buffer; + } + void TickFrame(); - void Acquire(Scheduler& scheduler, size_t required_entries = 0); + void Acquire(Scheduler& scheduler, size_t required_entries = 0, + bool use_descriptor_buffer_ = false); const DescriptorUpdateEntry* UpdateData() const noexcept { return upload_start; @@ -68,17 +83,46 @@ public: }; } + void AddBuffer(VkBuffer buffer, VkDeviceAddress base_address, VkDeviceSize offset, + VkDeviceSize size) { + if (!use_descriptor_buffer) { + AddBuffer(buffer, offset, size); + return; + } + *(payload_cursor++) = DescriptorAddress{ + .address = base_address == 0 ? 0 : base_address + offset, + .range = base_address == 0 ? VK_WHOLE_SIZE : size, + .format = VK_FORMAT_UNDEFINED, + }; + } + void AddTexelBuffer(VkBufferView texel_buffer) { *(payload_cursor++) = texel_buffer; } + void AddTexelBuffer(VkBufferView texel_buffer, VkDeviceAddress base_address, + VkDeviceSize offset, VkDeviceSize size, VkFormat format) { + if (!use_descriptor_buffer) { + AddTexelBuffer(texel_buffer); + return; + } + *(payload_cursor++) = DescriptorAddress{ + .address = base_address == 0 ? 0 : base_address + offset, + .range = base_address == 0 ? VK_WHOLE_SIZE : size, + .format = format, + }; + } + private: const Device& device; + const size_t frame_payload_size; + const bool supports_descriptor_buffer; + bool use_descriptor_buffer{false}; size_t frame_index{0}; DescriptorUpdateEntry* payload_cursor = nullptr; DescriptorUpdateEntry* payload_start = nullptr; const DescriptorUpdateEntry* upload_start = nullptr; - std::array payload; + std::vector payload; }; // TODO: should these be separate classes instead? diff --git a/src/video_core/shader_environment.cpp b/src/video_core/shader_environment.cpp index 4d5db7e511..be3e711cb1 100644 --- a/src/video_core/shader_environment.cpp +++ b/src/video_core/shader_environment.cpp @@ -284,7 +284,24 @@ std::optional GenericEnvironment::TryFindSize() { Tegra::Texture::TICEntry GenericEnvironment::ReadTextureInfo(GPUVAddr tic_addr, u32 tic_limit, bool via_header_index, u32 raw) { const auto handle{Tegra::Texture::TexturePair(raw, via_header_index)}; - ASSERT(handle.first <= tic_limit); + if (handle.first > tic_limit) { + LOG_CRITICAL(Shader, + "TIC index out of range: raw=0x{:08x} tic_index={} tsc_index={} tic_limit={} " + "tic_addr=0x{:x} via_header_index={} stage={} program_base=0x{:x} " + "start_address=0x{:x}", + raw, handle.first, handle.second, tic_limit, tic_addr, via_header_index, + static_cast(stage), program_base, start_address); + ASSERT(handle.first <= tic_limit); + Tegra::Texture::TICEntry fallback{}; + fallback.format.Assign(Tegra::Texture::TextureFormat::A8B8G8R8); + fallback.r_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.g_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.b_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.a_type.Assign(Tegra::Texture::ComponentType::UNORM); + fallback.texture_type.Assign(Tegra::Texture::TextureType::Texture2D); + fallback.normalized_coords.Assign(1); + return fallback; + } const GPUVAddr descriptor_addr{tic_addr + handle.first * sizeof(Tegra::Texture::TICEntry)}; Tegra::Texture::TICEntry entry; gpu_memory->ReadBlock(descriptor_addr, &entry, sizeof(entry)); diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 2cfdb94359..decc17687f 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -58,23 +58,9 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag void(slot_samplers.insert(runtime, sampler_descriptor)); if constexpr (HAS_DEVICE_MEMORY_INFO) { - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - expected_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); - minimum_memory = static_cast((device_local_memory - mem_threshold) / 2); + memory_budget = runtime.GetDeviceLocalMemory(); } else { - expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB; - critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; - minimum_memory = 0; + memory_budget = FALLBACK_MEMORY_BUDGET; } const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); @@ -114,71 +100,166 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag } template -void TextureCache

::RunGarbageCollector() { - bool high_priority_mode = false; - bool aggressive_mode = false; - u64 ticks_to_destroy = 0; - size_t num_iterations = 0; - const auto Configure = [&](bool allow_aggressive) { - high_priority_mode = total_used_memory >= expected_memory; - aggressive_mode = allow_aggressive && total_used_memory >= critical_memory; - ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL; - num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); - }; - const auto Cleanup = [this, &num_iterations, &high_priority_mode, &aggressive_mode](ImageId image_id) { - if (num_iterations == 0) { - return true; - } - --num_iterations; - auto& image = slot_images[image_id]; - if (True(image.flags & ImageFlagBits::IsDecoding)) { - return false; - } - const bool must_download = image.IsSafeDownload() && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { - return false; - } - if (must_download) { - auto map = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes); - const auto copies = FixSmallVectorADL(FullDownloadCopies(image.info)); - image.DownloadMemory(map, copies); - runtime.Finish(); - SwizzleImage(*gpu_memory, image.gpu_addr, image.info, copies, map.mapped_span, swizzle_data_buffer); - } - if (True(image.flags & ImageFlagBits::Tracked)) { - UntrackImage(image, image_id); - } - UnregisterImage(image_id); - DeleteImage(image_id, image.scale_tick > frame_tick + 5); - if (aggressive_mode && total_used_memory < critical_memory) { - num_iterations >>= 2; - aggressive_mode = false; - } else if (high_priority_mode && total_used_memory < expected_memory) { - num_iterations >>= 1; - high_priority_mode = false; - } - return false; - }; - Configure(false); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); - if (total_used_memory >= critical_memory) { - Configure(true); - lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); +void TextureCache

::QueueEvictionDownload(Image& image) { + auto copies = FullDownloadCopies(image.info); + auto staging = runtime.DownloadStagingBuffer(image.unswizzled_size_bytes, true); + image.DownloadMemory(staging, FixSmallVectorADL(copies)); + pending_eviction_downloads.push_back(PendingEvictionDownload{ + .staging = staging, + .gpu_memory = gpu_memory, + .copies = std::move(copies), + .info = image.info, + .gpu_addr = image.gpu_addr, + .sync_point = runtime.CurrentSyncPoint(), + }); +} + +template +void TextureCache

::TickEvictionDownloads(u64 completed_sync_point) { + while (!pending_eviction_downloads.empty() && + pending_eviction_downloads.front().sync_point <= completed_sync_point) { + auto& entry = pending_eviction_downloads.front(); + SwizzleImage(*entry.gpu_memory, entry.gpu_addr, entry.info, FixSmallVectorADL(entry.copies), + entry.staging.mapped_span.subspan(entry.staging.offset), swizzle_data_buffer); + runtime.FreeDeferredStagingBuffer(entry.staging); + pending_eviction_downloads.pop_front(); } } +template +void TextureCache

::FlushEvictionDownloads() { + if (pending_eviction_downloads.empty()) { + return; + } + const u64 last_sync_point = pending_eviction_downloads.back().sync_point; + runtime.WaitSyncPoint(last_sync_point); + TickEvictionDownloads(last_sync_point); +} + +template +u64 TextureCache

::ImageSizeBytes(const ImageBase& image) { + u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); + if ((IsPixelFormatASTC(image.info.format) && + True(image.flags & ImageFlagBits::AcceleratedUpload)) || + True(image.flags & ImageFlagBits::Converted)) { + tentative_size = TranscodedAstcSize(tentative_size, image.info.format); + } + u64 size = Common::AlignUp(tentative_size, 1024); + if (image.HasScaled()) { + size += GetScaledImageSizeBytes(image); + } + return size; +} + +template +u64 TextureCache

::DeviceUsage(bool force_refresh) { + if (!runtime.CanReportMemoryUsage()) { + return total_used_memory; + } + if (force_refresh || usage_refresh_countdown == 0) { + cached_device_usage = runtime.GetDeviceAllocationUsage(); + usage_refresh_countdown = USAGE_REFRESH_INTERVAL; + } else { + --usage_refresh_countdown; + } + return cached_device_usage; +} + +template +u64 TextureCache

::ReclaimMemory(u64 target_bytes, bool allow_download) { + if (target_bytes == 0 || in_reclaim) { + return 0; + } + in_reclaim = true; + u64 freed = 0; + const auto evict = [&](ImageId image_id) { + if (freed >= target_bytes) { + return true; + } + auto& image = slot_images[image_id]; + if (True(image.flags & ImageFlagBits::IsDecoding)) { + return false; + } + const bool must_download = image.IsSafeDownload(); + if (must_download && True(image.flags & ImageFlagBits::BadOverlap)) { + LOG_DEBUG(HW_GPU, "Recovering bad overlap on eviction: gpu_addr=0x{:x} fmt={} {}x{}x{}", + image.gpu_addr, static_cast(image.info.format), image.info.size.width, + image.info.size.height, image.info.size.depth); + } + bool queued_download = false; + if (must_download) { + if constexpr (HAS_TIMELINE_SYNC_POINTS) { + if (!allow_download) { + return false; + } + QueueEvictionDownload(image); + queued_download = true; + } else { + return false; + } + } + const u64 image_bytes = ImageSizeBytes(image); + if (True(image.flags & ImageFlagBits::Tracked)) { + UntrackImage(image, image_id); + } + UnregisterImage(image_id); + DeleteImage(image_id, !queued_download && image.scale_tick > frame_tick + 5); + freed += image_bytes; + return false; + }; + const u64 cold_tick = + frame_tick > RECLAIM_GUARD_FRAMES ? frame_tick - RECLAIM_GUARD_FRAMES : 0; + lru_cache.ForEachItemBelow(cold_tick, evict); + if (freed == 0) { + const u64 urgent_tick = frame_tick > RECLAIM_URGENT_GUARD_FRAMES + ? frame_tick - RECLAIM_URGENT_GUARD_FRAMES + : 0; + lru_cache.ForEachItemBelow(urgent_tick, evict); + } + in_reclaim = false; + usage_refresh_countdown = 0; + reclaim_stalled = freed == 0; + if (freed > 0) { + reclaim_wait_sync_point = runtime.CurrentSyncPoint(); + } + return freed; +} + +template +void TextureCache

::ReclaimDeferredResources(u64 completed_sync_point) { + sentenced_images.Reclaim(completed_sync_point); + sentenced_framebuffers.Reclaim(completed_sync_point); + sentenced_image_view.Reclaim(completed_sync_point); +} + +template +void TextureCache

::EnsureHeadroom(bool allow_download) { + if (reclaim_stalled) { + return; + } + if (runtime.CompletedSyncPoint() < reclaim_wait_sync_point) { + return; + } + const u64 limit = memory_budget > RECLAIM_HEADROOM ? memory_budget - RECLAIM_HEADROOM : 0; + const u64 usage = DeviceUsage(false); + if (usage <= limit) { + return; + } + const u64 target = (limit / 100) * RECLAIM_TARGET_PERCENT; + const u64 excess = usage - target; + const u64 usage_mib = (std::max)(usage >> 20, u64{1}); + const u64 share = (((excess >> 20) * (total_used_memory >> 20)) / usage_mib) << 20; + ReclaimMemory((std::min)(share, total_used_memory), allow_download); +} + template void TextureCache

::TickFrame() { - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); - } - if (total_used_memory > minimum_memory) { - RunGarbageCollector(); - } - sentenced_images.Tick(); - sentenced_framebuffers.Tick(); - sentenced_image_view.Tick(); + usage_refresh_countdown = 0; + reclaim_stalled = false; + const u64 completed_sync_point = runtime.CompletedSyncPoint(); + TickEvictionDownloads(completed_sync_point); + ReclaimDeferredResources(completed_sync_point); + EnsureHeadroom(true); TickAsyncDecode(); TickAsyncUnswizzle(); @@ -512,10 +593,24 @@ void TextureCache

::UpdateRenderTargets(bool is_clear) { up_scale = Settings::values.resolution_info.up_scale; down_shift = Settings::values.resolution_info.down_shift; } - render_targets.size = Extent2D{ - (maxwell3d->regs.surface_clip.width * up_scale) >> down_shift, - (maxwell3d->regs.surface_clip.height * up_scale) >> down_shift, + u32 clip_width = (maxwell3d->regs.surface_clip.width * up_scale) >> down_shift; + u32 clip_height = (maxwell3d->regs.surface_clip.height * up_scale) >> down_shift; + const auto& resolution = Settings::values.resolution_info; + const auto clamp_to_attachment = [&](ImageViewId view_id) { + if (!view_id) { + return; + } + const ImageViewBase& view = slot_image_views[view_id]; + clip_width = (std::min)(clip_width, is_rescaling ? resolution.ScaleUp(view.size.width) + : view.size.width); + clip_height = (std::min)(clip_height, is_rescaling ? resolution.ScaleUp(view.size.height) + : view.size.height); }; + for (size_t index = 0; index < NUM_RT; ++index) { + clamp_to_attachment(render_targets.color_buffer_ids[index]); + } + clamp_to_attachment(depth_buffer_id); + render_targets.size = Extent2D{clip_width, clip_height}; render_targets.is_rescaled = is_rescaling; if (render_targets != previous_render_targets) { @@ -596,6 +691,7 @@ void TextureCache

::WriteMemory(DAddr cpu_addr, size_t size) { template void TextureCache

::DownloadMemory(DAddr cpu_addr, size_t size) { + FlushEvictionDownloads(); boost::container::small_vector images; ForEachImageInRegion(cpu_addr, size, [&images](ImageId image_id, ImageBase& image) { if (!image.IsSafeDownload()) { @@ -894,6 +990,7 @@ void TextureCache

::CommitAsyncFlushes() { template void TextureCache

::PopAsyncFlushes() { + TickEvictionDownloads(runtime.CompletedSyncPoint()); if (committed_downloads.empty()) { return; } @@ -1294,8 +1391,9 @@ void TextureCache

::InvalidateScale(Image& image) { } RemoveImageViewReferences(image_view_ids); RemoveFramebuffers(image_view_ids); + const u64 sync_point = runtime.CurrentSyncPoint(); for (const ImageViewId image_view_id : image_view_ids) { - sentenced_image_view.Push(std::move(slot_image_views[image_view_id])); + sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point); slot_image_views.erase(image_view_id); } image.image_view_ids.clear(); @@ -1331,6 +1429,7 @@ void TextureCache

::QueueAsyncDecode(Image& image, ImageId image_id) { LOG_INFO(HW_GPU, "Queuing async texture decode"); image.flags |= ImageFlagBits::IsDecoding; + runtime.TransitionImageLayout(image); auto decode = std::make_unique(); auto* decode_ptr = decode.get(); decode->image_id = image_id; @@ -1363,6 +1462,7 @@ void TextureCache

::QueueAsyncUnswizzle(Image& image, ImageId image_id) { } image.flags |= ImageFlagBits::IsDecoding; + runtime.TransitionImageLayout(image); unswizzle_queue.push_back({ .image_id = image_id, @@ -1510,7 +1610,7 @@ ImageId TextureCache

::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, } } ASSERT_MSG(cpu_addr, "Tried to insert an image to an invalid gpu_addr=0x{:x}", gpu_addr); - const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr); + const ImageId image_id = JoinImages(info, gpu_addr, *cpu_addr, options); const Image& image = slot_images[image_id]; // Using "image.gpu_addr" instead of "gpu_addr" is important because it might be different const auto [it, is_new] = image_allocs_table.try_emplace(image.gpu_addr); @@ -1522,7 +1622,9 @@ ImageId TextureCache

::InsertImage(const ImageInfo& info, GPUVAddr gpu_addr, } template -ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr) { +ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr, + RelaxedOptions options) { + EnsureHeadroom(false); ImageInfo new_info = info; const size_t size_bytes = CalculateGuestSizeInBytes(new_info); const bool broken_views = runtime.HasBrokenTextureViewFormats(); @@ -1533,9 +1635,11 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA join_right_aliased_ids.clear(); join_ignore_textures.clear(); join_bad_overlap_ids.clear(); + join_stale_ids.clear(); join_copies_to_do.clear(); join_alias_indices.clear(); const bool this_is_linear = info.type == ImageType::Linear; + const bool is_render_target = True(options & RelaxedOptions::RenderTarget); const auto region_check = [&](ImageId overlap_id, ImageBase& overlap) { if (True(overlap.flags & ImageFlagBits::Remapped)) { join_ignore_textures.insert(overlap_id); @@ -1575,6 +1679,9 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA join_right_aliased_ids.push_back(overlap_id); overlap.flags |= ImageFlagBits::Alias; join_copies_to_do.emplace_back(JoinCopy{true, overlap_id}); + } else if (is_render_target && IsStaleReallocation(new_info, overlap, gpu_addr) && + slot_images[overlap_id].allocation_tick != frame_tick) { + join_stale_ids.push_back(overlap_id); } else { join_bad_overlap_ids.push_back(overlap_id); } @@ -1631,7 +1738,35 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA for (const ImageId overlap_id : join_ignore_textures) { Image& overlap = slot_images[overlap_id]; if (True(overlap.flags & ImageFlagBits::GpuModified)) { - UNIMPLEMENTED(); + if (new_image.TryFindBase(overlap.gpu_addr) && + (!can_rescale || ImageCanRescale(overlap))) { + if (can_rescale) { + ScaleUp(overlap); + } else { + ScaleDown(overlap); + } + join_copies_to_do.emplace_back(JoinCopy{false, overlap_id}); + continue; + } + if (overlap.IsSafeDownload() && + gpu_memory->GpuToCpuAddress(overlap.gpu_addr).has_value()) { + if (True(overlap.flags & ImageFlagBits::BadOverlap)) { + LOG_WARNING(HW_GPU, + "Recovering bad overlap on join: gpu_addr=0x{:x} fmt={} {}x{}x{}", + overlap.gpu_addr, static_cast(overlap.info.format), + overlap.info.size.width, overlap.info.size.height, + overlap.info.size.depth); + } + QueueEvictionDownload(overlap); + } else { + LOG_WARNING(HW_GPU, + "Dropping GPU modified overlap, contents are not recoverable: " + "gpu_addr=0x{:x} format={} size={}x{}x{} levels={} layers={}", + overlap.gpu_addr, static_cast(overlap.info.format), + overlap.info.size.width, overlap.info.size.height, + overlap.info.size.depth, overlap.info.resources.levels, + overlap.info.resources.layers); + } } if (True(overlap.flags & ImageFlagBits::Tracked)) { UntrackImage(overlap, overlap_id); @@ -1640,6 +1775,19 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA DeleteImage(overlap_id); } + for (const ImageId stale_id : join_stale_ids) { + Image& stale = slot_images[stale_id]; + LOG_DEBUG(HW_GPU, + "Retiring resized image: gpu_addr=0x{:x} fmt={} {}x{} -> {}x{}", stale.gpu_addr, + static_cast(stale.info.format), stale.info.size.width, + stale.info.size.height, new_image.info.size.width, new_image.info.size.height); + if (True(stale.flags & ImageFlagBits::Tracked)) { + UntrackImage(stale, stale_id); + } + UnregisterImage(stale_id); + DeleteImage(stale_id); + } + // TODO: Only upload what we need RefreshContents(new_image, new_image_id); @@ -1678,6 +1826,8 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA ImageBase& aliased = slot_images[aliased_id]; aliased.overlapping_images.push_back(new_image_id); new_image.overlapping_images.push_back(aliased_id); + const bool aliased_was_bad = True(aliased.flags & ImageFlagBits::BadOverlap); + const bool new_was_bad = True(new_image.flags & ImageFlagBits::BadOverlap); if (aliased.info.resources.levels == 1 && aliased.info.block.depth == 0 && aliased.overlapping_images.size() > 1) { aliased.flags |= ImageFlagBits::BadOverlap; @@ -1686,6 +1836,21 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA new_image.overlapping_images.size() > 1) { new_image.flags |= ImageFlagBits::BadOverlap; } + const bool aliased_is_bad = True(aliased.flags & ImageFlagBits::BadOverlap); + const bool new_is_bad = True(new_image.flags & ImageFlagBits::BadOverlap); + if ((!aliased_was_bad && aliased_is_bad) || (!new_was_bad && new_is_bad)) { + LOG_DEBUG(HW_GPU, + "Bad overlap: existing gpu_addr={:#x} {}x{}x{} fmt={} type={} rt={} | " + "incoming gpu_addr={:#x} {}x{}x{} fmt={} type={} rt={}", + aliased.gpu_addr, aliased.info.size.width, aliased.info.size.height, + aliased.info.size.depth, static_cast(aliased.info.format), + static_cast(aliased.info.type), + True(aliased.flags & ImageFlagBits::GpuModified), + new_image.gpu_addr, new_image.info.size.width, new_image.info.size.height, + new_image.info.size.depth, static_cast(new_image.info.format), + static_cast(new_image.info.type), + True(new_image.flags & ImageFlagBits::GpuModified)); + } } for (const auto& copy_object : join_copies_to_do) { @@ -1976,7 +2141,7 @@ ImageViewId TextureCache

::FindRenderTargetView(const ImageInfo& info, GPUVAdd bool delete_state = has_deleted_images; do { has_deleted_images = false; - image_id = FindOrInsertImage(info, gpu_addr); + image_id = FindOrInsertImage(info, gpu_addr, RelaxedOptions::RenderTarget); delete_state |= has_deleted_images; } while (has_deleted_images); has_deleted_images = delete_state; @@ -2185,13 +2350,7 @@ void TextureCache

::RegisterImage(ImageId image_id) { ASSERT_MSG(False(image.flags & ImageFlagBits::Registered), "Trying to register an already registered image"); image.flags |= ImageFlagBits::Registered; - u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - if ((IsPixelFormatASTC(image.info.format) && - True(image.flags & ImageFlagBits::AcceleratedUpload)) || - True(image.flags & ImageFlagBits::Converted)) { - tentative_size = TranscodedAstcSize(tentative_size, image.info.format); - } - total_used_memory += Common::AlignUp(tentative_size, 1024); + total_used_memory += ImageSizeBytes(image); image.lru_index = lru_cache.Insert(image_id, frame_tick); ForEachGPUPage(image.gpu_addr, image.guest_size_bytes, [this, image_id](u64 page) { @@ -2308,6 +2467,7 @@ void TextureCache

::TrackImage(ImageBase& image, ImageId image_id) { if (False(image.flags & ImageFlagBits::Sparse)) { if (image.cpu_addr < ~(1ULL << 40)) { device_memory.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, 1); + device_memory.UpdateTexturePagesCount(image.cpu_addr, image.guest_size_bytes, 1); } return; } @@ -2320,12 +2480,14 @@ void TextureCache

::TrackImage(ImageBase& image, ImageId image_id) { const DAddr cpu_addr = map.cpu_addr; const std::size_t size = map.size; device_memory.UpdatePagesCachedCount(cpu_addr, size, 1); + device_memory.UpdateTexturePagesCount(cpu_addr, size, 1); } return; } ForEachSparseSegment(image, [this]([[maybe_unused]] GPUVAddr gpu_addr, DAddr cpu_addr, size_t size) { device_memory.UpdatePagesCachedCount(cpu_addr, size, 1); + device_memory.UpdateTexturePagesCount(cpu_addr, size, 1); }); } @@ -2336,6 +2498,7 @@ void TextureCache

::UntrackImage(ImageBase& image, ImageId image_id) { if (False(image.flags & ImageFlagBits::Sparse)) { if (image.cpu_addr < ~(1ULL << 40)) { device_memory.UpdatePagesCachedCount(image.cpu_addr, image.guest_size_bytes, -1); + device_memory.UpdateTexturePagesCount(image.cpu_addr, image.guest_size_bytes, -1); } return; } @@ -2348,22 +2511,14 @@ void TextureCache

::UntrackImage(ImageBase& image, ImageId image_id) { const DAddr cpu_addr = map.cpu_addr; const std::size_t size = map.size; device_memory.UpdatePagesCachedCount(cpu_addr, size, -1); + device_memory.UpdateTexturePagesCount(cpu_addr, size, -1); } } template void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ImageBase& image = slot_images[image_id]; - if (image.HasScaled()) { - total_used_memory -= GetScaledImageSizeBytes(image); - } - u64 tentative_size = (std::max)(image.guest_size_bytes, image.unswizzled_size_bytes); - if ((IsPixelFormatASTC(image.info.format) && - True(image.flags & ImageFlagBits::AcceleratedUpload)) || - True(image.flags & ImageFlagBits::Converted)) { - tentative_size = TranscodedAstcSize(tentative_size, image.info.format); - } - total_used_memory -= Common::AlignUp(tentative_size, 1024); + total_used_memory -= std::min(total_used_memory, ImageSizeBytes(image)); const GPUVAddr gpu_addr = image.gpu_addr; const auto alloc_it = image_allocs_table.find(gpu_addr); if (alloc_it == image_allocs_table.end()) { @@ -2417,14 +2572,15 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { ASSERT_MSG(num_removed_overlaps == 1, "Invalid number of removed overlapps: {}", num_removed_overlaps); } + const u64 sync_point = runtime.CurrentSyncPoint(); for (const ImageViewId image_view_id : image_view_ids) { if (!immediate_delete) { - sentenced_image_view.Push(std::move(slot_image_views[image_view_id])); + sentenced_image_view.Push(std::move(slot_image_views[image_view_id]), sync_point); } slot_image_views.erase(image_view_id); } if (!immediate_delete) { - sentenced_images.Push(std::move(slot_images[image_id])); + sentenced_images.Push(std::move(slot_images[image_id]), sync_point); } slot_images.erase(image_id); @@ -2470,7 +2626,8 @@ void TextureCache

::RemoveFramebuffers(std::span removed_vi last_framebuffer_id = {}; last_framebuffer_serial = 0; } - sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id])); + sentenced_framebuffers.Push(std::move(slot_framebuffers[framebuffer_id]), + runtime.CurrentSyncPoint()); it = framebuffers.erase(it); } else { ++it; diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index fbc2bb4cf7..0e5cc3c44d 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -30,7 +30,7 @@ #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" -#include "video_core/delayed_destruction_ring.h" +#include "video_core/deferred_destruction_queue.h" #include "video_core/engines/fermi_2d.h" #include "video_core/surface.h" #include "video_core/texture_cache/descriptor_table.h" @@ -108,18 +108,21 @@ class TextureCache : public VideoCommon::ChannelSetupCaches::max)()}; #ifdef YUZU_LEGACY - static constexpr s64 TARGET_THRESHOLD = 3_GiB; + static constexpr u64 RECLAIM_HEADROOM = 192_MiB; #else - static constexpr s64 TARGET_THRESHOLD = 4_GiB; + static constexpr u64 RECLAIM_HEADROOM = 256_MiB; #endif - static constexpr s64 DEFAULT_EXPECTED_MEMORY = 1_GiB + 125_MiB; - static constexpr s64 DEFAULT_CRITICAL_MEMORY = 1_GiB + 625_MiB; - static constexpr size_t GC_EMERGENCY_COUNTS = 2; + static constexpr u64 FALLBACK_MEMORY_BUDGET = 2_GiB; + static constexpr u32 USAGE_REFRESH_INTERVAL = 16; + static constexpr u64 RECLAIM_GUARD_FRAMES = 16; + static constexpr u64 RECLAIM_URGENT_GUARD_FRAMES = 4; + static constexpr u64 RECLAIM_TARGET_PERCENT = 90; using Runtime = typename P::Runtime; using Image = typename P::Image; @@ -154,6 +157,10 @@ public: /// Notify the cache that a new frame has been queued void TickFrame(); + u64 ReclaimMemory(u64 target_bytes, bool allow_download); + + void ReclaimDeferredResources(u64 completed_sync_point); + /// Return a constant reference to the given image view id [[nodiscard]] const ImageView& GetImageView(ImageViewId id) const noexcept; @@ -257,7 +264,7 @@ public: /// Prepare an image to be used void PrepareImage(ImageId image_id, bool is_modification, bool invalidate); - std::recursive_mutex mutex; + std::mutex mutex; private: /// Iterate over all page indices in a range @@ -293,8 +300,17 @@ private: void OnGPUASRegister(size_t map_id) final override; - /// Runs the Garbage Collector. - void RunGarbageCollector(); + u64 ImageSizeBytes(const ImageBase& image); + + u64 DeviceUsage(bool force_refresh); + + void EnsureHeadroom(bool allow_download); + + void QueueEvictionDownload(Image& image); + + void TickEvictionDownloads(u64 completed_sync_point); + + void FlushEvictionDownloads(); /// Find or create an image view in the guest descriptor table ImageViewId VisitImageView(u32 index, bool compute); @@ -326,7 +342,8 @@ private: /// Create a new image and join perfectly matching existing images /// Remove joined images from the cache - [[nodiscard]] ImageId JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr); + [[nodiscard]] ImageId JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DAddr cpu_addr, + RelaxedOptions options); [[nodiscard]] ImageId FindDMAImage(const ImageInfo& info, GPUVAddr gpu_addr); @@ -451,9 +468,12 @@ private: bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; - u64 minimum_memory; - u64 expected_memory; - u64 critical_memory; + u64 memory_budget = 0; + u64 cached_device_usage = 0; + u64 reclaim_wait_sync_point = 0; + u32 usage_refresh_countdown = 0; + bool in_reclaim = false; + bool reclaim_stalled = false; size_t gpu_unswizzle_maxsize = 0; size_t swizzle_chunk_size = 0; u32 swizzle_slices_per_batch = 0; @@ -491,14 +511,19 @@ private: }; Common::LeastRecentlyUsedCache lru_cache; - #ifdef YUZU_LEGACY - static constexpr size_t TICKS_TO_DESTROY = 6; - #else - static constexpr size_t TICKS_TO_DESTROY = 8; -#endif - DelayedDestructionRing sentenced_images; - DelayedDestructionRing sentenced_image_view; - DelayedDestructionRing sentenced_framebuffers; + DeferredDestructionQueue sentenced_images; + DeferredDestructionQueue sentenced_image_view; + DeferredDestructionQueue sentenced_framebuffers; + + struct PendingEvictionDownload { + AsyncBuffer staging; + Tegra::MemoryManager* gpu_memory; + boost::container::small_vector copies; + VideoCommon::ImageInfo info; + GPUVAddr gpu_addr; + u64 sync_point; + }; + std::deque pending_eviction_downloads; ankerl::unordered_dense::map image_allocs_table; @@ -509,7 +534,8 @@ private: u64 frame_tick = 0; u64 last_sampler_gc_frame = (std::numeric_limits::max)(); - Common::ThreadWorker texture_decode_worker{1, "TextureDecoder"}; + Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, + Common::ThreadPlacement::Efficiency}; std::vector> async_decodes; std::deque unswizzle_queue; @@ -522,6 +548,7 @@ private: boost::container::small_vector join_right_aliased_ids; ankerl::unordered_dense::set join_ignore_textures; boost::container::small_vector join_bad_overlap_ids; + boost::container::small_vector join_stale_ids; struct JoinCopy { bool is_alias; ImageId id; diff --git a/src/video_core/texture_cache/types.h b/src/video_core/texture_cache/types.h index ecacd8c6bd..6a9f161faa 100644 --- a/src/video_core/texture_cache/types.h +++ b/src/video_core/texture_cache/types.h @@ -57,6 +57,7 @@ enum class RelaxedOptions : u32 { Format = 1 << 1, Samples = 1 << 2, ForceBrokenViews = 1 << 3, + RenderTarget = 1 << 4, }; DECLARE_ENUM_FLAG_OPERATORS(RelaxedOptions) diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index e55d0752ec..ba2044dcfb 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project @@ -747,7 +747,7 @@ boost::container::small_vector MakeShrinkImageCopies(const ImageI const bool is_dst_3d = dst.type == ImageType::e3D; if (is_dst_3d) { - ASSERT(src.type == ImageType::e3D); + ASSERT(src.type == ImageType::e3D || src.resources.layers == 1); ASSERT(src.resources.levels == 1); } const bool both_2d{src.type == ImageType::e2D && dst.type == ImageType::e2D}; @@ -1194,6 +1194,35 @@ bool IsLayerStrideCompatible(const ImageInfo& lhs, const ImageInfo& rhs) { return false; } +bool IsStaleReallocation(const ImageInfo& new_info, const ImageBase& overlap, + GPUVAddr gpu_addr) noexcept { + if (overlap.gpu_addr != gpu_addr) { + return false; + } + const ImageInfo& info = overlap.info; + if (new_info.type != ImageType::e2D || info.type != ImageType::e2D) { + return false; + } + if (new_info.resources.levels != 1 || info.resources.levels != 1) { + return false; + } + if (new_info.resources.layers != info.resources.layers) { + return false; + } + if (new_info.block != info.block || new_info.num_samples != info.num_samples) { + return false; + } + if (new_info.tile_width_spacing != info.tile_width_spacing) { + return false; + } + if (BytesPerBlock(new_info.format) != BytesPerBlock(info.format) || + DefaultBlockWidth(new_info.format) != DefaultBlockWidth(info.format) || + DefaultBlockHeight(new_info.format) != DefaultBlockHeight(info.format)) { + return false; + } + return new_info.size.width != info.size.width || new_info.size.height != info.size.height; +} + std::optional FindSubresource(const ImageInfo& candidate, const ImageBase& image, GPUVAddr candidate_addr, RelaxedOptions options, bool broken_views, bool native_bgr) { diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index 3e8bb00032..2ca7df0217 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -104,6 +104,9 @@ void SwizzleImage(Tegra::MemoryManager& gpu_memory, GPUVAddr gpu_addr, const Ima [[nodiscard]] bool IsLayerStrideCompatible(const ImageInfo& lhs, const ImageInfo& rhs); +[[nodiscard]] bool IsStaleReallocation(const ImageInfo& new_info, const ImageBase& overlap, + GPUVAddr gpu_addr) noexcept; + [[nodiscard]] std::optional FindSubresource(const ImageInfo& candidate, const ImageBase& image, GPUVAddr candidate_addr, diff --git a/src/video_core/textures/workers.cpp b/src/video_core/textures/workers.cpp index 780e0c4da9..9f37dfc48e 100644 --- a/src/video_core/textures/workers.cpp +++ b/src/video_core/textures/workers.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project @@ -10,7 +10,8 @@ namespace Tegra::Texture { Common::ThreadWorker& GetThreadWorkers() { static Common::ThreadWorker workers{(std::max)(std::thread::hardware_concurrency(), 2U) / 2, - "ImageTranscode"}; + "ImageTranscode", {}, + Common::ThreadPlacement::Efficiency}; return workers; } diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index 8fd9b6e69f..5552ccc18f 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -5,8 +5,11 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include #include +#include +#include #include #include #include @@ -16,6 +19,9 @@ #include #include "common/assert.h" +#include "common/fs/fs.h" +#include "common/fs/path_util.h" +#include "common/host_memory.h" #include "common/literals.h" #include #include "common/settings.h" @@ -393,6 +399,17 @@ std::vector ExtensionListForVulkan( return output; } +constexpr std::array STATIC_CACHE_MAGIC_NUMBER{'e', 'd', 'e', 'n', 's', 't', 'p', 'c'}; +constexpr u32 STATIC_CACHE_VERSION = 1; + +std::filesystem::path StaticPipelineCacheFilename() { + const auto shader_dir = Common::FS::GetEdenPath(Common::FS::EdenPath::ShaderDir); + if (!Common::FS::CreateDir(shader_dir)) { + return {}; + } + return shader_dir / "vulkan_static_pipelines.bin"; +} + } // Anonymous namespace void Device::RemoveExtension(bool& extension, const std::string& extension_name) { @@ -523,10 +540,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR features.shader_atomic_int64.shaderBufferInt64Atomics = false; features.shader_atomic_int64.shaderSharedInt64Atomics = false; features.features.shaderInt64 = false; - LOG_WARNING(Render_Vulkan, "Qualcomm drivers have broken workgroup memory explicit layout."); - RemoveExtensionFeature(extensions.workgroup_memory_explicit_layout, - features.workgroup_memory_explicit_layout, - VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME); #if defined(__ANDROID__) && defined(ARCHITECTURE_arm64) // BCn patching only safe on Android 9+ (API 28+). Older versions crash on driver load. @@ -713,6 +726,17 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR RemoveExtensionFeature(extensions.vertex_input_dynamic_state, features.vertex_input_dynamic_state, VK_EXT_VERTEX_INPUT_DYNAMIC_STATE_EXTENSION_NAME); } + // VK_EXT_descriptor_buffer requires VK_KHR_buffer_device_address + if (extensions.descriptor_buffer && !features.buffer_device_address.bufferDeviceAddress) { + LOG_WARNING(Render_Vulkan, "Descriptor buffer needs buffer device address, disabling."); + RemoveExtensionFeature(extensions.descriptor_buffer, features.descriptor_buffer, + VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME); + } + if (!extensions.descriptor_buffer) { + RemoveExtensionFeature(extensions.buffer_device_address, features.buffer_device_address, + VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME); + } + logical = vk::Device::Create(physical, queue_cis, ExtensionListForVulkan(loaded_extensions), first_next, dld); graphics_queue = logical.GetQueue(graphics_family); @@ -726,13 +750,16 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR if (extensions.memory_budget) { flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT; } + if (extensions.buffer_device_address) { + flags |= VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT; + } const VmaAllocatorCreateInfo allocator_info{ .flags = flags, .physicalDevice = physical, .device = *logical, .preferredLargeHeapBlockSize = is_integrated ? (64u * 1024u * 1024u) - : (256u * 1024u * 1024u), + : (128u * 1024u * 1024u), .pAllocationCallbacks = nullptr, .pDeviceMemoryCallbacks = nullptr, .pHeapSizeLimit = nullptr, @@ -744,15 +771,124 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR vk::Check(vmaCreateAllocator(&allocator_info, &allocator)); + { + const auto& limits = properties.properties.limits; + LOG_INFO(Render_Vulkan, "MSAA sample count support:"); + LOG_INFO(Render_Vulkan, " framebufferColorSampleCounts: {:#x}", + limits.framebufferColorSampleCounts); + LOG_INFO(Render_Vulkan, " framebufferDepthSampleCounts: {:#x}", + limits.framebufferDepthSampleCounts); + LOG_INFO(Render_Vulkan, " framebufferStencilSampleCounts: {:#x}", + limits.framebufferStencilSampleCounts); + LOG_INFO(Render_Vulkan, " sampledImageColorSampleCounts: {:#x}", + limits.sampledImageColorSampleCounts); + LOG_INFO(Render_Vulkan, " sampledImageDepthSampleCounts: {:#x}", + limits.sampledImageDepthSampleCounts); + LOG_INFO(Render_Vulkan, " sampledImageIntegerSampleCounts:{:#x}", + limits.sampledImageIntegerSampleCounts); + LOG_INFO(Render_Vulkan, " storageImageSampleCounts: {:#x}", + limits.storageImageSampleCounts); + } + + owns_static_pipeline_cache = surface != VkSurfaceKHR{}; + LoadStaticPipelineCache(); + // Initialize GPU logging if enabled InitializeGPULogging(); } Device::~Device() { + SaveStaticPipelineCache(); ShutdownGPULogging(); + vk::FlushDeletionQueue(); vmaDestroyAllocator(allocator); } +void Device::LoadStaticPipelineCache() { + const auto create = [this](size_t size, const void* data) { + static_pipeline_cache = logical.CreatePipelineCache({ + .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .initialDataSize = size, + .pInitialData = data, + }); + }; + if (!owns_static_pipeline_cache) { + create(0, nullptr); + return; + } + const auto filename = StaticPipelineCacheFilename(); + if (filename.empty()) { + create(0, nullptr); + return; + } + std::vector data; + try { + std::ifstream file(filename, std::ios::binary | std::ios::ate); + if (!file.is_open()) { + create(0, nullptr); + return; + } + file.exceptions(std::ifstream::failbit | std::ifstream::badbit); + const size_t total = static_cast(file.tellg()); + file.seekg(0, std::ios::beg); + std::array magic{}; + u32 version{}; + if (total < magic.size() + sizeof(version)) { + create(0, nullptr); + return; + } + file.read(magic.data(), magic.size()) + .read(reinterpret_cast(&version), sizeof(version)); + if (magic != STATIC_CACHE_MAGIC_NUMBER || version != STATIC_CACHE_VERSION) { + LOG_INFO(Render_Vulkan, "Discarding stale static pipeline cache"); + create(0, nullptr); + return; + } + data.resize(total - magic.size() - sizeof(version)); + file.read(data.data(), static_cast(data.size())); + } catch (const std::ios_base::failure& e) { + LOG_WARNING(Render_Vulkan, "Failed to read static pipeline cache: {}", e.what()); + create(0, nullptr); + return; + } + create(data.size(), data.empty() ? nullptr : data.data()); + LOG_INFO(Render_Vulkan, "Loaded {} bytes of static pipeline cache", data.size()); +} + +void Device::SaveStaticPipelineCache() const { + if (!owns_static_pipeline_cache || !static_pipeline_cache) { + return; + } + const auto filename = StaticPipelineCacheFilename(); + if (filename.empty()) { + return; + } + size_t size = 0; + std::vector data; + static_pipeline_cache.Read(&size, nullptr); + if (size == 0) { + return; + } + data.resize(size); + static_pipeline_cache.Read(&size, data.data()); + try { + std::ofstream file(filename, std::ios::binary | std::ios::trunc); + file.exceptions(std::ofstream::failbit); + if (!file.is_open()) { + return; + } + file.write(STATIC_CACHE_MAGIC_NUMBER.data(), STATIC_CACHE_MAGIC_NUMBER.size()) + .write(reinterpret_cast(&STATIC_CACHE_VERSION), + sizeof(STATIC_CACHE_VERSION)) + .write(data.data(), static_cast(size)); + } catch (const std::ios_base::failure& e) { + LOG_WARNING(Render_Vulkan, "Failed to write static pipeline cache: {}", e.what()); + Common::FS::RemoveFile(filename); + } +} + VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags wanted_usage, FormatType format_type) const { if (IsFormatSupported(wanted_format, wanted_usage, format_type)) { @@ -939,6 +1075,7 @@ bool Device::GetSuitability(bool requires_swapchain) { FOR_EACH_VK_FEATURE_EXT(FEATURE_EXTENSION); FOR_EACH_VK_EXTENSION(EXTENSION); + FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION); if (supported_extensions.contains(VK_KHR_ROBUSTNESS_2_EXTENSION_NAME)) { loaded_extensions.erase(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); @@ -951,6 +1088,17 @@ bool Device::GetSuitability(bool requires_swapchain) { extensions.robustness_2 = false; } +#ifdef __ANDROID__ + if (extensions.external_memory_ahb && !extensions.queue_family_foreign) { + LOG_INFO(Render_Vulkan, + "Not loading {} because its dependency {} is unavailable", + VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME, + VK_EXT_QUEUE_FAMILY_FOREIGN_EXTENSION_NAME); + loaded_extensions.erase(VK_ANDROID_EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER_EXTENSION_NAME); + extensions.external_memory_ahb = false; + } +#endif + #undef FEATURE_EXTENSION #undef EXTENSION @@ -1083,6 +1231,11 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PUSH_DESCRIPTOR_PROPERTIES_KHR; SetNext(next, properties.push_descriptor); } + if (extensions.descriptor_buffer) { + properties.descriptor_buffer.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_PROPERTIES_EXT; + SetNext(next, properties.descriptor_buffer); + } if (extensions.subgroup_size_control || features.subgroup_size_control.subgroupSizeControl) { properties.subgroup_size_control.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES; @@ -1098,6 +1251,26 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_PROPERTIES_KHR; SetNext(next, properties.maintenance5); } + if (extensions.external_memory_host) { + properties.external_memory_host.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT; + SetNext(next, properties.external_memory_host); + } + if (extensions.maintenance3 || instance_version >= VK_API_VERSION_1_1) { + properties.maintenance3.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_3_PROPERTIES; + SetNext(next, properties.maintenance3); + } + if (extensions.maintenance4 || features.maintenance4.maintenance4) { + properties.maintenance4.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_4_PROPERTIES; + SetNext(next, properties.maintenance4); + } + if (instance_version >= VK_API_VERSION_1_2) { + properties.depth_stencil_resolve.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DEPTH_STENCIL_RESOLVE_PROPERTIES; + SetNext(next, properties.depth_stencil_resolve); + } // Perform the property fetch. physical.GetProperties2(properties2); @@ -1209,10 +1382,20 @@ void Device::RemoveUnsuitableExtensions() { RemoveExtensionFeatureIfUnsuitable(extensions.depth_bias_control, features.depth_bias_control, VK_EXT_DEPTH_BIAS_CONTROL_EXTENSION_NAME); + // VK_EXT_depth_clamp_zero_one + extensions.depth_clamp_zero_one = features.depth_clamp_zero_one.depthClampZeroOne; + RemoveExtensionFeatureIfUnsuitable(extensions.depth_clamp_zero_one, + features.depth_clamp_zero_one, + VK_EXT_DEPTH_CLAMP_ZERO_ONE_EXTENSION_NAME); + // VK_EXT_depth_clip_control extensions.depth_clip_control = features.depth_clip_control.depthClipControl; RemoveExtensionFeatureIfUnsuitable(extensions.depth_clip_control, features.depth_clip_control, VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME); + // VK_EXT_depth_clip_enable + extensions.depth_clip_enable = features.depth_clip_enable.depthClipEnable; + RemoveExtensionFeatureIfUnsuitable(extensions.depth_clip_enable, features.depth_clip_enable, + VK_EXT_DEPTH_CLIP_ENABLE_EXTENSION_NAME); // VK_EXT_extended_dynamic_state extensions.extended_dynamic_state = features.extended_dynamic_state.extendedDynamicState; @@ -1326,6 +1509,12 @@ void Device::RemoveUnsuitableExtensions() { features.subgroup_size_control, VK_EXT_SUBGROUP_SIZE_CONTROL_EXTENSION_NAME); + // VK_EXT_texture_compression_astc_hdr + extensions.texture_compression_astc_hdr = false; + RemoveExtensionFeatureIfUnsuitable(extensions.texture_compression_astc_hdr, + features.texture_compression_astc_hdr, + VK_EXT_TEXTURE_COMPRESSION_ASTC_HDR_EXTENSION_NAME); + // VK_EXT_transform_feedback extensions.transform_feedback = features.transform_feedback.transformFeedback && @@ -1353,7 +1542,18 @@ void Device::RemoveUnsuitableExtensions() { VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME); } + // VK_KHR_shader_quad_control + extensions.shader_quad_control = features.shader_quad_control.shaderQuadControl; + RemoveExtensionFeatureIfUnsuitable(extensions.shader_quad_control, features.shader_quad_control, + VK_KHR_SHADER_QUAD_CONTROL_EXTENSION_NAME); + // VK_KHR_workgroup_memory_explicit_layout +#ifdef __ANDROID__ + if (!Settings::values.workgroup_memory_explicit_layout.GetValue()) { + LOG_INFO(Render_Vulkan, "Workgroup memory explicit layout disabled by user setting"); + features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout = false; + } +#endif extensions.workgroup_memory_explicit_layout = features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout && features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayoutScalarBlockLayout; @@ -1373,6 +1573,15 @@ void Device::RemoveUnsuitableExtensions() { extensions.maintenance3 = loaded_extensions.contains(VK_KHR_MAINTENANCE_3_EXTENSION_NAME); RemoveExtensionIfUnsuitable(extensions.maintenance3, VK_KHR_MAINTENANCE_3_EXTENSION_NAME); + // VK_KHR_dynamic_rendering + if (!Settings::values.dynamic_rendering.GetValue()) { + LOG_INFO(Render_Vulkan, "Dynamic rendering disabled by user setting"); + features.dynamic_rendering.dynamicRendering = false; + } + extensions.dynamic_rendering = features.dynamic_rendering.dynamicRendering; + RemoveExtensionFeatureIfUnsuitable(extensions.dynamic_rendering, features.dynamic_rendering, + VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME); + // VK_KHR_maintenance4 extensions.maintenance4 = features.maintenance4.maintenance4; RemoveExtensionFeatureIfUnsuitable(extensions.maintenance4, features.maintenance4, @@ -1444,8 +1653,28 @@ std::optional Device::GetSamplerHeapBudget() const { return sampler_heap_budget; } +Device::MemoryBudgetInfo Device::GetMemoryBudgetInfo() const { + std::array budgets{}; + vmaGetHeapBudgets(allocator, budgets.data()); + MemoryBudgetInfo info{}; + for (const size_t heap : valid_heap_memory) { + info.usage += budgets[heap].usage; + info.budget += budgets[heap].budget; + info.block_bytes += budgets[heap].statistics.blockBytes; + info.allocation_bytes += budgets[heap].statistics.allocationBytes; + } + return info; +} + +void Device::TickAllocatorFrame() const { + vmaSetCurrentFrameIndex(allocator, ++allocator_frame_index); +} + u64 Device::GetDeviceMemoryUsage() const { - VkPhysicalDeviceMemoryBudgetPropertiesEXT budget; + if (!extensions.memory_budget) { + return GetMemoryBudgetInfo().allocation_bytes; + } + VkPhysicalDeviceMemoryBudgetPropertiesEXT budget{}; budget.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT; budget.pNext = nullptr; physical.GetMemoryProperties(&budget); @@ -1467,12 +1696,27 @@ void Device::CollectPhysicalMemoryInfo() { device_access_memory = 0; u64 device_initial_usage = 0; u64 local_memory = 0; + const auto heap_has_usable_type = [&mem_properties](size_t heap) { + for (u32 index = 0; index < mem_properties.memoryTypeCount; ++index) { + if (mem_properties.memoryTypes[index].heapIndex != heap) { + continue; + } + if ((mem_properties.memoryTypes[index].propertyFlags & + VK_MEMORY_PROPERTY_PROTECTED_BIT) == 0) { + return true; + } + } + return false; + }; for (size_t element = 0; element < num_properties; ++element) { const bool is_heap_local = (mem_properties.memoryHeaps[element].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0; if (!is_integrated && !is_heap_local) { continue; } + if (!heap_has_usable_type(element)) { + continue; + } valid_heap_memory.push_back(element); if (is_heap_local) { local_memory += mem_properties.memoryHeaps[element].size; @@ -1484,18 +1728,35 @@ void Device::CollectPhysicalMemoryInfo() { } device_access_memory += mem_properties.memoryHeaps[element].size; } + const u64 committed_backing = Common::GetCommittedBackingSize(); + if (committed_backing != 0) { + LOG_INFO(Render_Vulkan, "Discounting {} MiB of guest memory committed by the host", + committed_backing >> 20); + local_memory -= std::min(local_memory, committed_backing); + device_access_memory -= std::min(device_access_memory, committed_backing); + } if (is_integrated) { - const s64 available_memory = static_cast(device_access_memory - device_initial_usage); - const u64 memory_size = Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive ? 6_GiB : 4_GiB; - device_access_memory = static_cast(std::max(std::min(available_memory - 8_GiB, memory_size), std::min(local_memory, memory_size))); + const bool aggressive = + Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive; + const s64 available_memory = + static_cast(device_access_memory) - static_cast(device_initial_usage); + const u64 reserve = std::min(local_memory / 8, 1_GiB); + const u64 usable = available_memory > static_cast(reserve) + ? static_cast(available_memory) - reserve + : 0; + const u64 cap = (local_memory / 8) * (aggressive ? 6 : 5); + device_access_memory = std::max(std::min(usable, cap), 1_GiB); } else { const u64 reserve_memory = std::min(device_access_memory / 8, 1_GiB); device_access_memory -= reserve_memory; if (Settings::values.vram_usage_mode.GetValue() != Settings::VramUsageMode::Aggressive) { // Account for resolution scaling in memory limits - const size_t normal_memory = 6_GiB; - const size_t scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1); - device_access_memory = std::min(device_access_memory, normal_memory + scaler_memory); + const u64 normal_memory = 6_GiB; + const u64 scaler_memory = 1_GiB * Settings::values.resolution_info.ScaleUp(1); + const u64 baseline = normal_memory + scaler_memory; + const u64 proportional = (device_access_memory / 4) * 3; + device_access_memory = + std::min(device_access_memory, std::max(baseline, proportional)); } } } diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index c449f60324..6820ee31da 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -36,6 +36,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, DescriptorIndexing, DESCRIPTOR_INDEXING, descriptor_indexing) \ FEATURE(EXT, HostQueryReset, HOST_QUERY_RESET, host_query_reset) \ FEATURE(KHR, 8BitStorage, 8BIT_STORAGE, bit8_storage) \ + FEATURE(KHR, BufferDeviceAddress, BUFFER_DEVICE_ADDRESS, buffer_device_address) \ FEATURE(KHR, TimelineSemaphore, TIMELINE_SEMAPHORE, timeline_semaphore) #define FOR_EACH_VK_FEATURE_1_3(FEATURE) \ @@ -43,6 +44,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, ShaderDemoteToHelperInvocation, SHADER_DEMOTE_TO_HELPER_INVOCATION, \ shader_demote_to_helper_invocation) \ FEATURE(EXT, SubgroupSizeControl, SUBGROUP_SIZE_CONTROL, subgroup_size_control) \ + FEATURE(KHR, DynamicRendering, DYNAMIC_RENDERING, dynamic_rendering) \ FEATURE(KHR, Maintenance4, MAINTENANCE_4, maintenance4) \ FEATURE(KHR, Synchronization2, SYNCHRONIZATION_2, synchronization2) @@ -54,7 +56,10 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(EXT, ColorWriteEnable, COLOR_WRITE_ENABLE, color_write_enable) \ FEATURE(EXT, CustomBorderColor, CUSTOM_BORDER_COLOR, custom_border_color) \ FEATURE(EXT, DepthBiasControl, DEPTH_BIAS_CONTROL, depth_bias_control) \ + FEATURE(EXT, DepthClampZeroOne, DEPTH_CLAMP_ZERO_ONE, depth_clamp_zero_one) \ FEATURE(EXT, DepthClipControl, DEPTH_CLIP_CONTROL, depth_clip_control) \ + FEATURE(EXT, DepthClipEnable, DEPTH_CLIP_ENABLE, depth_clip_enable) \ + FEATURE(EXT, DescriptorBuffer, DESCRIPTOR_BUFFER, descriptor_buffer) \ FEATURE(EXT, ExtendedDynamicState, EXTENDED_DYNAMIC_STATE, extended_dynamic_state) \ FEATURE(EXT, ExtendedDynamicState2, EXTENDED_DYNAMIC_STATE_2, extended_dynamic_state2) \ FEATURE(EXT, ExtendedDynamicState3, EXTENDED_DYNAMIC_STATE_3, extended_dynamic_state3) \ @@ -71,6 +76,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(KHR, Maintenance6, MAINTENANCE_6, maintenance6) \ FEATURE(KHR, PipelineExecutableProperties, PIPELINE_EXECUTABLE_PROPERTIES, \ pipeline_executable_properties) \ + FEATURE(KHR, ShaderQuadControl, SHADER_QUAD_CONTROL, shader_quad_control) \ FEATURE(KHR, WorkgroupMemoryExplicitLayout, WORKGROUP_MEMORY_EXPLICIT_LAYOUT, \ workgroup_memory_explicit_layout) \ FEATURE(EXT, TextureCompressionASTCHDR, TEXTURE_COMPRESSION_ASTC_HDR, \ @@ -79,10 +85,13 @@ VK_DEFINE_HANDLE(VmaAllocator) // Define miscellaneous extensions which may be used by the implementation here. #define FOR_EACH_VK_EXTENSION(EXTENSION) \ + EXTENSION(EXT, ASTC_DECODE_MODE, astc_decode_mode) \ EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering) \ EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization) \ EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted) \ + EXTENSION(EXT, EXTERNAL_MEMORY_HOST, external_memory_host) \ EXTENSION(EXT, MEMORY_BUDGET, memory_budget) \ + EXTENSION(EXT, PIPELINE_CREATION_FEEDBACK, pipeline_creation_feedback) \ EXTENSION(EXT, ROBUSTNESS_2, robustness_2) \ EXTENSION(EXT, SAMPLER_FILTER_MINMAX, sampler_filter_minmax) \ EXTENSION(EXT, SHADER_STENCIL_EXPORT, shader_stencil_export) \ @@ -111,6 +120,14 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION(IMG, FILTER_CUBIC, filter_cubic_img) \ EXTENSION(QCOM, FILTER_CUBIC_WEIGHTS, filter_cubic_weights) +#ifdef __ANDROID__ +#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) \ + EXTENSION(EXT, QUEUE_FAMILY_FOREIGN, queue_family_foreign) \ + EXTENSION(ANDROID, EXTERNAL_MEMORY_ANDROID_HARDWARE_BUFFER, external_memory_ahb) +#else +#define FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION) +#endif + // Define extensions which must be supported. #define FOR_EACH_VK_MANDATORY_EXTENSION(EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_VERTEX_ATTRIBUTE_DIVISOR_EXTENSION_NAME) \ @@ -255,6 +272,17 @@ public: return allocator; } + struct MemoryBudgetInfo { + u64 usage; + u64 budget; + u64 block_bytes; + u64 allocation_bytes; + }; + + MemoryBudgetInfo GetMemoryBudgetInfo() const; + + void TickAllocatorFrame() const; + /// Returns the logical device. const vk::Device& GetLogical() const { return logical; @@ -265,6 +293,10 @@ public: return physical; } + VkPipelineCache StaticPipelineCache() const noexcept { + return *static_pipeline_cache; + } + /// Returns the main graphics queue. vk::Queue GetGraphicsQueue() const { return graphics_queue; @@ -369,8 +401,7 @@ FN_MAX_LIMIT_LIST } bool IsOptimalAstcSupported() const { - return features.features.textureCompressionASTC_LDR && - features.texture_compression_astc_hdr.textureCompressionASTC_HDR; + return is_optimal_astc_supported; } /// Returns true if BCn is natively supported. @@ -385,7 +416,8 @@ FN_MAX_LIMIT_LIST /// Returns true if descriptor aliasing is natively supported. bool IsDescriptorAliasingSupported() const { - return GetDriverID() != VK_DRIVER_ID_QUALCOMM_PROPRIETARY; + return features.descriptor_indexing.descriptorBindingPartiallyBound && + features.descriptor_indexing.descriptorBindingVariableDescriptorCount; } bool IsSampledImageArrayNonUniformIndexingSupported() const { @@ -468,6 +500,29 @@ FN_MAX_LIMIT_LIST return properties.push_descriptor.maxPushDescriptors; } + /// Returns true if the device supports descriptor buffers. + bool IsExtPipelineCreationFeedbackSupported() const { + return extensions.pipeline_creation_feedback; + } + + bool IsRobustBufferAccessEnabled() const { + return features.features.robustBufferAccess == VK_TRUE; + } + + bool IsExtDescriptorBufferSupported() const { + return extensions.descriptor_buffer; + } + + /// Returns the descriptor buffer properties of the device. + const VkPhysicalDeviceDescriptorBufferPropertiesEXT& DescriptorBufferProperties() const { + return properties.descriptor_buffer; + } + + /// Returns true if the device supports buffer device address. + bool IsBufferDeviceAddressSupported() const { + return extensions.buffer_device_address; + } + /// Returns true if formatless image load is supported. bool IsFormatlessImageLoadSupported() const { return features.features.shaderStorageImageReadWithoutFormat; @@ -554,6 +609,11 @@ FN_MAX_LIMIT_LIST features.workgroup_memory_explicit_layout.workgroupMemoryExplicitLayout16BitAccess && features.features.shaderInt16; } + + /// Returns true if the device supports VK_KHR_shader_quad_control. + bool IsKhrShaderQuadControlSupported() const { + return extensions.shader_quad_control && features.shader_quad_control.shaderQuadControl; + } /// Returns true if the device supports VK_KHR_image_format_list. bool IsKhrImageFormatListSupported() const { @@ -586,7 +646,6 @@ FN_MAX_LIMIT_LIST } /// Returns true if depth/stencil operations can be performed efficiently. - /// Either through shader export or hardware blits. bool CanPerformDepthStencilOperations() const { return extensions.shader_stencil_export || is_blit_depth24_stencil8_supported || is_blit_depth32_stencil8_supported; @@ -602,6 +661,16 @@ FN_MAX_LIMIT_LIST return extensions.depth_clip_control; } + /// Returns true if the device supports VK_EXT_depth_clamp_zero_one. + bool IsExtDepthClampZeroOneSupported() const { + return extensions.depth_clamp_zero_one; + } + + /// Returns true if the device supports VK_EXT_depth_clip_enable. + bool IsExtDepthClipEnableSupported() const { + return extensions.depth_clip_enable; + } + /// Returns true if the device supports VK_EXT_depth_bias_control. bool IsExtDepthBiasControlSupported() const { return extensions.depth_bias_control; @@ -734,6 +803,38 @@ FN_MAX_LIMIT_LIST return features.line_rasterization.stippledRectangularLines != VK_FALSE; } + VkLineRasterizationModeEXT GetLineRasterizationMode(bool wants_smooth) const { + if (wants_smooth && SupportsSmoothLines()) { + return VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT; + } + if (SupportsRectangularLines()) { + return VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT; + } + return VK_LINE_RASTERIZATION_MODE_DEFAULT_EXT; + } + + bool SupportsStippleForMode(VkLineRasterizationModeEXT mode) const { + switch (mode) { + case VK_LINE_RASTERIZATION_MODE_RECTANGULAR_SMOOTH_EXT: + return features.line_rasterization.stippledSmoothLines != VK_FALSE; + case VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT: + return features.line_rasterization.stippledBresenhamLines != VK_FALSE; + default: + return features.line_rasterization.stippledRectangularLines != VK_FALSE; + } + } + + float ClampLineWidth(float width) const { + if (!features.features.wideLines) { + return 1.0f; + } + const auto& range = properties.properties.limits.lineWidthRange; + if (!(width >= range[0])) { + return range[0]; + } + return width > range[1] ? range[1] : width; + } + bool SupportsAlphaToOne() const { return features.features.alphaToOne != VK_FALSE; } @@ -812,10 +913,45 @@ FN_MAX_LIMIT_LIST features.shader_atomic_int64.shaderSharedInt64Atomics; } + /// Returns true if the device supports VK_EXT_conditional_rendering. bool IsExtConditionalRendering() const { return extensions.conditional_rendering; } + /// Returns true if the device supports VK_EXT_astc_decode_mode. + bool IsExtAstcDecodeModeSupported() const { + return extensions.astc_decode_mode; + } + + /// Returns true if the device supports VK_EXT_external_memory_host. + bool IsExtExternalMemoryHostSupported() const { + return extensions.external_memory_host; + } + + bool IsExtExternalMemoryAhbSupported() const { +#ifdef __ANDROID__ + return extensions.external_memory_ahb && extensions.queue_family_foreign; +#else + return false; +#endif + } + + bool IsDescriptorBindingPartiallyBoundSupported() const { + return features.descriptor_indexing.descriptorBindingPartiallyBound; + } + + u64 GetMinImportedHostPointerAlignment() const { + return properties.external_memory_host.minImportedHostPointerAlignment; + } + + u64 GetMaxBufferSize() const { + return properties.maintenance4.maxBufferSize; + } + + u64 GetMaxMemoryAllocationSize() const { + return properties.maintenance3.maxMemoryAllocationSize; + } + bool HasTimelineSemaphore() const; /// Returns true if the device supports VK_KHR_synchronization2. @@ -876,6 +1012,10 @@ FN_MAX_LIMIT_LIST u64 GetDeviceMemoryUsage() const; + VkSampleCountFlags GetStorageImageSampleCounts() const { + return properties.properties.limits.storageImageSampleCounts; + } + u32 GetSetsPerPool() const { return sets_per_pool; } @@ -941,6 +1081,33 @@ FN_MAX_LIMIT_LIST return extensions.maintenance3; } + static constexpr bool ENABLE_DYNAMIC_RENDERING = true; + + /// Returns true if the device supports VK_KHR_dynamic_rendering. + bool IsKhrDynamicRenderingSupported() const { + return ENABLE_DYNAMIC_RENDERING && extensions.dynamic_rendering; + } + + /// Returns the resolve mode to use for multisampled depth. + VkResolveModeFlagBits GetDepthResolveMode() const { + return PickResolveMode(properties.depth_stencil_resolve.supportedDepthResolveModes); + } + + /// Same as GetDepthResolveMode, for the stencil aspect. + VkResolveModeFlagBits GetStencilResolveMode() const { + return PickResolveMode(properties.depth_stencil_resolve.supportedStencilResolveModes); + } + + /// Returns true if depth and stencil may resolve with independent modes. + bool IsIndependentResolveSupported() const { + return properties.depth_stencil_resolve.independentResolve == VK_TRUE; + } + + /// Returns true if one aspect may resolve while the other does not. + bool IsIndependentResolveNoneSupported() const { + return properties.depth_stencil_resolve.independentResolveNone == VK_TRUE; + } + /// Returns true if the device supports VK_KHR_maintenance4. bool IsKhrMaintenance4Supported() const { return extensions.maintenance4; @@ -1022,6 +1189,22 @@ FN_MAX_LIMIT_LIST void ShutdownGPULogging(); private: + /// Picks a usable resolve mode out of a supported-modes mask. SAMPLE_ZERO is preferred because + /// it is the only mode the spec guarantees whenever any depth/stencil resolve is supported, + /// and averaging depth samples is not meaningful for depth testing anyway. + static VkResolveModeFlagBits PickResolveMode(VkResolveModeFlags supported) { + if ((supported & VK_RESOLVE_MODE_SAMPLE_ZERO_BIT) != 0) { + return VK_RESOLVE_MODE_SAMPLE_ZERO_BIT; + } + if ((supported & VK_RESOLVE_MODE_MIN_BIT) != 0) { + return VK_RESOLVE_MODE_MIN_BIT; + } + if ((supported & VK_RESOLVE_MODE_MAX_BIT) != 0) { + return VK_RESOLVE_MODE_MAX_BIT; + } + return VK_RESOLVE_MODE_NONE; + } + /// Checks if the physical device is suitable and configures the object state /// with all necessary info about its properties. bool GetSuitability(bool requires_swapchain); @@ -1057,14 +1240,20 @@ private: /// Returns true if the device natively supports blitting depth stencil images. bool TestDepthStencilBlits(VkFormat format) const; + void LoadStaticPipelineCache(); + void SaveStaticPipelineCache() const; + private: VkInstance instance; ///< Vulkan instance. VmaAllocator allocator; ///< VMA allocator. + mutable u32 allocator_frame_index{}; vk::DeviceDispatch dld; ///< Device function pointers. vk::PhysicalDevice physical; ///< Physical device. vk::Device logical; ///< Logical device. vk::Queue graphics_queue; ///< Main graphics queue. vk::Queue present_queue; ///< Main present queue. + vk::PipelineCache static_pipeline_cache; + bool owns_static_pipeline_cache{}; u32 instance_version{}; ///< Vulkan instance version. u32 graphics_family{}; ///< Main graphics queue family index. u32 present_family{}; ///< Main present queue family index. @@ -1079,6 +1268,7 @@ private: FOR_EACH_VK_FEATURE_1_4(FEATURE); FOR_EACH_VK_FEATURE_EXT(FEATURE); FOR_EACH_VK_EXTENSION(EXTENSION); + FOR_EACH_VK_PLATFORM_EXTENSION(EXTENSION); #undef EXTENSION #undef FEATURE @@ -1107,9 +1297,14 @@ private: VkPhysicalDeviceSubgroupProperties subgroup_properties{}; VkPhysicalDeviceFloatControlsProperties float_controls{}; VkPhysicalDevicePushDescriptorPropertiesKHR push_descriptor{}; + VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{}; VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{}; VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{}; + VkPhysicalDeviceMaintenance3Properties maintenance3{}; + VkPhysicalDeviceMaintenance4Properties maintenance4{}; VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{}; + VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{}; + VkPhysicalDeviceExternalMemoryHostPropertiesEXT external_memory_host{}; VkPhysicalDeviceProperties properties{}; }; diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index e57864ede8..7a641e0bcb 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -25,31 +25,36 @@ #include "video_core/gpu_logging/gpu_logging.h" #include "common/settings.h" +#ifdef __ANDROID__ +#include +#endif + namespace Vulkan { namespace { -// Helpers translating MemoryUsage to flags/usage - - [[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) { - switch (usage) { - case MemoryUsage::DeviceLocal: - return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - case MemoryUsage::Upload: - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - case MemoryUsage::Download: - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; - case MemoryUsage::Stream: - return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + [[nodiscard]] std::optional FindImportMemoryType( + const VkPhysicalDeviceMemoryProperties &props, u32 type_mask) { + const auto find = [&](VkMemoryPropertyFlags wanted) -> std::optional { + for (u32 i = 0; i < props.memoryTypeCount; ++i) { + if (((type_mask >> i) & 1u) != 0 && + (props.memoryTypes[i].propertyFlags & wanted) == wanted) { + return i; + } + } + return std::nullopt; + }; + auto type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | + VK_MEMORY_PROPERTY_HOST_CACHED_BIT); + if (!type_index) { + type_index = find(VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); } - ASSERT_MSG(false, "Invalid memory usage={}", usage); - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + return type_index; } +// Helpers translating MemoryUsage to flags/usage + [[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferredVmaFlags(MemoryUsage usage) { if (usage == MemoryUsage::Download) { return VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; @@ -86,125 +91,286 @@ namespace Vulkan { } -// This avoids calling vkGetBufferMemoryRequirements* directly. - template - static VkBuffer GetVkHandleFromBuffer(const T &buf) { - if constexpr (requires { static_cast(buf); }) { - return static_cast(buf); - } else if constexpr (requires {{ buf.GetHandle() } -> std::convertible_to; }) { - return buf.GetHandle(); - } else if constexpr (requires {{ buf.Handle() } -> std::convertible_to; }) { - return buf.Handle(); - } else if constexpr (requires {{ buf.vk_handle() } -> std::convertible_to; }) { - return buf.vk_handle(); - } else { - static_assert(sizeof(T) == 0, "Cannot extract VkBuffer handle from vk::Buffer"); - return VK_NULL_HANDLE; - } - } - } // namespace -//MemoryCommit is now VMA-backed - MemoryCommit::MemoryCommit(VmaAllocator alloc, VmaAllocation a, - const VmaAllocationInfo &info) noexcept - : allocator{alloc}, allocation{a}, memory{info.deviceMemory}, - offset{info.offset}, size{info.size}, mapped_ptr{info.pMappedData} { - // Log GPU memory allocation - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_memory_tracking.GetValue()) { - GPU::Logging::GPULogger::GetInstance().LogMemoryAllocation( - reinterpret_cast(memory), - static_cast(size), - 0 // Memory property flags (not easily available from VMA) - ); + HostMemoryImport::HostMemoryImport(const Device &device_, void *base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base) + : device{device_} { + if (ImportHardwareBuffers(hardware_buffers, hardware_buffer_window, hardware_buffer_base, + size)) { + return; } + if (!hardware_buffers.empty()) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, guest memory is backed by hardware buffers that " + "could not be imported"); + return; + } + if (ImportHostPointer(base, size)) { + return; + } + LOG_INFO(Render_Vulkan, "Unified memory disabled, no host memory import path"); } - MemoryCommit::~MemoryCommit() { Release(); } - - MemoryCommit::MemoryCommit(MemoryCommit &&rhs) noexcept - : allocator{std::exchange(rhs.allocator, nullptr)}, - allocation{std::exchange(rhs.allocation, nullptr)}, - memory{std::exchange(rhs.memory, VK_NULL_HANDLE)}, - offset{std::exchange(rhs.offset, 0)}, - size{std::exchange(rhs.size, 0)}, - mapped_ptr{std::exchange(rhs.mapped_ptr, nullptr)} {} - - MemoryCommit &MemoryCommit::operator=(MemoryCommit &&rhs) noexcept { - if (this != &rhs) { - Release(); - allocator = std::exchange(rhs.allocator, nullptr); - allocation = std::exchange(rhs.allocation, nullptr); - memory = std::exchange(rhs.memory, VK_NULL_HANDLE); - offset = std::exchange(rhs.offset, 0); - size = std::exchange(rhs.size, 0); - mapped_ptr = std::exchange(rhs.mapped_ptr, nullptr); + bool HostMemoryImport::ImportHostPointer(void *base, size_t size) { + if (!device.IsExtExternalMemoryHostSupported()) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, VK_EXT_external_memory_host is not supported"); + return false; } - return *this; - } - - std::span MemoryCommit::Map() - { - if (!allocation) return {}; - if (!mapped_ptr) { - if (vmaMapMemory(allocator, allocation, &mapped_ptr) != VK_SUCCESS) return {}; + const u64 alignment = device.GetMinImportedHostPointerAlignment(); + if (alignment == 0 || !Common::IsAligned(reinterpret_cast(base), alignment) || + !Common::IsAligned(size, alignment)) { + LOG_INFO(Render_Vulkan, + "Unified memory disabled, host allocation does not satisfy alignment {}", + alignment); + return false; } - const size_t n = static_cast(std::min(size, - (std::numeric_limits::max)())); - return std::span{static_cast(mapped_ptr), n}; - } - - std::span MemoryCommit::Map() const - { - if (!allocation) return {}; - if (!mapped_ptr) { - void *p = nullptr; - if (vmaMapMemory(allocator, allocation, &p) != VK_SUCCESS) return {}; - const_cast(this)->mapped_ptr = p; + using namespace Common::Literals; + VkDeviceSize candidate_window = 1_GiB; + const u64 max_buffer_size = device.GetMaxBufferSize(); + if (max_buffer_size != 0 && max_buffer_size < candidate_window) { + candidate_window = max_buffer_size; } - const size_t n = static_cast(std::min(size, - (std::numeric_limits::max)())); - return std::span{static_cast(mapped_ptr), n}; - } - - void MemoryCommit::Unmap() - { - if (allocation && mapped_ptr) { - vmaUnmapMemory(allocator, allocation); - mapped_ptr = nullptr; + const u64 max_allocation_size = device.GetMaxMemoryAllocationSize(); + if (max_allocation_size != 0 && max_allocation_size < candidate_window) { + candidate_window = max_allocation_size; } - } + candidate_window = Common::AlignDown(candidate_window, alignment); + if (candidate_window == 0) { + return false; + } + window_size = candidate_window; - void MemoryCommit::Release() { - if (allocation && allocator) { - // Log GPU memory deallocation - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_memory_tracking.GetValue() && - memory != VK_NULL_HANDLE) { - GPU::Logging::GPULogger::GetInstance().LogMemoryDeallocation( - reinterpret_cast(memory) - ); + const auto &logical = device.GetLogical(); + const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; + + for (size_t offset = 0; offset < size; offset += window_size) { + u8 *const window_base = static_cast(base) + offset; + const VkDeviceSize window_len = + (std::min)(static_cast(size - offset), window_size); + VkMemoryHostPointerPropertiesEXT host_props{ + .sType = VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT, + .pNext = nullptr, + .memoryTypeBits = 0, + }; + if (logical.GetMemoryHostPointerPropertiesEXT( + VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, window_base, + &host_props) != VK_SUCCESS || + host_props.memoryTypeBits == 0) { + break; } - - if (mapped_ptr) { - vmaUnmapMemory(allocator, allocation); - mapped_ptr = nullptr; + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & host_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > window_len) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const u32 heap_index = memory_props.memoryTypes[*type_index].heapIndex; + const VkDeviceSize heap_size = memory_props.memoryHeaps[heap_index].size; + if (imported_size + window_len > heap_size / 2) { + LOG_INFO(Render_Vulkan, + "Stopping guest memory import at {} MiB to leave room on heap {} of {} MiB", + imported_size >> 20, heap_index, heap_size >> 20); + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportMemoryHostPointerInfoEXT import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT, + .pNext = nullptr, + .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT, + .pHostPointer = window_base, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &import_info, + .allocationSize = window_len, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + }); + imported_size += static_cast(window_len); + } + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Host pointer import failed"); + return false; + } + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory for unified memory access in {} windows", + imported_size >> 20, windows.size()); + return true; + } + + bool HostMemoryImport::ImportHardwareBuffers( + [[maybe_unused]] std::span hardware_buffers, + [[maybe_unused]] size_t hardware_buffer_window, + [[maybe_unused]] size_t hardware_buffer_base, [[maybe_unused]] size_t size) { +#ifdef __ANDROID__ + if (hardware_buffers.empty() || hardware_buffer_window == 0 || + !device.IsExtExternalMemoryAhbSupported()) { + return false; + } + const u64 max_allocation_size = device.GetMaxMemoryAllocationSize(); + if (max_allocation_size != 0 && hardware_buffer_window > max_allocation_size) { + LOG_WARNING(Render_Vulkan, + "Hardware buffer windows of {} MiB exceed the {} MiB allocation limit", + hardware_buffer_window >> 20, max_allocation_size >> 20); + return false; + } + if (hardware_buffer_base >= size) { + return false; + } + const auto &logical = device.GetLogical(); + const auto memory_props = device.GetPhysical().GetMemoryProperties().memoryProperties; + window_size = hardware_buffer_window; + base_offset = hardware_buffer_base; + for (size_t i = 0; i < hardware_buffers.size(); ++i) { + const size_t offset = hardware_buffer_base + i * hardware_buffer_window; + if (offset >= size) { + break; + } + const VkDeviceSize window_len = (std::min)( + static_cast(size - offset), + static_cast(hardware_buffer_window)); + VkAndroidHardwareBufferPropertiesANDROID ahb_props{ + .sType = VK_STRUCTURE_TYPE_ANDROID_HARDWARE_BUFFER_PROPERTIES_ANDROID, + .pNext = nullptr, + .allocationSize = 0, + .memoryTypeBits = 0, + }; + if (logical.GetAndroidHardwareBufferPropertiesANDROID(hardware_buffers[i], + &ahb_props) != VK_SUCCESS || + ahb_props.memoryTypeBits == 0 || ahb_props.allocationSize < window_len) { + break; + } + const VkExternalMemoryBufferCreateInfo external_info{ + .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, + .pNext = nullptr, + .handleTypes = + VK_EXTERNAL_MEMORY_HANDLE_TYPE_ANDROID_HARDWARE_BUFFER_BIT_ANDROID, + }; + const VkBufferCreateInfo buffer_ci{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = &external_info, + .flags = 0, + .size = window_len, + .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }; + VkBuffer new_buffer{}; + if (logical.CreateBufferRaw(buffer_ci, &new_buffer) != VK_SUCCESS) { + break; + } + const VkMemoryRequirements requirements = + logical.GetBufferMemoryRequirements(new_buffer); + const u32 type_mask = requirements.memoryTypeBits & ahb_props.memoryTypeBits; + if (type_mask == 0 || requirements.size > ahb_props.allocationSize) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const auto type_index = FindImportMemoryType(memory_props, type_mask); + if (!type_index) { + logical.DestroyBufferRaw(new_buffer); + break; + } + const VkImportAndroidHardwareBufferInfoANDROID import_info{ + .sType = VK_STRUCTURE_TYPE_IMPORT_ANDROID_HARDWARE_BUFFER_INFO_ANDROID, + .pNext = nullptr, + .buffer = hardware_buffers[i], + }; + const VkMemoryDedicatedAllocateInfo dedicated_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO, + .pNext = &import_info, + .image = VK_NULL_HANDLE, + .buffer = new_buffer, + }; + const VkMemoryAllocateInfo alloc_info{ + .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, + .pNext = &dedicated_info, + .allocationSize = ahb_props.allocationSize, + .memoryTypeIndex = *type_index, + }; + vk::DeviceMemory memory = logical.TryAllocateMemory(alloc_info); + if (!memory) { + logical.DestroyBufferRaw(new_buffer); + break; + } + if (logical.BindBufferMemory(new_buffer, *memory, 0) != VK_SUCCESS) { + logical.DestroyBufferRaw(new_buffer); + break; + } + windows.push_back(Window{ + .memory = std::move(memory), + .buffer = new_buffer, + }); + imported_size += static_cast(window_len); + } + if (windows.empty()) { + LOG_INFO(Render_Vulkan, "Hardware buffer import failed"); + window_size = 0; + base_offset = 0; + return false; + } + foreign_ownership = true; + LOG_INFO(Render_Vulkan, + "Imported {} MiB of guest memory at {:#x} via hardware buffers in {} windows", + imported_size >> 20, base_offset, windows.size()); + return true; +#else + return false; +#endif + } + + HostMemoryImport::~HostMemoryImport() { + for (Window &window : windows) { + if (window.buffer != VK_NULL_HANDLE) { + device.GetLogical().DestroyBufferRaw(window.buffer); } - vmaFreeMemory(allocator, allocation); } - allocation = nullptr; - allocator = nullptr; - memory = VK_NULL_HANDLE; - offset = 0; - size = 0; } MemoryAllocator::MemoryAllocator(const Device &device_) : device{device_}, allocator{device.GetAllocator()}, - properties{device_.GetPhysical().GetMemoryProperties().memoryProperties}, - buffer_image_granularity{ - device_.GetPhysical().GetProperties().limits.bufferImageGranularity} { + properties{device_.GetPhysical().GetMemoryProperties().memoryProperties} { // Preserve the previous "RenderDoc small heap" trimming behavior that we had in original vma minus the heap bug if (device.HasDebuggingToolAttached()) @@ -224,10 +390,25 @@ namespace Vulkan { MemoryAllocator::~MemoryAllocator() = default; + void MemoryAllocator::SetReclaimCallback(ReclaimCallback callback) { + reclaim_callback = std::move(callback); + vk::SetAllocatorOwnerThread(); + } + + bool MemoryAllocator::ReclaimAtLeast(u64 hint_bytes) const { + if (!reclaim_callback || in_reclaim) { + return false; + } + in_reclaim = true; + const u64 freed = reclaim_callback(hint_bytes); + in_reclaim = false; + return freed > 0; + } + vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const { const VmaAllocationCreateInfo alloc_ci = { - .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, + .flags = device.CanReportMemoryUsage() ? VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT : 0u, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, .requiredFlags = 0, .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, @@ -240,7 +421,26 @@ namespace Vulkan { VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + DEBUG_ASSERT(vk::OnAllocatorOwnerThread()); + + VkResult res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && ReclaimAtLeast(IMAGE_RECLAIM_HINT)) { + res = vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + } + + if (res != VK_SUCCESS) { + auto relaxed_ci = alloc_ci; + relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS) { + relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + res = vmaCreateImage(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + } + } + + vk::Check(res); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -262,7 +462,8 @@ namespace Vulkan { && device.GetDriverID() == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA) ? VK_MEMORY_PROPERTY_HOST_CACHED_BIT : 0; const VmaAllocationCreateInfo alloc_ci = { - .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage), + .flags = (device.CanReportMemoryUsage() ? VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT : 0u) | + MemoryUsageVmaFlags(usage), .usage = MemoryUsageVma(usage), .requiredFlags = 0, .preferredFlags = MemoryUsagePreferredVmaFlags(usage) | anv_flags, @@ -277,7 +478,28 @@ namespace Vulkan { VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + DEBUG_ASSERT(vk::OnAllocatorOwnerThread()); + + VkResult res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && ReclaimAtLeast(ci.size)) { + res = vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info); + } + + if (res != VK_SUCCESS) { + auto relaxed_ci = alloc_ci; + relaxed_ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, &alloc_info); + + if (res != VK_SUCCESS && + (relaxed_ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + relaxed_ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + res = vmaCreateBuffer(allocator, &ci, &relaxed_ci, &handle, &allocation, + &alloc_info); + } + } + + vk::Check(res); vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers @@ -299,77 +521,4 @@ namespace Vulkan { device.GetDispatchLoader()); } - MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements &reqs, MemoryUsage usage) - { - const auto vma_usage = MemoryUsageVma(usage); - VmaAllocationCreateInfo ci{}; - ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage); - ci.usage = vma_usage; - ci.memoryTypeBits = reqs.memoryTypeBits & valid_memory_types; - ci.requiredFlags = 0; - ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage); - - VmaAllocation a{}; - VmaAllocationInfo info{}; - - VkResult res = vmaAllocateMemory(allocator, &reqs, &ci, &a, &info); - - if (res != VK_SUCCESS) { - // Relax 1: drop budget constraint - auto ci2 = ci; - ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; - res = vmaAllocateMemory(allocator, &reqs, &ci2, &a, &info); - - // Relax 2: if we preferred DEVICE_LOCAL, drop that preference - if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { - auto ci3 = ci2; - ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - res = vmaAllocateMemory(allocator, &reqs, &ci3, &a, &info); - } - } - - vk::Check(res); - return MemoryCommit(allocator, a, info); - } - - MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage) { - // Allocate memory appropriate for this buffer automatically - const auto vma_usage = MemoryUsageVma(usage); - - VmaAllocationCreateInfo ci{}; - ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage); - ci.usage = vma_usage; - ci.requiredFlags = 0; - ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage); - ci.pool = VK_NULL_HANDLE; - ci.pUserData = nullptr; - ci.priority = 0.0f; - - const VkBuffer raw = *buffer; - - VmaAllocation a{}; - VmaAllocationInfo info{}; - - // Let VMA infer memory requirements from the buffer - VkResult res = vmaAllocateMemoryForBuffer(allocator, raw, &ci, &a, &info); - - if (res != VK_SUCCESS) { - auto ci2 = ci; - ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; - res = vmaAllocateMemoryForBuffer(allocator, raw, &ci2, &a, &info); - - if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { - auto ci3 = ci2; - ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - res = vmaAllocateMemoryForBuffer(allocator, raw, &ci3, &a, &info); - } - } - - vk::Check(res); - vk::Check(vmaBindBufferMemory2(allocator, a, 0, raw, nullptr)); - return MemoryCommit(allocator, a, info); - } - - - } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index 581f2e66d2..92a2c82a07 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -15,6 +16,8 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" #include "video_core/vulkan_common/vma.h" +struct AHardwareBuffer; + namespace Vulkan { class Device; @@ -39,49 +42,64 @@ namespace Vulkan { } } -/// Ownership handle of a memory commitment (real VMA allocation). - class MemoryCommit { + class HostMemoryImport { public: - MemoryCommit() noexcept = default; + explicit HostMemoryImport(const Device &device_, void *base, size_t size, + std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base); - MemoryCommit(VmaAllocator allocator, VmaAllocation allocation, - const VmaAllocationInfo &info) noexcept; + ~HostMemoryImport(); - ~MemoryCommit(); + HostMemoryImport(const HostMemoryImport &) = delete; - MemoryCommit(const MemoryCommit &) = delete; + HostMemoryImport &operator=(const HostMemoryImport &) = delete; - MemoryCommit &operator=(const MemoryCommit &) = delete; + [[nodiscard]] bool IsValid() const noexcept { + return !windows.empty(); + } - MemoryCommit(MemoryCommit &&) noexcept; + [[nodiscard]] size_t GetSize() const noexcept { + return imported_size; + } - MemoryCommit &operator=(MemoryCommit &&) noexcept; + [[nodiscard]] size_t GetBaseOffset() const noexcept { + return base_offset; + } - [[nodiscard]] std::span Map(); + [[nodiscard]] bool NeedsForeignOwnershipTransfer() const noexcept { + return foreign_ownership; + } - [[nodiscard]] std::span Map() const; + [[nodiscard]] VkDeviceSize GetWindowSize() const noexcept { + return window_size; + } - void Unmap(); + [[nodiscard]] VkBuffer GetWindowBuffer(size_t index) const noexcept { + return windows[index].buffer; + } - explicit operator bool() const noexcept { return allocation != nullptr; } - - VkDeviceMemory Memory() const noexcept { return memory; } - - VkDeviceSize Offset() const noexcept { return offset; } - - VkDeviceSize Size() const noexcept { return size; } - - VmaAllocation Allocation() const noexcept { return allocation; } + [[nodiscard]] size_t GetWindowCount() const noexcept { + return windows.size(); + } private: - void Release(); + struct Window { + vk::DeviceMemory memory; + VkBuffer buffer{}; + }; - VmaAllocator allocator{}; ///< VMA allocator - VmaAllocation allocation{}; ///< VMA allocation handle - VkDeviceMemory memory{}; ///< Underlying VkDeviceMemory chosen by VMA - VkDeviceSize offset{}; ///< Offset of this allocation inside VkDeviceMemory - VkDeviceSize size{}; ///< Size of the allocation - void *mapped_ptr{}; ///< Optional persistent mapped pointer + bool ImportHostPointer(void *base, size_t size); + + bool ImportHardwareBuffers(std::span hardware_buffers, + size_t hardware_buffer_window, size_t hardware_buffer_base, + size_t size); + + const Device &device; + std::vector windows; + VkDeviceSize window_size{}; + size_t imported_size{}; + size_t base_offset{}; + bool foreign_ownership{}; }; /// Memory allocator container. @@ -107,36 +125,21 @@ namespace Vulkan { vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage) const; - /** - * Commits a memory with the specified requirements. - * - * @param requirements Requirements returned from a Vulkan call. - * @param usage Indicates how the memory will be used. - * - * @returns A memory commit. - */ - MemoryCommit Commit(const VkMemoryRequirements &requirements, MemoryUsage usage); + using ReclaimCallback = std::function; - /// Commits memory required by the buffer and binds it (for buffers created outside VMA). - MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage); + void SetReclaimCallback(ReclaimCallback callback); private: - static bool IsAutoUsage(VmaMemoryUsage u) noexcept { - switch (u) { - case VMA_MEMORY_USAGE_AUTO: - case VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE: - case VMA_MEMORY_USAGE_AUTO_PREFER_HOST: - return true; - default: - return false; - } - } + bool ReclaimAtLeast(u64 hint_bytes) const; + + static constexpr u64 IMAGE_RECLAIM_HINT = 64ULL * 1024 * 1024; const Device &device; ///< Device handle. VmaAllocator allocator; ///< VMA allocator. const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. - VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; + ReclaimCallback reclaim_callback; + mutable bool in_reclaim{false}; }; } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 4943705859..024aca07d4 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -5,11 +5,16 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include #include +#include #include +#include #include #include +#include "common/assert.h" #include "common/common_types.h" #include "common/logging.h" #include "video_core/vulkan_common/vk_enum_string_helper.h" @@ -20,6 +25,60 @@ namespace Vulkan::vk { namespace { +std::thread::id allocator_owner_thread; + +template +struct PendingRelease { + VmaAllocator allocator; + HandleType handle; + VmaAllocation allocation; + u64 timeline; +}; + +std::mutex deletion_mutex; +std::atomic deletion_timeline{1}; +std::vector> pending_images; +std::vector> pending_buffers; + +template +void PushPendingRelease(std::vector>& pending, VmaAllocator allocator, + HandleType handle, VmaAllocation allocation) noexcept { + std::scoped_lock lock{deletion_mutex}; + pending.push_back(PendingRelease{ + .allocator = allocator, + .handle = handle, + .allocation = allocation, + .timeline = deletion_timeline.load(std::memory_order_acquire), + }); +} + +template +void ExtractReleased(std::vector>& pending, + std::vector>& released, u64 completed_value) { + const auto split = std::partition(pending.begin(), pending.end(), + [completed_value](const PendingRelease& entry) { + return entry.timeline > completed_value; + }); + released.assign(split, pending.end()); + pending.erase(split, pending.end()); +} + +void DrainDeletionQueue(u64 completed_value) noexcept { + std::vector> images; + std::vector> buffers; + { + std::scoped_lock lock{deletion_mutex}; + ExtractReleased(pending_images, images, completed_value); + ExtractReleased(pending_buffers, buffers, completed_value); + } + for (const auto& entry : images) { + vmaDestroyImage(entry.allocator, entry.handle, entry.allocation); + } + for (const auto& entry : buffers) { + vmaDestroyBuffer(entry.allocator, entry.handle, entry.allocation); + } +} + template void SortPhysicalDevices(std::vector& devices, const InstanceDispatch& dld, Func&& func) { @@ -92,6 +151,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdBeginConditionalRenderingEXT); X(vkCmdBeginQuery); X(vkCmdBeginRenderPass); + X(vkCmdBeginRendering); X(vkCmdBeginTransformFeedbackEXT); X(vkCmdBeginDebugUtilsLabelEXT); X(vkCmdBindDescriptorSets); @@ -119,6 +179,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdEndConditionalRenderingEXT); X(vkCmdEndQuery); X(vkCmdEndRenderPass); + X(vkCmdEndRendering); X(vkCmdEndTransformFeedbackEXT); X(vkCmdEndDebugUtilsLabelEXT); X(vkCmdFillBuffer); @@ -216,12 +277,16 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkGetBufferMemoryRequirements2); X(vkGetDeviceQueue); X(vkGetEventStatus); + X(vkGetMemoryHostPointerPropertiesEXT); X(vkGetFenceStatus); X(vkGetImageMemoryRequirements); X(vkGetPipelineCacheData); X(vkGetMemoryFdKHR); #ifdef _WIN32 X(vkGetMemoryWin32HandleKHR); +#endif +#ifdef __ANDROID__ + X(vkGetAndroidHardwareBufferPropertiesANDROID); #endif X(vkGetQueryPoolResults); X(vkGetPipelineExecutablePropertiesKHR); @@ -230,6 +295,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkMapMemory); X(vkQueueSubmit); X(vkQueueSubmit2); + X(vkResetCommandPool); X(vkResetFences); X(vkResetQueryPool); X(vkSetDebugUtilsObjectNameEXT); @@ -237,6 +303,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkUnmapMemory); X(vkUpdateDescriptorSetWithTemplate); X(vkUpdateDescriptorSets); + X(vkGetBufferDeviceAddress); + X(vkGetDescriptorSetLayoutSizeEXT); + X(vkGetDescriptorSetLayoutBindingOffsetEXT); + X(vkGetDescriptorEXT); + X(vkCmdBindDescriptorBuffersEXT); + X(vkCmdSetDescriptorBufferOffsetsEXT); X(vkWaitForFences); X(vkWaitSemaphores); @@ -257,6 +329,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { Proc(dld.vkCmdDrawIndexedIndirectCount, dld, "vkCmdDrawIndexedIndirectCountKHR", device); } + // Support for dynamic rendering is optional until Vulkan 1.3 + if (!dld.vkCmdBeginRendering) { + Proc(dld.vkCmdBeginRendering, dld, "vkCmdBeginRenderingKHR", device); + Proc(dld.vkCmdEndRendering, dld, "vkCmdEndRenderingKHR", device); + } + // Synchronization2 is core in Vulkan 1.3, otherwise requires VK_KHR_synchronization2 if (!dld.vkCmdPipelineBarrier2) { Proc(dld.vkCmdPipelineBarrier2, dld, "vkCmdPipelineBarrier2KHR", device); @@ -502,13 +580,35 @@ DebugReportCallback Instance::CreateDebugReportCallback( return DebugReportCallback(object, handle, *dld); } +void SetAllocatorOwnerThread() { + allocator_owner_thread = std::this_thread::get_id(); +} + +bool OnAllocatorOwnerThread() noexcept { + return allocator_owner_thread == std::thread::id{} || + allocator_owner_thread == std::this_thread::get_id(); +} + +void SetDeletionTimeline(u64 value) noexcept { + deletion_timeline.store(value, std::memory_order_release); +} + +void TickDeletionQueue(u64 completed_value) noexcept { + DEBUG_ASSERT(OnAllocatorOwnerThread()); + DrainDeletionQueue(completed_value); +} + +void FlushDeletionQueue() noexcept { + DrainDeletionQueue((std::numeric_limits::max)()); +} + void Image::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE, name); } void Image::Release() const noexcept { if (handle) { - vmaDestroyImage(allocator, handle, allocation); + PushPendingRelease(pending_images, allocator, handle, allocation); } } @@ -530,7 +630,7 @@ void Buffer::SetObjectNameEXT(const char* name) const { void Buffer::Release() const noexcept { if (handle) { - vmaDestroyBuffer(allocator, handle, allocation); + PushPendingRelease(pending_buffers, allocator, handle, allocation); } } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index ef975e2bda..77a4bda15c 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -131,6 +131,16 @@ private: VkResult result; }; +void SetAllocatorOwnerThread(); + +[[nodiscard]] bool OnAllocatorOwnerThread() noexcept; + +void SetDeletionTimeline(u64 value) noexcept; + +void TickDeletionQueue(u64 completed_value) noexcept; + +void FlushDeletionQueue() noexcept; + /// Throws a Vulkan exception if result is not success. inline void Check(VkResult result) { if (result != VK_SUCCESS) { @@ -208,6 +218,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdBeginDebugUtilsLabelEXT vkCmdBeginDebugUtilsLabelEXT{}; PFN_vkCmdBeginQuery vkCmdBeginQuery{}; PFN_vkCmdBeginRenderPass vkCmdBeginRenderPass{}; + PFN_vkCmdBeginRendering vkCmdBeginRendering{}; PFN_vkCmdBeginTransformFeedbackEXT vkCmdBeginTransformFeedbackEXT{}; PFN_vkCmdBindDescriptorSets vkCmdBindDescriptorSets{}; PFN_vkCmdBindIndexBuffer vkCmdBindIndexBuffer{}; @@ -236,6 +247,7 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdEndDebugUtilsLabelEXT vkCmdEndDebugUtilsLabelEXT{}; PFN_vkCmdEndQuery vkCmdEndQuery{}; PFN_vkCmdEndRenderPass vkCmdEndRenderPass{}; + PFN_vkCmdEndRendering vkCmdEndRendering{}; PFN_vkCmdEndTransformFeedbackEXT vkCmdEndTransformFeedbackEXT{}; PFN_vkCmdFillBuffer vkCmdFillBuffer{}; PFN_vkCmdPipelineBarrier vkCmdPipelineBarrier{}; @@ -332,12 +344,16 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{}; PFN_vkGetDeviceQueue vkGetDeviceQueue{}; PFN_vkGetEventStatus vkGetEventStatus{}; + PFN_vkGetMemoryHostPointerPropertiesEXT vkGetMemoryHostPointerPropertiesEXT{}; PFN_vkGetFenceStatus vkGetFenceStatus{}; PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{}; PFN_vkGetPipelineCacheData vkGetPipelineCacheData{}; PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{}; #ifdef _WIN32 PFN_vkGetMemoryWin32HandleKHR vkGetMemoryWin32HandleKHR{}; +#endif +#ifdef __ANDROID__ + PFN_vkGetAndroidHardwareBufferPropertiesANDROID vkGetAndroidHardwareBufferPropertiesANDROID{}; #endif PFN_vkGetPipelineExecutablePropertiesKHR vkGetPipelineExecutablePropertiesKHR{}; PFN_vkGetPipelineExecutableStatisticsKHR vkGetPipelineExecutableStatisticsKHR{}; @@ -346,12 +362,19 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkMapMemory vkMapMemory{}; PFN_vkQueueSubmit vkQueueSubmit{}; PFN_vkQueueSubmit2 vkQueueSubmit2{}; + PFN_vkResetCommandPool vkResetCommandPool{}; PFN_vkResetFences vkResetFences{}; PFN_vkResetQueryPool vkResetQueryPool{}; PFN_vkSetDebugUtilsObjectNameEXT vkSetDebugUtilsObjectNameEXT{}; PFN_vkSetDebugUtilsObjectTagEXT vkSetDebugUtilsObjectTagEXT{}; PFN_vkUnmapMemory vkUnmapMemory{}; PFN_vkUpdateDescriptorSetWithTemplate vkUpdateDescriptorSetWithTemplate{}; + PFN_vkGetBufferDeviceAddress vkGetBufferDeviceAddress{}; + PFN_vkGetDescriptorSetLayoutSizeEXT vkGetDescriptorSetLayoutSizeEXT{}; + PFN_vkGetDescriptorSetLayoutBindingOffsetEXT vkGetDescriptorSetLayoutBindingOffsetEXT{}; + PFN_vkGetDescriptorEXT vkGetDescriptorEXT{}; + PFN_vkCmdBindDescriptorBuffersEXT vkCmdBindDescriptorBuffersEXT{}; + PFN_vkCmdSetDescriptorBufferOffsetsEXT vkCmdSetDescriptorBufferOffsetsEXT{}; PFN_vkUpdateDescriptorSets vkUpdateDescriptorSets{}; PFN_vkWaitForFences vkWaitForFences{}; PFN_vkWaitSemaphores vkWaitSemaphores{}; @@ -925,6 +948,10 @@ public: CommandBuffers Allocate(std::size_t num_buffers, VkCommandBufferLevel level = VK_COMMAND_BUFFER_LEVEL_PRIMARY) const; + void Reset(VkCommandPoolResetFlags flags = 0) const { + Check(dld->vkResetCommandPool(owner, handle, flags)); + } + /// Set object name. void SetObjectNameEXT(const char* name) const; }; @@ -1071,6 +1098,34 @@ public: VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer, void* pnext = nullptr) const noexcept; + VkResult GetMemoryHostPointerPropertiesEXT( + VkExternalMemoryHandleTypeFlagBits handle_type, const void* host_pointer, + VkMemoryHostPointerPropertiesEXT* out_properties) const noexcept { + return dld->vkGetMemoryHostPointerPropertiesEXT(handle, handle_type, host_pointer, + out_properties); + } + +#ifdef __ANDROID__ + VkResult GetAndroidHardwareBufferPropertiesANDROID( + const struct AHardwareBuffer* buffer, + VkAndroidHardwareBufferPropertiesANDROID* out_properties) const noexcept { + return dld->vkGetAndroidHardwareBufferPropertiesANDROID(handle, buffer, out_properties); + } +#endif + + VkResult CreateBufferRaw(const VkBufferCreateInfo& ci, VkBuffer* out_buffer) const noexcept { + return dld->vkCreateBuffer(handle, &ci, nullptr, out_buffer); + } + + void DestroyBufferRaw(VkBuffer buffer) const noexcept { + dld->vkDestroyBuffer(handle, buffer, nullptr); + } + + VkResult BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory, + VkDeviceSize offset) const noexcept { + return dld->vkBindBufferMemory(handle, buffer, memory, offset); + } + VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept; std::vector GetPipelineExecutablePropertiesKHR( @@ -1087,6 +1142,34 @@ public: dld->vkUpdateDescriptorSetWithTemplate(handle, set, update_template, data); } + [[nodiscard]] VkDeviceAddress GetBufferDeviceAddress(VkBuffer buffer) const noexcept { + const VkBufferDeviceAddressInfo info{ + .sType = VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, + .pNext = nullptr, + .buffer = buffer, + }; + return dld->vkGetBufferDeviceAddress(handle, &info); + } + + [[nodiscard]] VkDeviceSize GetDescriptorSetLayoutSizeEXT( + VkDescriptorSetLayout layout) const noexcept { + VkDeviceSize size{}; + dld->vkGetDescriptorSetLayoutSizeEXT(handle, layout, &size); + return size; + } + + [[nodiscard]] VkDeviceSize GetDescriptorSetLayoutBindingOffsetEXT( + VkDescriptorSetLayout layout, u32 binding) const noexcept { + VkDeviceSize offset{}; + dld->vkGetDescriptorSetLayoutBindingOffsetEXT(handle, layout, binding, &offset); + return offset; + } + + void GetDescriptorEXT(const VkDescriptorGetInfoEXT& info, size_t size, + void* descriptor) const noexcept { + dld->vkGetDescriptorEXT(handle, &info, size, descriptor); + } + VkResult AcquireNextImageKHR(VkSwapchainKHR swapchain, u64 timeout, VkSemaphore semaphore, VkFence fence, u32* image_index) const noexcept { return dld->vkAcquireNextImageKHR(handle, swapchain, timeout, semaphore, fence, @@ -1186,6 +1269,14 @@ public: dld->vkCmdEndRenderPass(handle); } + void BeginRendering(const VkRenderingInfo& rendering_info) const noexcept { + dld->vkCmdBeginRendering(handle, &rendering_info); + } + + void EndRendering() const noexcept { + dld->vkCmdEndRendering(handle); + } + void BeginQuery(VkQueryPool query_pool, u32 query, VkQueryControlFlags flags) const noexcept { dld->vkCmdBeginQuery(handle, query_pool, query, flags); } @@ -1400,6 +1491,40 @@ public: PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, {}, {}, image_barrier); } + void BindDescriptorBuffersEXT(Span bindings) const noexcept { + dld->vkCmdBindDescriptorBuffersEXT(handle, bindings.size(), bindings.data()); + } + + void SetDescriptorBufferOffsetsEXT(VkPipelineBindPoint bind_point, VkPipelineLayout layout, + u32 first_set, Span buffer_indices, + Span offsets) const noexcept { + dld->vkCmdSetDescriptorBufferOffsetsEXT(handle, bind_point, layout, first_set, + buffer_indices.size(), buffer_indices.data(), + offsets.data()); + } + + [[nodiscard]] bool HasPipelineBarrier2() const noexcept { + return dld->vkCmdPipelineBarrier2 != nullptr; + } + + void PipelineBarrier2(VkDependencyFlags dependency_flags, + Span memory_barriers, + Span buffer_barriers, + Span image_barriers) const noexcept { + const VkDependencyInfo dependency_info{ + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .pNext = nullptr, + .dependencyFlags = dependency_flags, + .memoryBarrierCount = memory_barriers.size(), + .pMemoryBarriers = memory_barriers.data(), + .bufferMemoryBarrierCount = buffer_barriers.size(), + .pBufferMemoryBarriers = buffer_barriers.data(), + .imageMemoryBarrierCount = image_barriers.size(), + .pImageMemoryBarriers = image_barriers.data(), + }; + dld->vkCmdPipelineBarrier2(handle, &dependency_info); + } + void CopyBufferToImage(VkBuffer src_buffer, VkImage dst_image, VkImageLayout dst_image_layout, Span regions) const noexcept { dld->vkCmdCopyBufferToImage(handle, src_buffer, dst_image, dst_image_layout, regions.size(),