diff --git a/CMakeLists.txt b/CMakeLists.txt index f647204a4b..91a77b717b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,8 +82,8 @@ cmake_dependent_option(YUZU_USE_BUNDLED_QT "Download bundled Qt binaries" "${MSV option(ENABLE_DEBUG_TOOLS "Enable debugging tools (maxwell disassembler, SPIRV translator, etc)" OFF) option(ENABLE_WERROR "Enable -Werror diagnostics" ON) -# Lossless Scaling frame generation. Only Android. -cmake_dependent_option(ENABLE_LSFG "Enable Lossless Scaling frame generation" ON "ANDROID" OFF) +# Lossless Scaling frame generation. +option(ENABLE_LSFG "Enable Lossless Scaling frame generation" ON) option(ENABLE_RESHADE "Enable ReShade FX post-processing effects" ON) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/adapters/GameAdapter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/adapters/GameAdapter.kt index 9ea2a9ee17..382f56f464 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/adapters/GameAdapter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/adapters/GameAdapter.kt @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later package org.yuzu.yuzu_emu.adapters diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt index ca3309245e..f6ad6b0145 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/BooleanSetting.kt @@ -20,18 +20,21 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { EMULATE_BGR565("emulate_bgr565"), RESCALE_HACK("rescale_hack"), CPUOPT_UNSAFE_HOST_MMU("cpuopt_unsafe_host_mmu"), + RELOCATE_36BIT_ADDRESS_SPACE("relocate_36bit_address_space"), USE_DOCKED_MODE("use_docked_mode"), USE_AUTO_STUB("use_auto_stub"), RENDERER_USE_DISK_SHADER_CACHE("use_disk_shader_cache"), RENDERER_FORCE_MAX_CLOCK("force_max_clock"), RENDERER_ASYNCHRONOUS_GPU_EMULATION("use_asynchronous_gpu_emulation"), - RENDERER_ASYNC_PRESENTATION("async_presentation"), RENDERER_ASYNCHRONOUS_SHADERS("use_asynchronous_shaders"), RENDERER_REACTIVE_FLUSHING("use_reactive_flushing"), + RENDERER_BARRIER_FEEDBACK_LOOPS("barrier_feedback_loops"), ENABLE_BUFFER_HISTORY("enable_buffer_history"), USE_OPTIMIZED_VERTEX_BUFFERS("use_optimized_vertex_buffers"), ENABLE_GPU_BUFFER_READBACK("enable_gpu_buffer_readback"), + USE_UNIFIED_MEMORY("use_unified_memory"), SYNC_MEMORY_OPERATIONS("sync_memory_operations"), + STALL_ON_GPU_FENCE_WAIT("stall_on_gpu_fence_wait"), BUFFER_REORDER_DISABLE("disable_buffer_reorder"), RENDERER_DEBUG("debug"), RENDERER_PATCH_OLD_QCOM_DRIVERS("patch_old_qcom_drivers"), @@ -39,7 +42,6 @@ enum class BooleanSetting(override val key: String) : AbstractBooleanSetting { RENDERER_SAMPLE_SHADING("sample_shading"), RENDERER_FRAME_GEN("frame_gen"), RENDERER_FRAME_GEN_FLOW_SCALE_AUTO("frame_gen_flow_scale_auto"), - GPU_UNSWIZZLE_ENABLED("gpu_unswizzle_enabled"), PICTURE_IN_PICTURE("picture_in_picture"), USE_CUSTOM_RTC("custom_rtc_enabled"), BLACK_BACKGROUNDS("black_backgrounds"), diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt index ea31692fbf..e8087cef5b 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/IntSetting.kt @@ -32,7 +32,6 @@ enum class IntSetting(override val key: String) : AbstractIntSetting { RENDERER_DYNA_STATE("dyna_state"), DMA_ACCURACY("dma_accuracy"), GPU_FENCE_BEHAVIOR("gpu_fence_behavior"), - FRAME_PACING_MODE("frame_pacing_mode"), AUDIO_OUTPUT_ENGINE("output_engine"), MAX_ANISOTROPY("max_anisotropy"), THEME("theme"), @@ -51,9 +50,6 @@ enum class IntSetting(override val key: String) : AbstractIntSetting { FAST_CPU_TIME("fast_cpu_time"), CPU_TICKS("cpu_ticks"), FAST_GPU_TIME("fast_gpu_time"), - GPU_UNSWIZZLE_TEXTURE_SIZE("gpu_unswizzle_texture_size"), - GPU_UNSWIZZLE_STREAM_SIZE("gpu_unswizzle_stream_size"), - GPU_UNSWIZZLE_CHUNK_SIZE("gpu_unswizzle_chunk_size"), BAT_TEMPERATURE_UNIT("bat_temperature_unit"), CABINET_APPLET("cabinet_applet_mode"), CONTROLLER_APPLET("controller_applet_mode"), diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/GpuUnswizzleSetting.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/GpuUnswizzleSetting.kt deleted file mode 100644 index a9a40f6903..0000000000 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/GpuUnswizzleSetting.kt +++ /dev/null @@ -1,89 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -package org.yuzu.yuzu_emu.features.settings.model.view - -import androidx.annotation.ArrayRes -import androidx.annotation.StringRes -import org.yuzu.yuzu_emu.features.settings.model.AbstractSetting -import org.yuzu.yuzu_emu.features.settings.model.BooleanSetting -import org.yuzu.yuzu_emu.features.settings.model.IntSetting - -class GpuUnswizzleSetting( - @StringRes titleId: Int = 0, - titleString: String = "", - @StringRes descriptionId: Int = 0, - descriptionString: String = "", - @ArrayRes val textureSizeChoicesId: Int, - @ArrayRes val textureSizeValuesId: Int, - @ArrayRes val streamSizeChoicesId: Int, - @ArrayRes val streamSizeValuesId: Int, - @ArrayRes val chunkSizeChoicesId: Int, - @ArrayRes val chunkSizeValuesId: Int -) : SettingsItem( - object : AbstractSetting { - override val key: String = SettingsItem.GPU_UNSWIZZLE_COMBINED - override val defaultValue: Any = false - override val isSaveable = true - override val isRuntimeModifiable = true - override val isSwitchable = true - override val pairedSettingKey: String = "" - override var global: Boolean - get() { - return BooleanSetting.GPU_UNSWIZZLE_ENABLED.global && - IntSetting.GPU_UNSWIZZLE_TEXTURE_SIZE.global && - IntSetting.GPU_UNSWIZZLE_STREAM_SIZE.global && - IntSetting.GPU_UNSWIZZLE_CHUNK_SIZE.global - } - set(value) { - BooleanSetting.GPU_UNSWIZZLE_ENABLED.global = value - IntSetting.GPU_UNSWIZZLE_TEXTURE_SIZE.global = value - IntSetting.GPU_UNSWIZZLE_STREAM_SIZE.global = value - IntSetting.GPU_UNSWIZZLE_CHUNK_SIZE.global = value - } - override fun getValueAsString(needsGlobal: Boolean): String = "combined" - override fun reset() { - BooleanSetting.GPU_UNSWIZZLE_ENABLED.reset() - IntSetting.GPU_UNSWIZZLE_TEXTURE_SIZE.reset() - IntSetting.GPU_UNSWIZZLE_STREAM_SIZE.reset() - IntSetting.GPU_UNSWIZZLE_CHUNK_SIZE.reset() - } - }, - titleId, - titleString, - descriptionId, - descriptionString -) { - override val type = SettingsItem.TYPE_GPU_UNSWIZZLE - - // Check if GPU unswizzle is enabled via the dedicated boolean setting - fun isEnabled(needsGlobal: Boolean = false): Boolean = - BooleanSetting.GPU_UNSWIZZLE_ENABLED.getBoolean(needsGlobal) - - fun setEnabled(value: Boolean) = - BooleanSetting.GPU_UNSWIZZLE_ENABLED.setBoolean(value) - - fun enable() = setEnabled(true) - - fun disable() = setEnabled(false) - - fun getTextureSize(needsGlobal: Boolean = false): Int = - IntSetting.GPU_UNSWIZZLE_TEXTURE_SIZE.getInt(needsGlobal) - - fun setTextureSize(value: Int) = - IntSetting.GPU_UNSWIZZLE_TEXTURE_SIZE.setInt(value) - - fun getStreamSize(needsGlobal: Boolean = false): Int = - IntSetting.GPU_UNSWIZZLE_STREAM_SIZE.getInt(needsGlobal) - - fun setStreamSize(value: Int) = - IntSetting.GPU_UNSWIZZLE_STREAM_SIZE.setInt(value) - - fun getChunkSize(needsGlobal: Boolean = false): Int = - IntSetting.GPU_UNSWIZZLE_CHUNK_SIZE.getInt(needsGlobal) - - fun setChunkSize(value: Int) = - IntSetting.GPU_UNSWIZZLE_CHUNK_SIZE.setInt(value) - - fun reset() = setting.reset() -} diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt index 3f38831a17..d4be5d0140 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/model/view/SettingsItem.kt @@ -141,14 +141,12 @@ abstract class SettingsItem( const val TYPE_SPINBOX = 12 const val TYPE_LAUNCHABLE = 13 const val TYPE_PATH = 14 - const val TYPE_GPU_UNSWIZZLE = 15 const val TYPE_FX_TOOLBAR = 16 const val TYPE_FX_PRESET = 17 const val TYPE_FX_SHADER = 18 const val TYPE_FX_BUTTON = 19 const val FASTMEM_COMBINED = "fastmem_combined" - const val GPU_UNSWIZZLE_COMBINED = "gpu_unswizzle_combined" val emptySetting = object : AbstractSetting { override val key: String = "" @@ -753,13 +751,6 @@ abstract class SettingsItem( descriptionId = R.string.renderer_asynchronous_gpu_emulation_description ) ) - put( - SwitchSetting( - BooleanSetting.RENDERER_ASYNC_PRESENTATION, - titleId = R.string.renderer_async_presentation, - descriptionId = R.string.renderer_async_presentation_description - ) - ) put( SingleChoiceSetting( IntSetting.DMA_ACCURACY, @@ -794,45 +785,6 @@ abstract class SettingsItem( valuesId = R.array.gpuValues ) ) - put( - SingleChoiceSetting( - IntSetting.GPU_UNSWIZZLE_TEXTURE_SIZE, - titleId = R.string.gpu_unswizzle_texture_size, - descriptionId = R.string.gpu_unswizzle_texture_size_description, - choicesId = R.array.gpuTextureSizeSwizzleEntries, - valuesId = R.array.gpuTextureSizeSwizzleValues - ) - ) - put( - SingleChoiceSetting( - IntSetting.GPU_UNSWIZZLE_STREAM_SIZE, - titleId = R.string.gpu_unswizzle_stream_size, - descriptionId = R.string.gpu_unswizzle_stream_size_description, - choicesId = R.array.gpuSwizzleEntries, - valuesId = R.array.gpuSwizzleValues - ) - ) - put( - SingleChoiceSetting( - IntSetting.GPU_UNSWIZZLE_CHUNK_SIZE, - titleId = R.string.gpu_unswizzle_chunk_size, - descriptionId = R.string.gpu_unswizzle_chunk_size_description, - choicesId = R.array.gpuSwizzleChunkEntries, - valuesId = R.array.gpuSwizzleChunkValues - ) - ) - put( - GpuUnswizzleSetting( - titleId = R.string.gpu_unswizzle_settings, - descriptionId = R.string.gpu_unswizzle_settings_description, - textureSizeChoicesId = R.array.gpuTextureSizeSwizzleEntries, - textureSizeValuesId = R.array.gpuTextureSizeSwizzleValues, - streamSizeChoicesId = R.array.gpuSwizzleEntries, - streamSizeValuesId = R.array.gpuSwizzleValues, - chunkSizeChoicesId = R.array.gpuSwizzleChunkEntries, - chunkSizeValuesId = R.array.gpuSwizzleChunkValues - ) - ) put( SingleChoiceSetting( IntSetting.FAST_CPU_TIME, @@ -894,6 +846,13 @@ abstract class SettingsItem( descriptionId = R.string.cpuopt_unsafe_host_mmu_description ) ) + put( + SwitchSetting( + BooleanSetting.RELOCATE_36BIT_ADDRESS_SPACE, + titleId = R.string.relocate_36bit_address_space, + descriptionId = R.string.relocate_36bit_address_space_description + ) + ) put( SwitchSetting( BooleanSetting.RENDERER_REACTIVE_FLUSHING, @@ -901,6 +860,13 @@ abstract class SettingsItem( descriptionId = R.string.renderer_reactive_flushing_description ) ) + put( + SwitchSetting( + BooleanSetting.RENDERER_BARRIER_FEEDBACK_LOOPS, + titleId = R.string.renderer_barrier_feedback_loops, + descriptionId = R.string.renderer_barrier_feedback_loops_description + ) + ) put( SwitchSetting( BooleanSetting.ENABLE_BUFFER_HISTORY, @@ -915,6 +881,13 @@ abstract class SettingsItem( descriptionId = R.string.enable_gpu_buffer_readback_description ) ) + put( + SwitchSetting( + BooleanSetting.USE_UNIFIED_MEMORY, + titleId = R.string.use_unified_memory, + descriptionId = R.string.use_unified_memory_description + ) + ) put( SwitchSetting( BooleanSetting.USE_OPTIMIZED_VERTEX_BUFFERS, @@ -929,6 +902,13 @@ abstract class SettingsItem( descriptionId = R.string.sync_memory_operations_description ) ) + put( + SwitchSetting( + BooleanSetting.STALL_ON_GPU_FENCE_WAIT, + titleId = R.string.stall_on_gpu_fence_wait, + descriptionId = R.string.stall_on_gpu_fence_wait_description + ) + ) put( SwitchSetting( BooleanSetting.BUFFER_REORDER_DISABLE, diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/GpuUnswizzleDialogFragment.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/GpuUnswizzleDialogFragment.kt deleted file mode 100644 index e14bc7639e..0000000000 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/GpuUnswizzleDialogFragment.kt +++ /dev/null @@ -1,206 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -package org.yuzu.yuzu_emu.features.settings.ui - -import android.app.Dialog -import android.content.DialogInterface -import android.os.Bundle -import android.view.LayoutInflater -import android.widget.ArrayAdapter -import androidx.fragment.app.DialogFragment -import androidx.fragment.app.activityViewModels -import com.google.android.material.dialog.MaterialAlertDialogBuilder -import org.yuzu.yuzu_emu.R -import org.yuzu.yuzu_emu.databinding.DialogGpuUnswizzleBinding -import org.yuzu.yuzu_emu.features.settings.model.view.GpuUnswizzleSetting - -class GpuUnswizzleDialogFragment : DialogFragment() { - private var position = 0 - private val settingsViewModel: SettingsViewModel by activityViewModels() - private lateinit var binding: DialogGpuUnswizzleBinding - - override fun onCreate(savedInstanceState: Bundle?) { - super.onCreate(savedInstanceState) - position = requireArguments().getInt(POSITION) - - if (settingsViewModel.clickedItem == null) dismiss() - } - - override fun onCreateDialog(savedInstanceState: Bundle?): Dialog { - binding = DialogGpuUnswizzleBinding.inflate(LayoutInflater.from(requireContext())) - val item = settingsViewModel.clickedItem as GpuUnswizzleSetting - - // Setup texture size dropdown - val textureSizeEntries = resources.getStringArray(item.textureSizeChoicesId) - val textureSizeValues = resources.getIntArray(item.textureSizeValuesId) - val textureSizeAdapter = ArrayAdapter( - requireContext(), - android.R.layout.simple_dropdown_item_1line, - textureSizeEntries.toMutableList() - ) - binding.dropdownTextureSize.setAdapter(textureSizeAdapter) - - // Setup stream size dropdown - val streamSizeEntries = resources.getStringArray(item.streamSizeChoicesId) - val streamSizeValues = resources.getIntArray(item.streamSizeValuesId) - val streamSizeAdapter = ArrayAdapter( - requireContext(), - android.R.layout.simple_dropdown_item_1line, - streamSizeEntries.toMutableList() - ) - binding.dropdownStreamSize.setAdapter(streamSizeAdapter) - - // Setup chunk size dropdown - val chunkSizeEntries = resources.getStringArray(item.chunkSizeChoicesId) - val chunkSizeValues = resources.getIntArray(item.chunkSizeValuesId) - val chunkSizeAdapter = ArrayAdapter( - requireContext(), - android.R.layout.simple_dropdown_item_1line, - chunkSizeEntries.toMutableList() - ) - binding.dropdownChunkSize.setAdapter(chunkSizeAdapter) - - // Load current values - val isEnabled = item.isEnabled() - binding.switchEnable.isChecked = isEnabled - - if (isEnabled) { - val textureSizeIndex = textureSizeValues.indexOf(item.getTextureSize()) - if (textureSizeIndex >= 0) { - binding.dropdownTextureSize.setText(textureSizeEntries[textureSizeIndex], false) - } - - val streamSizeIndex = streamSizeValues.indexOf(item.getStreamSize()) - if (streamSizeIndex >= 0) { - binding.dropdownStreamSize.setText(streamSizeEntries[streamSizeIndex], false) - } - - val chunkSizeIndex = chunkSizeValues.indexOf(item.getChunkSize()) - if (chunkSizeIndex >= 0) { - binding.dropdownChunkSize.setText(chunkSizeEntries[chunkSizeIndex], false) - } - } else { - // Set default/recommended values when disabling - binding.dropdownTextureSize.setText(textureSizeEntries[3], false) - binding.dropdownStreamSize.setText(streamSizeEntries[3], false) - binding.dropdownChunkSize.setText(chunkSizeEntries[3], false) - } - - // Clear adapter filters after setText to fix rotation bug - textureSizeAdapter.filter.filter(null) - streamSizeAdapter.filter.filter(null) - chunkSizeAdapter.filter.filter(null) - - // Enable/disable dropdowns based on switch state - updateDropdownsState(isEnabled) - binding.switchEnable.setOnCheckedChangeListener { _, checked -> - updateDropdownsState(checked) - } - - val dialog = MaterialAlertDialogBuilder(requireContext()) - .setTitle(item.title) - .setView(binding.root) - .create() - - // Setup button listeners - binding.btnDefault.setOnClickListener { - // Reset to defaults - item.reset() - // Refresh values with adapters reset - val textureSizeIndex = textureSizeValues.indexOf(item.getTextureSize()) - if (textureSizeIndex >= 0) { - binding.dropdownTextureSize.setText(textureSizeEntries[textureSizeIndex], false) - } - val streamSizeIndex = streamSizeValues.indexOf(item.getStreamSize()) - if (streamSizeIndex >= 0) { - binding.dropdownStreamSize.setText(streamSizeEntries[streamSizeIndex], false) - } - val chunkSizeIndex = chunkSizeValues.indexOf(item.getChunkSize()) - if (chunkSizeIndex >= 0) { - binding.dropdownChunkSize.setText(chunkSizeEntries[chunkSizeIndex], false) - } - // Clear filters - textureSizeAdapter.filter.filter(null) - streamSizeAdapter.filter.filter(null) - chunkSizeAdapter.filter.filter(null) - - settingsViewModel.setAdapterItemChanged(position) - settingsViewModel.setShouldReloadSettingsList(true) - } - - binding.btnCancel.setOnClickListener { - dialog.dismiss() - } - - binding.btnOk.setOnClickListener { - if (binding.switchEnable.isChecked) { - item.enable() - // Save the selected values - val selectedTextureIndex = textureSizeEntries.indexOf( - binding.dropdownTextureSize.text.toString() - ) - if (selectedTextureIndex >= 0) { - item.setTextureSize(textureSizeValues[selectedTextureIndex]) - } - - val selectedStreamIndex = streamSizeEntries.indexOf( - binding.dropdownStreamSize.text.toString() - ) - if (selectedStreamIndex >= 0) { - item.setStreamSize(streamSizeValues[selectedStreamIndex]) - } - - val selectedChunkIndex = chunkSizeEntries.indexOf( - binding.dropdownChunkSize.text.toString() - ) - if (selectedChunkIndex >= 0) { - item.setChunkSize(chunkSizeValues[selectedChunkIndex]) - } - } else { - // Disable GPU unswizzle - item.disable() - } - - settingsViewModel.setAdapterItemChanged(position) - settingsViewModel.setShouldReloadSettingsList(true) - dialog.dismiss() - } - - // Ensure filters are cleared after dialog is shown - binding.root.post { - textureSizeAdapter.filter.filter(null) - streamSizeAdapter.filter.filter(null) - chunkSizeAdapter.filter.filter(null) - } - - return dialog - } - - private fun updateDropdownsState(enabled: Boolean) { - binding.layoutTextureSize.isEnabled = enabled - binding.dropdownTextureSize.isEnabled = enabled - binding.layoutStreamSize.isEnabled = enabled - binding.dropdownStreamSize.isEnabled = enabled - binding.layoutChunkSize.isEnabled = enabled - binding.dropdownChunkSize.isEnabled = enabled - } - - companion object { - const val TAG = "GpuUnswizzleDialogFragment" - const val POSITION = "Position" - - fun newInstance( - settingsViewModel: SettingsViewModel, - item: GpuUnswizzleSetting, - position: Int - ): GpuUnswizzleDialogFragment { - val dialog = GpuUnswizzleDialogFragment() - val args = Bundle() - args.putInt(POSITION, position) - dialog.arguments = args - settingsViewModel.clickedItem = item - return dialog - } - } -} diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsAdapter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsAdapter.kt index 8d47db94e0..c84d0160f5 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsAdapter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsAdapter.kt @@ -106,10 +106,6 @@ class SettingsAdapter( PathViewHolder(ListItemSettingBinding.inflate(inflater), this) } - SettingsItem.TYPE_GPU_UNSWIZZLE -> { - GpuUnswizzleViewHolder(ListItemSettingBinding.inflate(inflater), this) - } - SettingsItem.TYPE_FX_TOOLBAR -> { FxToolbarViewHolder( ListItemSettingFxToolbarBinding.inflate(inflater, parent, false), @@ -511,14 +507,6 @@ class SettingsAdapter( settingsViewModel.setShouldShowPathResetDialog(true) } - fun onGpuUnswizzleClick(item: GpuUnswizzleSetting, position: Int) { - GpuUnswizzleDialogFragment.newInstance( - settingsViewModel, - item, - position - ).show(fragment.childFragmentManager, GpuUnswizzleDialogFragment.TAG) - } - private class DiffCallback : DiffUtil.ItemCallback() { override fun areItemsTheSame(oldItem: SettingsItem, newItem: SettingsItem): Boolean { return oldItem.setting.key == newItem.setting.key diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt index ca282ba159..369ba4178f 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/SettingsFragmentPresenter.kt @@ -545,11 +545,14 @@ class SettingsFragmentPresenter( add(IntSetting.RENDERER_NVDEC_EMULATION.key) add(BooleanSetting.SYNC_MEMORY_OPERATIONS.key) + add(BooleanSetting.STALL_ON_GPU_FENCE_WAIT.key) add(BooleanSetting.RENDERER_USE_DISK_SHADER_CACHE.key) add(BooleanSetting.RENDERER_FORCE_MAX_CLOCK.key) add(BooleanSetting.RENDERER_REACTIVE_FLUSHING.key) + add(BooleanSetting.RENDERER_BARRIER_FEEDBACK_LOOPS.key) add(BooleanSetting.ENABLE_BUFFER_HISTORY.key) add(BooleanSetting.ENABLE_GPU_BUFFER_READBACK.key) + add(BooleanSetting.USE_UNIFIED_MEMORY.key) add(BooleanSetting.USE_OPTIMIZED_VERTEX_BUFFERS.key) add(HeaderSetting(R.string.hacks)) @@ -560,8 +563,6 @@ class SettingsFragmentPresenter( add(BooleanSetting.RENDERER_ASYNCHRONOUS_SHADERS.key) add(IntSetting.ANDROID_PIPELINE_WORKERS.key) add(BooleanSetting.RENDERER_ASYNCHRONOUS_GPU_EMULATION.key) - add(BooleanSetting.RENDERER_ASYNC_PRESENTATION.key) - add(SettingsItem.GPU_UNSWIZZLE_COMBINED) add(HeaderSetting(R.string.extensions)) @@ -1527,6 +1528,7 @@ class SettingsFragmentPresenter( add(HeaderSetting(R.string.cpu)) add(IntSetting.CPU_BACKEND.key) + add(BooleanSetting.RELOCATE_36BIT_ADDRESS_SPACE.key) add(IntSetting.CPU_ACCURACY.key) add(BooleanSetting.USE_AUTO_STUB.key) add(SettingsItem.FASTMEM_COMBINED) diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/viewholder/GpuUnswizzleViewHolder.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/viewholder/GpuUnswizzleViewHolder.kt deleted file mode 100644 index 06701eb077..0000000000 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/features/settings/ui/viewholder/GpuUnswizzleViewHolder.kt +++ /dev/null @@ -1,71 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -package org.yuzu.yuzu_emu.features.settings.ui.viewholder - -import android.view.View -import org.yuzu.yuzu_emu.R -import org.yuzu.yuzu_emu.databinding.ListItemSettingBinding -import org.yuzu.yuzu_emu.features.settings.model.view.GpuUnswizzleSetting -import org.yuzu.yuzu_emu.features.settings.model.view.SettingsItem -import org.yuzu.yuzu_emu.features.settings.ui.SettingsAdapter -import org.yuzu.yuzu_emu.utils.ViewUtils.setVisible - -class GpuUnswizzleViewHolder(val binding: ListItemSettingBinding, adapter: SettingsAdapter) : - SettingViewHolder(binding.root, adapter) { - private lateinit var setting: GpuUnswizzleSetting - - override fun bind(item: SettingsItem) { - setting = item as GpuUnswizzleSetting - binding.textSettingName.text = setting.title - binding.textSettingDescription.setVisible(item.description.isNotEmpty()) - binding.textSettingDescription.text = item.description - - binding.textSettingValue.setVisible(true) - val resMgr = binding.root.context.resources - - if (setting.isEnabled()) { - // Show a summary of current settings - val textureSizeEntries = resMgr.getStringArray(setting.textureSizeChoicesId) - val textureSizeValues = resMgr.getIntArray(setting.textureSizeValuesId) - val textureSizeIndex = textureSizeValues.indexOf(setting.getTextureSize()) - val textureSizeLabel = if (textureSizeIndex >= 0) textureSizeEntries[textureSizeIndex] else "?" - - val streamSizeEntries = resMgr.getStringArray(setting.streamSizeChoicesId) - val streamSizeValues = resMgr.getIntArray(setting.streamSizeValuesId) - val streamSizeIndex = streamSizeValues.indexOf(setting.getStreamSize()) - val streamSizeLabel = if (streamSizeIndex >= 0) streamSizeEntries[streamSizeIndex] else "?" - - val chunkSizeEntries = resMgr.getStringArray(setting.chunkSizeChoicesId) - val chunkSizeValues = resMgr.getIntArray(setting.chunkSizeValuesId) - val chunkSizeIndex = chunkSizeValues.indexOf(setting.getChunkSize()) - val chunkSizeLabel = if (chunkSizeIndex >= 0) chunkSizeEntries[chunkSizeIndex] else "?" - - binding.textSettingValue.text = "$textureSizeLabel ⋅ $streamSizeLabel ⋅ $chunkSizeLabel" - } else { - binding.textSettingValue.text = resMgr.getString(R.string.gpu_unswizzle_disabled) - } - - binding.buttonClear.setVisible(setting.clearable) - binding.buttonClear.setOnClickListener { - adapter.onClearClick(setting, bindingAdapterPosition) - } - - setStyle(setting.isEditable, binding) - } - - override fun onClick(clicked: View) { - if (!setting.isEditable) { - return - } - - adapter.onGpuUnswizzleClick(setting, bindingAdapterPosition) - } - - override fun onLongClick(clicked: View): Boolean { - if (setting.isEditable) { - return adapter.onLongClick(setting, bindingAdapterPosition) - } - return false - } -} diff --git a/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/GameProperties.kt b/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/GameProperties.kt index a186b91688..24073e2a92 100644 --- a/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/GameProperties.kt +++ b/src/android/app/src/main/java/org/yuzu/yuzu_emu/model/GameProperties.kt @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: 2023 yuzu Emulator Project diff --git a/src/android/app/src/main/jni/emu_window/emu_window.cpp b/src/android/app/src/main/jni/emu_window/emu_window.cpp index 8dfea40693..614fd2569c 100644 --- a/src/android/app/src/main/jni/emu_window/emu_window.cpp +++ b/src/android/app/src/main/jni/emu_window/emu_window.cpp @@ -64,8 +64,8 @@ void EmuWindow_Android::OnTouchReleased(int id) { EmulationSession::GetInstance().GetInputSubsystem().GetTouchScreen()->TouchReleased(id); } -void EmuWindow_Android::OnFrameDisplayed() { - UpdateObservedFrameRate(); +void EmuWindow_Android::OnFrameDisplayed(u32 presented_frames) { + UpdateObservedFrameRate(presented_frames); UpdateFrameRateHint(); if (!m_first_frame) { @@ -75,15 +75,16 @@ void EmuWindow_Android::OnFrameDisplayed() { } } -void EmuWindow_Android::UpdateObservedFrameRate() { +void EmuWindow_Android::UpdateObservedFrameRate(u32 presented_frames) { + m_presented_frames = static_cast((std::max)(presented_frames, 1u)); const auto now = Clock::now(); if (m_last_frame_display_time.time_since_epoch().count() != 0) { const auto frame_time = std::chrono::duration(now - m_last_frame_display_time); const float seconds = frame_time.count(); if (seconds > 0.0f) { - const float instantaneous_rate = 1.0f / seconds; + const float instantaneous_rate = m_presented_frames / seconds; if (std::isfinite(instantaneous_rate) && instantaneous_rate >= 1.0f && - instantaneous_rate <= 240.0f) { + instantaneous_rate <= 240.0f * m_presented_frames) { constexpr float SmoothingFactor = 0.15f; if (m_smoothed_present_rate <= 0.0f) { m_smoothed_present_rate = instantaneous_rate; @@ -124,18 +125,9 @@ float EmuWindow_Android::GetFrameTimeVerifiedHint() const { return QuantizeFrameRateHint(verified_rate); } -float EmuWindow_Android::GetPresentedFrameMultiplier() { - if (!Settings::values.frame_gen.GetValue()) { - return 1.0f; - } - return static_cast(std::clamp(Settings::values.frame_gen_multiplier.GetValue(), 2, 4)); -} - float EmuWindow_Android::GetFrameRateHint() const { - const float presented_multiplier = GetPresentedFrameMultiplier(); - const float observed_rate = - std::clamp(m_smoothed_present_rate * presented_multiplier, 0.0f, 240.0f); - const float frame_time_verified_hint = GetFrameTimeVerifiedHint() * presented_multiplier; + const float observed_rate = std::clamp(m_smoothed_present_rate, 0.0f, 240.0f); + const float frame_time_verified_hint = GetFrameTimeVerifiedHint() * m_presented_frames; if (m_last_frame_rate_hint > 0.0f && observed_rate > 0.0f) { const float tolerance = std::max(m_last_frame_rate_hint * 0.12f, 4.0f); @@ -159,7 +151,7 @@ float EmuWindow_Android::GetFrameRateHint() const { return frame_time_verified_hint; } - const float nominal_rate = 60.0f * presented_multiplier; + const float nominal_rate = 60.0f * m_presented_frames; if (!Settings::values.use_speed_limit.GetValue()) { return QuantizeFrameRateHint(nominal_rate); } diff --git a/src/android/app/src/main/jni/emu_window/emu_window.h b/src/android/app/src/main/jni/emu_window/emu_window.h index fca83b91fa..67883946c9 100644 --- a/src/android/app/src/main/jni/emu_window/emu_window.h +++ b/src/android/app/src/main/jni/emu_window/emu_window.h @@ -41,7 +41,7 @@ public: ~EmuWindow_Android() = default; void OnSurfaceChanged(ANativeWindow* surface); - void OnFrameDisplayed() override; + void OnFrameDisplayed(u32 presented_frames) override; void OnTouchPressed(int id, float x, float y); void OnTouchMoved(int id, float x, float y); @@ -58,10 +58,9 @@ private: using Clock = std::chrono::steady_clock; void UpdateFrameRateHint(); - void UpdateObservedFrameRate(); + void UpdateObservedFrameRate(u32 presented_frames); [[nodiscard]] float GetFrameRateHint() const; [[nodiscard]] float GetFrameTimeVerifiedHint() const; - [[nodiscard]] static float GetPresentedFrameMultiplier(); [[nodiscard]] static float QuantizeFrameRateHint(float frame_rate); float m_window_width{}; @@ -73,6 +72,7 @@ private: float m_last_frame_rate_hint = -1.0f; float m_pending_frame_rate_hint = -1.0f; float m_smoothed_present_rate = 0.0f; + float m_presented_frames = 1.0f; Clock::time_point m_last_frame_display_time{}; Clock::time_point m_pending_frame_rate_since{}; std::uint32_t m_pending_frame_rate_hint_votes = 0; diff --git a/src/android/app/src/main/jni/native.cpp b/src/android/app/src/main/jni/native.cpp index 746b00ad11..bd4d843473 100644 --- a/src/android/app/src/main/jni/native.cpp +++ b/src/android/app/src/main/jni/native.cpp @@ -738,6 +738,35 @@ const char* fallback_cpu_detection() { return s_result.c_str(); } +#ifdef ARCHITECTURE_arm64 +struct SystemDriverInfo { + VkPhysicalDeviceProperties properties{}; + VkPhysicalDeviceDriverProperties driver{}; +}; + +SystemDriverInfo QuerySystemDriverInfo(JNIEnv* env, jstring j_hook_lib_dir) { + const std::string hook_lib_dir = Common::Android::GetJString(env, j_hook_lib_dir); + const Common::DynamicLibrary library{adrenotools_open_libvulkan( + RTLD_NOW, 0, nullptr, hook_lib_dir.c_str(), nullptr, nullptr, nullptr, nullptr)}; + Vulkan::vk::InstanceDispatch dld; + const Vulkan::vk::Instance instance = Vulkan::CreateInstance(library, dld, VK_API_VERSION_1_1); + const std::vector devices = instance.EnumeratePhysicalDevices(); + if (devices.empty()) { + throw Vulkan::vk::Exception(VK_ERROR_INITIALIZATION_FAILED); + } + SystemDriverInfo info{}; + info.driver.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES; + VkPhysicalDeviceProperties2 properties2{ + .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, + .pNext = &info.driver, + .properties = {}, + }; + Vulkan::vk::PhysicalDevice(devices[0], dld).GetProperties2(properties2); + info.properties = properties2.properties; + return info; +} +#endif + } // namespace extern "C" { @@ -857,34 +886,18 @@ jboolean JNICALL Java_org_yuzu_yuzu_1emu_utils_GpuDriverHelper_supportsCustomDri jobjectArray Java_org_yuzu_yuzu_1emu_utils_GpuDriverHelper_getSystemDriverInfo( JNIEnv* env, jobject j_obj, jobject j_surf, jstring j_hook_lib_dir) { + std::string version_string{"1.1.0"}; + std::string driver_name{"generic"}; #ifdef ARCHITECTURE_arm64 - const char* file_redirect_dir_{}; - int featureFlags{}; - std::string hook_lib_dir = Common::Android::GetJString(env, j_hook_lib_dir); - auto handle = adrenotools_open_libvulkan(RTLD_NOW, featureFlags, nullptr, hook_lib_dir.c_str(), - nullptr, nullptr, file_redirect_dir_, nullptr); - auto driver_library = std::make_shared(handle); - InputCommon::InputSubsystem input_subsystem; - auto window = - std::make_unique(ANativeWindow_fromSurface(env, j_surf), driver_library); - - Vulkan::vk::InstanceDispatch dld; - Vulkan::vk::Instance vk_instance = Vulkan::CreateInstance( - *driver_library, dld, VK_API_VERSION_1_1, Core::Frontend::WindowSystemType::Android); - - auto surface = Vulkan::CreateSurface(vk_instance, window->GetWindowInfo()); - - auto device = Vulkan::CreateDevice(vk_instance, dld, *surface); - - auto driver_version = device.GetDriverVersion(); - auto version_string = - fmt::format("{}.{}.{}", VK_API_VERSION_MAJOR(driver_version), - VK_API_VERSION_MINOR(driver_version), VK_API_VERSION_PATCH(driver_version)); - auto driver_name = device.GetDriverName(); -#else - auto driver_version = "1.0.0"; - auto version_string = "1.1.0"; //Assume lowest Vulkan level - auto driver_name = "generic"; + try { + const SystemDriverInfo info = QuerySystemDriverInfo(env, j_hook_lib_dir); + const u32 driver_version = info.properties.driverVersion; + version_string = + fmt::format("{}.{}.{}", VK_API_VERSION_MAJOR(driver_version), + VK_API_VERSION_MINOR(driver_version), VK_API_VERSION_PATCH(driver_version)); + driver_name = Vulkan::vk::GetDriverName(info.driver); + } catch (...) { + } #endif jobjectArray j_driver_info = env->NewObjectArray(2, Common::Android::GetStringClass(), Common::Android::ToJString(env, version_string)); env->SetObjectArrayElement(j_driver_info, 1, Common::Android::ToJString(env, driver_name)); @@ -893,32 +906,12 @@ jobjectArray Java_org_yuzu_yuzu_1emu_utils_GpuDriverHelper_getSystemDriverInfo( jstring Java_org_yuzu_yuzu_1emu_utils_GpuDriverHelper_getGpuModel(JNIEnv *env, jobject j_obj, jobject j_surf, jstring j_hook_lib_dir) { #ifdef ARCHITECTURE_arm64 - const char* file_redirect_dir_{}; - int featureFlags{}; - std::string hook_lib_dir = Common::Android::GetJString(env, j_hook_lib_dir); - auto handle = adrenotools_open_libvulkan(RTLD_NOW, featureFlags, nullptr, hook_lib_dir.c_str(), - nullptr, nullptr, file_redirect_dir_, nullptr); - auto driver_library = std::make_shared(handle); - InputCommon::InputSubsystem input_subsystem; - auto window = - std::make_unique(ANativeWindow_fromSurface(env, j_surf), driver_library); - - Vulkan::vk::InstanceDispatch dld; - Vulkan::vk::Instance vk_instance = Vulkan::CreateInstance( - *driver_library, dld, VK_API_VERSION_1_1, Core::Frontend::WindowSystemType::Android); - - auto surface = Vulkan::CreateSurface(vk_instance, window->GetWindowInfo()); - - auto device = Vulkan::CreateDevice(vk_instance, dld, *surface); - - const std::string model_name{device.GetModelName()}; - - window.release(); - - return Common::Android::ToJString(env, model_name); -#else - return Common::Android::ToJString(env, "no-info"); + try { + return Common::Android::ToJString(env, QuerySystemDriverInfo(env, j_hook_lib_dir).properties.deviceName); + } catch (...) { + } #endif + return Common::Android::ToJString(env, "no-info"); } jboolean Java_org_yuzu_yuzu_1emu_NativeLibrary_reloadKeys(JNIEnv* env, jclass clazz) { diff --git a/src/android/app/src/main/res/layout/dialog_gpu_unswizzle.xml b/src/android/app/src/main/res/layout/dialog_gpu_unswizzle.xml deleted file mode 100644 index 8299551618..0000000000 --- a/src/android/app/src/main/res/layout/dialog_gpu_unswizzle.xml +++ /dev/null @@ -1,96 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/android/app/src/main/res/values-ar/strings.xml b/src/android/app/src/main/res/values-ar/strings.xml index 44f86d380f..cf2c968f99 100644 --- a/src/android/app/src/main/res/values-ar/strings.xml +++ b/src/android/app/src/main/res/values-ar/strings.xml @@ -574,8 +574,6 @@ يجبر وحدة معالجة الرسومات على العمل بأقصى سرعة ممكنة (سيظل يتم تطبيق القيود الحرارية). محاكاة غير متزامنة لوحدة معالجة الرسومات يمكن لهذه الحيلة أن تزيد الأداء عن طريق تشغيل محاكاة وحدة معالجة الرسومات بشكل غير متزامن على حساب مشاكل الرسومات وزيادة معدلات الأعطال بسبب العمليات المتعلقة بالتوقيت. - عرض غير متزامن - يمكن لهذه الحيلة أن تزيد من الأداء عن طريق نقل عملية العرض إلى خيط معالجة منفصل على حساب مشاكل الرسوميات. استخدم التنظيف التفاعلي يحسن دقة العرض في بعض الألعاب على حساب الأداء. تمكين سجل التخزين المؤقت @@ -599,17 +597,6 @@ يُمكّن هذا الخيار من التعامل مع عملية إعادة تحجيم الألعاب بطريقة تقليدية باستخدام مسار إعادة التحجيم السريع استخدم تظليل غير متزامن يقوم بتجميع التظليل بشكل غير متزامن. قد يقلل ذلك من التقطعات ولكنه قد يؤدي أيضًا إلى حدوث أخطاء. - إعدادات إلغاء ترتيب بيانات وحدة معالجة الرسومات - قم بضبط معلمات فكّ تشابك النسيج المستندة إلى وحدة معالجة الرسومات أو تعطيلها تمامًا. اضبط هذه الإعدادات لتحقيق التوازن بين الأداء وجودة تحميل النسيج. - تفعيل إلغاء ترتيب بيانات وحدة معالجة الرسومات - تعطيل - الحد الأقصى لحجم النسيج في وحدة معالجة الرسومات بعد إعادة ترتيب البيانات - يُحدد هذا الخيار الحد الأقصى لحجم (ميغابايت) معالجة الصور باستخدام وحدة معالجة الرسومات. مع أن وحدة معالجة الرسومات أسرع في معالجة الصور المتوسطة والكبيرة، إلا أن وحدة المعالجة المركزية قد تكون أكثر كفاءة في معالجة الصور الصغيرة جدًا. اضبط هذا الخيار لتحقيق التوازن الأمثل بين سرعة معالجة الرسومات واستهلاك وحدة المعالجة المركزية. - حجم تدفق إلغاء ترتيب بيانات وحدة معالجة الرسومات - يحدد هذا الخيار حد البيانات لكل إطار لمعالجة الصور الكبيرة. القيم الأعلى تُسرّع تحميل الصور على حساب زيادة زمن استجابة الإطارات؛ أما القيم الأقل فتُقلل من الحمل الزائد على وحدة معالجة الرسومات، ولكنها قد تتسبب في ظهور الصور بشكل مفاجئ. - حجم كتلة إلغاء ترتيب بيانات وحدة معالجة الرسومات - يُحدد هذا الخيار عدد شرائح العمق التي تتم معالجتها لكل دفعة من الصور ثلاثية الأبعاد (3D). زيادة هذا العدد تُحسّن كفاءة الإنتاجية على وحدات معالجة الرسومات القوية، ولكنها قد تُسبب تقطعًا أو انقطاعًا في عمل برنامج التشغيل على الأجهزة ذات المواصفات الأقل قوة. - افتراضي إضافات @@ -1050,25 +1037,10 @@ كسر السرعة - صغير جدًا (16 ميغابايت) - صغير (32 ميغابايت) - قياسي (128 ميغابايت) - كبير (256 ميغابايت) - كبير جدًا (512 ميغابايت) - منخفض جدًا (4 ميغابايت) - منخفض (8 ميغابايت) - قياسي (16 ميغابايت) - متوسط (32 ميغابايت) - عالي (64 ميغابايت) - منخفض جدًا (32) - منخفض (64) - قياسي (128) - متوسط (256) - عالي (512) مئوية diff --git a/src/android/app/src/main/res/values-de/strings.xml b/src/android/app/src/main/res/values-de/strings.xml index 6ed158fc72..307f739870 100644 --- a/src/android/app/src/main/res/values-de/strings.xml +++ b/src/android/app/src/main/res/values-de/strings.xml @@ -508,7 +508,6 @@ Wird der Handheld-Modus verwendet, verringert es die Auflösung und erhöht die Überspringt bestimmte Cache-Invalidierungen auf CPU-Seite während Speicherupdates, reduziert die CPU-Auslastung und verbessert die Leistung. Kann in einigen Spielen zu Fehlern oder Abstürzen führen. Asynchrone Shader Kompiliert Shader asynchron. Dies kann Ruckler reduzieren, aber auch Grafikfehler verursachen. - Standard Erweiterungen @@ -918,25 +917,10 @@ Wirklich fortfahren? Übertaktung - Sehr klein (16 MB) - Klein (32 MB) - Normal (128 MB) - Groß (256 MB) - Sehr groß (512 MB) - Sehr niedrig (4 MB) - Niedrig (8 MB) - Normal (16 MB) - Mittel (32 MB) - Hoch (64 MB) - Sehr niedrig (32) - Niedrig (64) - Normal (128) - Mittel (256) - Hoch (512) Celsius diff --git a/src/android/app/src/main/res/values-es/strings.xml b/src/android/app/src/main/res/values-es/strings.xml index 448d55b92b..11c7fe296f 100644 --- a/src/android/app/src/main/res/values-es/strings.xml +++ b/src/android/app/src/main/res/values-es/strings.xml @@ -517,8 +517,6 @@ Fuerza a la GPU a ejecutarse a la velocidad máxima de reloj posible (se seguirán aplicando restricciones térmicas). Emulación de GPU asíncrona Este hack puede aumentar el rendimiento ejecutando la emulación de la GPU de forma asíncrona, a costa de problemas gráficos y un aumento en la tasa de fallos debido a operaciones relacionadas con la sincronización. - Presentación asíncrona - Este hack puede aumentar el rendimiento al mover la presentación a un hilo independiente de la CPU a costa de problemas gráficos. Usar limpieza reactiva Mejora la precisión de renderizado en algunos juegos, pero reduce el rendimiento. Activar el historial del búfer @@ -541,17 +539,6 @@ Permite el manejo de versiones anteriores para el paso de configuración de reescalado para juegos mediante el uso de una ruta de reescalado rápida. Usar sombreadores asíncronos Compila los sombreadores de forma asíncrona. Esto puede reducir los tirones, pero también puede introducir errores gráficos. - Ajustes de desentrelazado de la GPU - Configura los parámetros de desentrelazado de texturas basadas ​​en la GPU o desactívelos por completo. Modifique estos ajustes para equilibrar el rendimiento y la calidad de las texturas cargadas. - Activar desentrelazado de la GPU - Desactivado - Tamaño máximo de textura de desentrelazado de la GPU - Establece el tamaño máximo (en MB) para el desentrelazado de texturas basada en GPU. Aunque la GPU es más rápida para texturas medianas y grandes, la CPU puede ser más eficiente para texturas muy pequeñas. Ajuste este valor para encontrar el equilibrio entre la aceleración de la GPU y la sobrecarga de la CPU. - Tamaño del flujo de desentrelazado de la GPU - Establece el límite de datos por fotograma para desentrelazar texturas grandes. Los valores altos aceleran la carga de texturas, a coste de una mayor latencia por fotograma; los valores bajos reducen la carga de la GPU, pero pueden causar parpadeos visibles en las texturas. - Tamaño del trozo de desentrelazado de la GPU - Determina la cantidad de cortes de profundidad procesados ​​en un solo envío de texturas 3D. Aumentar este valor puede mejorar el rendimiento en una GPU de gama alta, pero puede causar tirones y problemas en los tiempos de respuesta en hardware más modesto. - Por defecto Extensiones @@ -988,25 +975,10 @@ Overclock - Muy pequeño (16 MB) - Pequeño (32 MB) - Normal (128 MB) - Grande (256 MB) - Muy grande (512 MB) - Muy bajo (4 MB) - Bajo (8 MB) - Normal (16 MB) - Medio (32 MB) - Alto (64 MB) - Muy bajo (32) - Bajo (64) - Normal (128) - Medio (256) - Alto (512) Celsius diff --git a/src/android/app/src/main/res/values-fr/strings.xml b/src/android/app/src/main/res/values-fr/strings.xml index 7507051dd7..a73d1ac9fb 100644 --- a/src/android/app/src/main/res/values-fr/strings.xml +++ b/src/android/app/src/main/res/values-fr/strings.xml @@ -482,8 +482,6 @@ Emuler BGR565 Utiliser les shaders asynchrones Compile les shaders de manière asynchrone. Cela peut réduire les saccades mais peut aussi provoquer des problèmes graphiques. - Désactivé - Par défaut Extensions @@ -884,25 +882,10 @@ 8 Go (Dangereux) - Très petit (16 Mo) - Petit (32 Mo) - Normal (128 Mo) - Large (256 Mo) - Très large (512 Mo) - Très faible (4 Mo) - Faible (8 Mo) - Normal (16 Mo) - Moyen (32 Mo) - Élevé (64 Mo) - Très faible (32) - Faible (64) - Normal (128) - Moyen (256) - Élevé (512) Celsius diff --git a/src/android/app/src/main/res/values-ru/strings.xml b/src/android/app/src/main/res/values-ru/strings.xml index 7a4b6cf91e..6dd50bbf29 100644 --- a/src/android/app/src/main/res/values-ru/strings.xml +++ b/src/android/app/src/main/res/values-ru/strings.xml @@ -554,8 +554,6 @@ Заставляет ГПУ работать на максимально возможных тактовых частотах (тепловые ограничения все равно будут применяться). Асинхронная эмуляция ГПУ Может повысить производительность за счёт асинхронного запуска эмуляции ГПУ, но ценой появления графических ошибок и увеличения частоты вылетов из-за операций, зависящих от синхронизации. - Асинхронная презентация - Может повысить производительность за счёт перемещения вывода кадров в отдельный поток ЦП, но ценой возникновения графических проблем. Реактивная очистка Повышение точности рендеринга в некоторых играх за счет снижения производительности. Включить историю буфера @@ -579,17 +577,6 @@ Включает старый метод обработки этапа перенастройки масштабирования для игр за счёт использования быстрого алгоритма перемасштабирования. Использовать асинхронные шейдеры Компилирует шейдеры асинхронно. Это может уменьшить подтормаживания, но также может вызвать графические артефакты. - Настройки распаковки текстур (Unswizzle) - Настройте параметры распаковки текстур на стороне ГПУ либо полностью отключите эту функцию. Изменение этих параметров позволяет найти баланс между производительностью и качеством загрузки текстур. - Включить распаковку текстур (Unswizzle) - Отключено - Макс. размер текстуры Unswizzle - Задает максимальный размер (в МБ) текстур для преобразования формата (unswizzle) на ГПУ. Хотя ГПУ быстрее работает со средними и большими текстурами, ЦП может быть эффективнее для очень маленьких. Настройте это значение, чтобы найти баланс между ускорением на ГПУ и нагрузкой на ЦП. - Размер потока Unswizzle - Задает лимит данных на кадр для преобразования крупных текстур (unswizzle). Высокие значения ускоряют загрузку текстур ценой увеличения задержки кадра; низкие значения снижают нагрузку на ГПУ, но могут вызывать заметную постепенную подгрузку текстур. - Размер блока Unswizzle - Задает количество слоёв глубины, обрабатываемых за одну пачку для 3D-текстур. Увеличение этого значения улучшает пропускную способность на мощных ГПУ, но может вызывать подтормаживания или таймауты драйвера на слабом железе. - По умолчанию Расширения @@ -1030,25 +1017,10 @@ Разгон - Очень малый (16 МБ) - Малый (32 МБ) - Обычный (128 МБ) - Большой (256 МБ) - Очень большой (512 МБ) - Очень низкий (4 МБ) - Низкий (8 МБ) - Обычный (16 МБ) - Средний (32 МБ) - Высокий (64 МБ) - Очень малый (32) - Малый (64) - Обычный (128) - Средний (256) - Большой (512) Цельсий diff --git a/src/android/app/src/main/res/values-uk/strings.xml b/src/android/app/src/main/res/values-uk/strings.xml index 9923fc72c4..62c9c090e1 100644 --- a/src/android/app/src/main/res/values-uk/strings.xml +++ b/src/android/app/src/main/res/values-uk/strings.xml @@ -481,8 +481,6 @@ Змушує GPU працювати на максимальній тактовій частоті. Асинхронна емуляція ГП Це обхідне рішення може покращити продуктивність завдяки асинхронному виконанню емуляції ГП, але спричинить проблеми з графікою та збільшить частоту збоїв чутливих до таймінгів операцій. - Асинхронне подання - Це обхідне рішення може покращити продуктивність завдяки переміщенню подання на окремий потік ЦП, але спричинить проблеми з графікою. Реактивне очищення Покращує точність рендерингу в деяких іграх. Увімкнути історію буфера @@ -502,17 +500,6 @@ Вмикає застарілу обробку масштабування для ігор, використовуючи швидкий шлях масштабування Асинхронні шейдери Компілює шейдери асинхронно. Це може зменшити затримки, але також може спричинити графічні баги. - Налаштування розпакування за допомогою ГП - Налаштуйте розпакування текстур за допомогою ГП або повністю вимкнути його. Відкоригуйте ці налаштування, щоб урівноважити продуктивність і якість завантаження текстур. - Увімкнути розпакування за допомогою ГП - Вимкнено - Максимальний розмір текстур для розпакування ГП за допомогою ГП - Встановлює максимальний розмір (МБ) для розпакування текстур за допомогою ГП. ГП швидше справляється з текстурами середніх і великих розмірів, а ЦП ефективніший для дуже маленьких. Налаштуйте, щоб збалансувати ГП-прискоренням і навантаженням на ЦП. - Розмір потоку розпакування за допомогою ГП - Встановлює обмеження даних на кадр для розпакування великих текстур. Вищі значення пришвидшують завантаження текстур за рахунок більших кадрових затримок; менші значення зменшують перевантаження ГП але може спричинити помітні появи текстур. - Розмір блоків розпакування за допомогою ГП - Визначає кількість зрізів глибини, оброблених за партію 3D-текстур. Збільшення здатне покращити пропускну здатність на потужних ГП, але може призвести до затримок або затримок драйвера зі слабшим устаткуванням. - Стандартно Розширення @@ -929,25 +916,10 @@ 8 ГБ (Небезпечно) - Дуже малий (16 МБ) - Малий (32 МБ) - Нормальний (128 МБ) - Великий (256 МБ) - Дуже великий (512 МБ) - Дуже низький (4 МБ) - Низький (8 МБ) - Нормальний (16 МБ) - Середній (32 МБ) - Високий (64 МБ) - Дуже низький (32) - Низький (64) - Нормальний (128) - Середній (256) - Високий (512) Цельсій diff --git a/src/android/app/src/main/res/values-zh-rCN/strings.xml b/src/android/app/src/main/res/values-zh-rCN/strings.xml index 193d00c3d1..568d1d384e 100644 --- a/src/android/app/src/main/res/values-zh-rCN/strings.xml +++ b/src/android/app/src/main/res/values-zh-rCN/strings.xml @@ -535,8 +535,6 @@ 强制 GPU 以最大时钟运行 (温控依然生效)。 GPU 异步模拟 此技巧可通过异步运行 GPU 模拟来提升性能，但在执行与时序相关的操作时，可能带来图形显示问题以及增加崩溃概率。 - 异步呈现 - 此技巧通过将图形呈现移至独立的 CPU 线程来提升性能，但可能会带来图形显示问题。 启用反应性刷新 通过牺牲性能来提升某些游戏的渲染精度。 启用缓冲区历史 @@ -560,17 +558,6 @@ 启用通过使用快速缩放路径，来为游戏提供缩放配置处理的传统处理方式 使用异步着色器 以异步方式编译着色器。采用此方式或可减少卡顿，但也可能引入故障点。 - GPU Unswizzle 设置 - 配置基于 GPU 的纹理 unswizzling 参数，或完全禁用该功能。通过调整这些设置，以尝试在性能与纹理加载质量之间取得平衡。 - 启用 GPU Unswizzle - 禁用 - GPU Unswizzle 最大纹理尺寸 - 设置基于 GPU 的纹理 unswizzling 的最大尺寸（MB）。虽然 GPU 处理中等和大型纹理的速度更快，但对于非常小的纹理，CPU 可能更为高效。通过调节此项设置，以平衡GPU 加速与 CPU 开销。 - GPU Unswizzle 流大小 - 设置用于 unswizzling 大型纹理时的每帧数据限制。较高的数值可以加速纹理的加载过程，但会带来更高的帧延迟。而较低的数值则可以降低 GPU 的开销，但也可能会导致可见的纹理闪现。 - GPU Unswizzle 块大小 - 定义了 3D 纹理每批次处理的深度切片数量。增加此数值可在高性能 GPU 上提升吞吐效率，但在性能较弱的硬件上可能会导致卡顿或驱动超时。 - 默认 扩展 @@ -1004,25 +991,10 @@ 超频 - 极小 (16 MB) - 较小 (32 MB) - 正常 (128 MB) - 较大 (256 MB) - 极大 (512 MB) - 极低 (4 MB) - 低 (8 MB) - 正常 (16 MB) - 中 (32 MB) - 高 (64 MB) - 极低 (32) - 低 (64) - 正常 (128) - 中 (256) - 高 (512) 摄氏度 diff --git a/src/android/app/src/main/res/values-zh-rTW/strings.xml b/src/android/app/src/main/res/values-zh-rTW/strings.xml index 0a1c3ba4c2..15a3b9645d 100644 --- a/src/android/app/src/main/res/values-zh-rTW/strings.xml +++ b/src/android/app/src/main/res/values-zh-rTW/strings.xml @@ -555,8 +555,6 @@ 強制 GPU 以可能的最大時脈執行 (熱溫限制仍會被套用) GPU 非同步模擬 此修改可透過使用 GPU 非同步模擬來提升性能，不過可能會導致圖形問題且會提高在進行時序相關操作時當機的機率 - 非同步呈現 - 此修改可以透過將渲染移至單獨的 CPU 執行緒來提升性能，但可能會導致圖形問題 使用重新啟用排清 犧牲效能，以改善部分遊戲的轉譯準確度 啟用緩衝區歷史 @@ -580,17 +578,6 @@ 透過快速重新縮放處理來啟用遊戲重新縮放設定階段的舊版處理方式 使用非同步著色器 使用非同步編譯著色器。這可能會減少卡頓，但也可能導致圖形錯誤。 - GPU Unswizzle 設定 - 設定基於 GPU 的 Unswizzling 參數或完全停用此功能。調整此設定以在性能與紋理載入品質中取得平衡 - 啟用 GPU Unswizzle - 停用 - GPU Unswizzle 最大紋理尺寸 - 設定 GPU 紋理 Unswizzling 的最大值 （MB）。雖然 GPU 處理中型和大型紋理的速度較快，但 CPU 處理非常小的紋理可能更有效率。整此設定以在 GPU 加速與 CPU 負載之間取得平衡 - GPU Unswizzle 流大小 - 設定每個影格處理大型紋理 Unswizzling 的資料上限。數值越高載入紋理的速度越快，但會增加畫面延遲；數值較低則能降低 GPU 負載，不過有可能導致至紋理載入不完全甚至突然出現 - GPU Unswizzle 塊大小 - 設定 3D 紋理每批次處理的深度切片數量。增加此數值可提升規格較佳 GPU 的處理效率，但在較低規格的硬體上可能導致卡頓或驅動程式逾時 - 預設 擴充功能 diff --git a/src/android/app/src/main/res/values/arrays.xml b/src/android/app/src/main/res/values/arrays.xml index b3aed5758c..63352386a4 100644 --- a/src/android/app/src/main/res/values/arrays.xml +++ b/src/android/app/src/main/res/values/arrays.xml @@ -591,54 +591,6 @@ 2 - - @string/gpu_texturesizeswizzle_verysmall - @string/gpu_texturesizeswizzle_small - @string/gpu_texturesizeswizzle_normal - @string/gpu_texturesizeswizzle_large - @string/gpu_texturesizeswizzle_verylarge - - - - 0 - 1 - 2 - 3 - 4 - - - - @string/gpu_swizzle_verylow - @string/gpu_swizzle_low - @string/gpu_swizzle_normal - @string/gpu_swizzle_medium - @string/gpu_swizzle_high - - - - 0 - 1 - 2 - 3 - 4 - - - - @string/gpu_swizzlechunk_verylow - @string/gpu_swizzlechunk_low - @string/gpu_swizzlechunk_normal - @string/gpu_swizzlechunk_medium - @string/gpu_swizzlechunk_high - - - - 0 - 1 - 2 - 3 - 4 - - @string/temperature_celsius @string/temperature_fahrenheit diff --git a/src/android/app/src/main/res/values/strings.xml b/src/android/app/src/main/res/values/strings.xml index 730f6bbade..bd0716013c 100644 --- a/src/android/app/src/main/res/values/strings.xml +++ b/src/android/app/src/main/res/values/strings.xml @@ -105,6 +105,8 @@ Synchronize the core tick speed to the maximum speed percentage to improve performance without altering the game\'s actual speed. Enable Host MMU Emulation This optimization speeds up memory accesses by the guest program. Enabling it causes guest memory reads/writes to be done directly into memory and make use of Host\'s MMU. Disabling this forces all memory accesses to use Software MMU Emulation. + Run 36-bit games natively + Gives games built for the 36-bit address space the 39-bit layout so they can run with NCE instead of the JIT. Experimental: a game that relies on 36-bit addresses can crash. Debug knobs For development use only. 0 to 65535 @@ -346,7 +348,7 @@ Match motion estimation to the game Estimate motion at the resolution the game actually renders instead of the upscaled output. Costs nothing in accuracy, since upscaling adds no motion detail. Motion estimation resolution - Resolution of the optical flow pass, as a fraction of the output. Lowering it is the cheapest way to reclaim performance. + Resolution of the optical flow pass, as a fraction of the game\'s render resolution. Lowering it is the cheapest way to reclaim performance. Frame generation unavailable This GPU driver lacks the Vulkan memory model or half precision (float16) support that the Lossless Scaling shaders require. Optional. Provide your own Lossless.dll to enable frame generation later @@ -573,6 +575,8 @@ Pick how ASTC-compressed textures are decoded for rendering: CPU (slow, safe), GPU (fast, recommended), or CPU Async (no stutters, may cause issues) Sync Memory Operations + Pause CPU on GPU fence timeouts + When a game keeps timing out while waiting for the GPU, pauses every CPU thread until the GPU catches up. Disabling it keeps the other threads running, which can reduce stutter but may break games that rely on it. Ensures data consistency between compute and memory operations. This option should fix issues in some games, but may also reduce performance in some cases. Unreal Engine 4 games often see the most significant changes thereof. Disk shader cache Reduces stuttering by locally storing and loading generated shaders. @@ -580,13 +584,15 @@ Forces the GPU to run at the maximum possible clocks (thermal constraints will still be applied). GPU async emulation This hack can increase performance by running GPU emulation asynchronously at the cost of graphical issues and increased crash rates by timing-related operations. - Asynchronous presentation - This hack can increase performance by moving presentation to a separate CPU thread at the cost of graphical issues. Use reactive flushing Improves rendering accuracy in some games at the cost of performance. + Barrier feedback loops + Improves rendering of transparency effects in specific games at the cost of performance. Enable buffer history Enables access to previous buffer states. This option may improve rendering quality and performance consistency in some games. Enable GPU Buffer Readback + Unified Memory + Backs part of the emulated memory with GPU-shared buffers so textures are unswizzled directly from it, skipping a CPU copy. Needs 12 GB of RAM or more and a GPU with I/O coherency; it turns itself off otherwise. Takes effect on the next game launch. Preserves GPU-modified buffer data by reading it back before uploads. Some games require this to render certain effects properly. May cause issues if the hardware cannot handle the additional workload. Optimized Vertex Buffers Enables optimized vertex buffer binding for improved performance. Requires Mesa 26.0+ Turnip drivers/ QCOM drivers. Will crash on older Turnip drivers (25.3 and below). @@ -605,17 +611,6 @@ Enables a legacy handling for the rescale configuration pass for games by using a quick rescale path Use asynchronous shaders Compiles shaders asynchronously. This may reduce stutters but may also introduce glitches. - GPU Unswizzle Settings - Configure GPU-based texture unswizzling parameters or disable it entirely. Adjust these settings to balance performance and texture loading quality. - Enable GPU Unswizzle - Disabled - GPU Unswizzle Max Texture Size - Sets the maximum size (MB) for GPU-based texture unswizzling. While the GPU is faster for medium and large textures, the CPU may be more efficient for very small ones. Adjust this to find the balance between GPU acceleration and CPU overhead. - GPU Unswizzle Stream Size - Sets the data limit per frame for unswizzling large textures. Higher values speed up texture loading at the cost of higher frame latency; lower values reduce GPU overhead but may cause visible texture pop-in. - GPU Unswizzle Chunk Size - Defines the number of depth slices processed per batch for 3D textures. Increasing this improves throughput efficiency on powerful GPUs but may cause stuttering or driver timeouts on weaker hardware. - Default Extensions @@ -1065,25 +1060,10 @@ Overclock - Very Small (16 MB) - Small (32 MB) - Normal (128 MB) - Large (256 MB) - Very Large (512 MB) - Very Low (4 MB) - Low (8 MB) - Normal (16 MB) - Medium (32 MB) - High (64 MB) - Very Low (32) - Low (64) - Normal (128) - Medium (256) - High (512) Celsius diff --git a/src/common/CMakeLists.txt b/src/common/CMakeLists.txt index 61e0e64aa8..3a6d2acf8a 100644 --- a/src/common/CMakeLists.txt +++ b/src/common/CMakeLists.txt @@ -108,8 +108,6 @@ add_library( settings_setting.h slot_vector.h socket_types.h - sparse_large_vector.cpp - sparse_large_vector.h spin_lock.h stb.cpp stb.h @@ -139,6 +137,8 @@ add_library( uuid.cpp uuid.h vector_math.h + virtual_buffer.cpp + virtual_buffer.h zstd_compression.cpp zstd_compression.h fs/ryujinx_compat.h fs/ryujinx_compat.cpp diff --git a/src/common/fiber.cpp b/src/common/fiber.cpp index 0e7f28c5b9..69eca732eb 100644 --- a/src/common/fiber.cpp +++ b/src/common/fiber.cpp @@ -9,6 +9,7 @@ #include "common/assert.h" #include "common/fiber.h" +#include "common/virtual_buffer.h" #include diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 75a9da80e9..dd0780a283 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -178,14 +178,6 @@ public: Release(); } - void* Allocate(size_t size) { - auto* ptr = VirtualAlloc(nullptr, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); - if (ptr == nullptr) { - LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, error {}", size, GetLastError()); - } - return ptr; - } - void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { std::unique_lock lock{placeholder_mutex}; if (!IsNiechePlaceholder(virtual_offset, length)) { @@ -406,10 +398,6 @@ private: // For managarm: see https://github.com/managarm/managarm/issues/1370 #else // ^^^ Windows ^^^ vvv POSIX vvv -#ifndef MAP_NOCORE -#define MAP_NOCORE 0 -#endif - #ifdef ARCHITECTURE_arm64 static void* ChooseVirtualBase(size_t virtual_size) { @@ -434,7 +422,7 @@ static void* ChooseVirtualBase(size_t virtual_size) { // Note: we may be able to take advantage of MAP_FIXED_NOREPLACE here. void* map_pointer = mmap(reinterpret_cast(hint_address), virtual_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0); + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); // If we successfully mapped, we're done. if (reinterpret_cast(map_pointer) == hint_address) { @@ -454,11 +442,11 @@ static void* ChooseVirtualBase(size_t virtual_size) { static void* ChooseVirtualBase(size_t virtual_size) { #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__OpenBSD__) || defined(__sun__) || defined(__HAIKU__) || defined(__managarm__) || defined(__AIX__) - void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER | MAP_NOCORE, -1, 0); + void* virtual_base = mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_ALIGNED_SUPER, -1, 0); if (virtual_base != MAP_FAILED) return virtual_base; #endif - return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_NOCORE, -1, 0); + return mmap(nullptr, virtual_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); } #endif @@ -552,13 +540,13 @@ public: } if (use_anon) { LOG_WARNING(Common_Memory, "Using private mappings instead of shared ones"); - backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE | MAP_NOCORE, -1, 0)); + backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); if (fd > 0) { fd = -1; close(fd); } } else { - backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_NOCORE, fd, 0)); + backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0)); } if (backing_base == MAP_FAILED) { LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); @@ -582,14 +570,6 @@ public: Release(); } - void* Allocate(size_t size) { - auto* ptr = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0); - if (ptr == MAP_FAILED) { - LOG_CRITICAL(HW_Memory, "Failed to allocate fallback buffer with size {:#x}, {}", size, strerror(errno)); - } - return ptr; - } - void Map(size_t virtual_offset, size_t host_offset, size_t length, MemoryPermission perms) { // Intersect the range with our address space. AdjustMap(&virtual_offset, &length); @@ -710,10 +690,12 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) { #if defined(__OPENORBIS__) || defined(__managarm__) LOG_WARNING(HW_Memory, "Platform doesn't support fastmem"); - backing_base = static_cast(mmap(nullptr, backing_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); + fallback_buffer.emplace(backing_size); + backing_base = fallback_buffer->data(); virtual_base = nullptr; #else // Try to allocate a fastmem arena. + // The implementation will fail with std::bad_alloc on errors. impl = std::make_unique(AlignUp(backing_size, PageAlignment), AlignUp(virtual_size, PageAlignment) + HugePageSize); if (impl->Init()) { backing_base = impl->backing_base; @@ -724,26 +706,16 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) virtual_base_offset = virtual_base - impl->virtual_base; } } else { - LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it"); - fallback_buffer = true; - backing_base = static_cast(impl->Allocate(backing_size)); - virtual_base = nullptr; impl.reset(); + LOG_WARNING(HW_Memory, "Platform can support fastmem, but can't create it"); + fallback_buffer.emplace(backing_size); + backing_base = fallback_buffer->data(); + virtual_base = nullptr; } #endif } -HostMemory::~HostMemory() { -#ifdef _WIN32 - if (fallback_buffer) { - VirtualFree(backing_base, backing_size, MEM_RELEASE); - } -#else - if (fallback_buffer) { - munmap(backing_base, backing_size); - } -#endif -} +HostMemory::~HostMemory() = default; HostMemory::HostMemory(HostMemory&&) noexcept = default; diff --git a/src/common/host_memory.h b/src/common/host_memory.h index a7fd0532b7..24f4670732 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -10,6 +10,7 @@ #include #include "common/common_funcs.h" #include "common/common_types.h" +#include "common/virtual_buffer.h" namespace Common { @@ -85,7 +86,7 @@ private: u8* virtual_base{}; size_t virtual_base_offset{}; // Windows requires it for kernels whom lack proper support for some functions! - bool fallback_buffer{false}; + std::optional> fallback_buffer; }; } // namespace Common diff --git a/src/common/page_table.cpp b/src/common/page_table.cpp index 73278cfc59..33fff39002 100644 --- a/src/common/page_table.cpp +++ b/src/common/page_table.cpp @@ -13,11 +13,39 @@ PageTable::PageTable() = default; PageTable::~PageTable() noexcept = default; -void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_bits) { - auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_bits); - entries.ResizeAndClear(num_page_table_entries); +bool PageTable::BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context, + Common::ProcessAddress address) const { + out_context->next_offset = GetInteger(address); + out_context->next_page = address / page_size; + + return this->ContinueTraversal(out_entry, out_context); +} + +bool PageTable::ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const { + // Setup invalid defaults. + out_entry->phys_addr = 0; + out_entry->block_size = page_size; + // Validate that we can read the actual entry. + if (auto const page = context->next_page; page < entries.size()) { + // Validate that the entry is mapped. + if (auto const paddr = entries[page].addr; paddr != 0) { + // Populate the results. + out_entry->phys_addr = paddr + context->next_offset; + context->next_page += 1; + context->next_offset += page_size; + return true; + } + } + context->next_page += 1; + context->next_offset += page_size; + return false; +} + +void PageTable::Resize(std::size_t address_space_width_in_bits, std::size_t page_size_in_bits) { + auto const num_page_table_entries = 1ULL << (address_space_width_in_bits - page_size_in_bits); + entries.resize(num_page_table_entries); current_address_space_width_in_bits = address_space_width_in_bits; - current_page_bits = page_bits; + page_size = 1ULL << page_size_in_bits; } } // namespace Common diff --git a/src/common/page_table.h b/src/common/page_table.h index 1588eca616..0dfd152331 100644 --- a/src/common/page_table.h +++ b/src/common/page_table.h @@ -9,22 +9,22 @@ #include #include "common/common_types.h" -#include "common/sparse_large_vector.h" #include "common/typed_address.h" +#include "common/virtual_buffer.h" namespace Common { enum class PageType : u8 { /// Page is unmapped and should cause an access error. - Unmapped = 0b00, + Unmapped, /// Page is mapped to regular memory. This is the only type you can get pointers to. - Memory = 0b01, + Memory, /// Page is mapped to regular memory, but inaccessible from CPU fastmem and must use /// the callbacks. - DebugMemory = 0b10, + DebugMemory, /// Page is mapped to regular memory, but also needs to check for rasterizer cache flushing and /// invalidation - RasterizerCachedMemory = 0b11, + RasterizerCachedMemory, }; /** @@ -42,86 +42,57 @@ struct PageTable { u64 next_offset{}; }; - /// Masks out bits reserved for attribute tagging. - static constexpr u64 ATTRIBUTE_MASK = ((1ULL << 44) - 1) << 12; - - /// Specifies sign bit for page table entries. - static constexpr u64 SIGN_BIT = 45 + 12; // 44 bits of data + page offset + /// Number of bits reserved for attribute tagging. + /// This can be at most the guaranteed alignment of the pointers in the page table. + static constexpr int ATTRIBUTE_BITS = 2; /** - * Atomic tuple of host pointer, page type, and block id. - * This uses the lower bits of a given pointer to store the attributes. + * Pair of host pointer and page type attribute. + * This uses the lower bits of a given pointer to store the attribute tag. * Writing and reading the pointer attribute pair is guaranteed to be atomic for the same method * call. In other words, they are guaranteed to be synchronized at all times. */ - class PageEntryData { + class PageInfo { public: - struct Data { - Data(bool marked_, PageType type_, u16 block_, u64 page_) - : marked(static_cast(marked_) & 0b1) - , type(static_cast(type_) & ((1ULL << 2) - 1)) - , block(static_cast(block_) & ((1ULL << 9) - 1)) - , page((page_ >> 12) & ((1ULL << 45) - 1)) - , block2((static_cast(block_) >> 9) & ((1ULL << 7) - 1)) {} - u64 marked : 1; - u64 type : 2; - u64 block : 9; - u64 page : 45; // 44 bits of actual data (64 - page offset (12) - reserved (8)) + a sign bit - u64 block2 : 7; - }; - - [[nodiscard]] Data Raw() const noexcept { - return std::bit_cast(data_raw.load(std::memory_order_relaxed)); - } - /// Returns the page pointer - [[nodiscard]] uintptr_t Pointer(bool ignored_marked = false) const noexcept { - return ExtractPointer(std::bit_cast(data_raw.load(std::memory_order_relaxed)), ignored_marked); + [[nodiscard]] uintptr_t Pointer() const noexcept { + return ExtractPointer(raw.load(std::memory_order_relaxed)); } /// Returns the page type attribute [[nodiscard]] PageType Type() const noexcept { - return static_cast(std::bit_cast(data_raw.load(std::memory_order_relaxed)).type); - } - - /// Returns the block identifier. - [[nodiscard]] u16 Block() const noexcept { - return ExtractBlock(std::bit_cast(data_raw.load(std::memory_order_relaxed))); + return ExtractType(raw.load(std::memory_order_relaxed)); } /// Returns the page pointer and attribute pair, extracted from the same atomic read - [[nodiscard]] std::tuple PointerTypeBlock(bool ignore_marked = false) const noexcept { - const auto non_atomic_raw = std::bit_cast(data_raw.load(std::memory_order_relaxed)); - return {ExtractPointer(non_atomic_raw, ignore_marked), static_cast(non_atomic_raw.type), ExtractBlock(non_atomic_raw)}; + [[nodiscard]] std::pair PointerType() const noexcept { + const uintptr_t non_atomic_raw = raw.load(std::memory_order_relaxed); + return {ExtractPointer(non_atomic_raw), ExtractType(non_atomic_raw)}; } - /// Write page info atomically - constexpr void Store(bool marked, PageType type, u16 block, uintptr_t pointer) noexcept { - data_raw.store(std::bit_cast(Data{marked, type, block, pointer})); + /// Returns the raw representation of the page information. + /// Use ExtractPointer and ExtractType to unpack the value. + [[nodiscard]] uintptr_t Raw() const noexcept { + return raw.load(std::memory_order_relaxed); } - constexpr void MarkRasterizerCached() noexcept { - data_raw.fetch_or(0b111); - } - - constexpr void MarkDebug(u64 ptr, u16 block) noexcept { - Store(true, PageType::DebugMemory, block, ptr); + /// Write a page pointer and type pair atomically + void Store(uintptr_t pointer, PageType type) noexcept { + raw.store(pointer | uintptr_t(type)); } /// Unpack a pointer from a page info raw representation - [[nodiscard]] static uintptr_t ExtractPointer(Data raw, bool ignore_marked = false) noexcept { - return raw.marked && !ignore_marked ? 0 - // shift raw.page's fake sign bit to the actual sign bit, then sign extend - : ((s64)(raw.page << (64 - 44))) >> (64 - 44 - 12); + [[nodiscard]] static uintptr_t ExtractPointer(uintptr_t raw) noexcept { + return raw & (~uintptr_t{0} << ATTRIBUTE_BITS); } - [[nodiscard]] static u16 ExtractBlock(Data raw) noexcept { - return static_cast(raw.block | (raw.block2 << 9)); + /// Unpack a page type from a page info raw representation + [[nodiscard]] static PageType ExtractType(uintptr_t raw) noexcept { + return static_cast(raw & ((uintptr_t{1} << ATTRIBUTE_BITS) - 1)); } private: - std::atomic data_raw; - static_assert(sizeof(Data) == sizeof(std::atomic)); + std::atomic raw; }; PageTable(); @@ -129,8 +100,13 @@ struct PageTable { PageTable(const PageTable&) = delete; PageTable& operator=(const PageTable&) = delete; - PageTable(PageTable&&) noexcept = delete; - PageTable& operator=(PageTable&&) noexcept = delete; + + PageTable(PageTable&&) noexcept = default; + PageTable& operator=(PageTable&&) noexcept = default; + + bool BeginTraversal(TraversalEntry* out_entry, TraversalContext* out_context, + Common::ProcessAddress address) const; + bool ContinueTraversal(TraversalEntry* out_entry, TraversalContext* context) const; /** * Resizes the page table to be able to accommodate enough pages within @@ -145,14 +121,30 @@ struct PageTable { return current_address_space_width_in_bits; } + bool GetPhysicalAddress(Common::PhysicalAddress* out_phys_addr, + Common::ProcessAddress virt_addr) const { + if (virt_addr > (1ULL << this->GetAddressSpaceBits())) { + return false; + } + + *out_phys_addr = entries[virt_addr / page_size].addr + GetInteger(virt_addr); + return true; + } + /// Vector of memory pointers backing each page. An entry can only be non-null if the /// corresponding attribute element is of type `Memory`. - SparseLargeVector entries; - static_assert(sizeof(PageEntryData) == 8); + struct PageEntryData { + PageInfo ptr; + u64 block; + u64 addr; + u64 padding; + }; + VirtualBuffer entries; + static_assert(sizeof(PageEntryData) == 32); u8* fastmem_arena{}; std::size_t current_address_space_width_in_bits{}; - std::size_t current_page_bits{}; + std::size_t page_size{}; }; } // namespace Common diff --git a/src/common/settings.cpp b/src/common/settings.cpp index bc46b61bb4..7605305a04 100644 --- a/src/common/settings.cpp +++ b/src/common/settings.cpp @@ -48,7 +48,6 @@ SWITCHABLE(AnisotropyMode, true); SWITCHABLE(AntiAliasing, false); SWITCHABLE(AspectRatio, true); SWITCHABLE(AstcDecodeMode, true); -SWITCHABLE(AstcRecompression, true); SWITCHABLE(AudioMode, true); SWITCHABLE(CpuBackend, true); SWITCHABLE(CpuAccuracy, true); diff --git a/src/common/settings.h b/src/common/settings.h index c1c2d4f0df..59f3d448da 100644 --- a/src/common/settings.h +++ b/src/common/settings.h @@ -65,7 +65,6 @@ SWITCHABLE(AnisotropyMode, true); SWITCHABLE(AntiAliasing, false); SWITCHABLE(AspectRatio, true); SWITCHABLE(AstcDecodeMode, true); -SWITCHABLE(AstcRecompression, true); SWITCHABLE(AudioMode, true); SWITCHABLE(CpuBackend, true); SWITCHABLE(CpuAccuracy, true); @@ -260,6 +259,8 @@ struct Values { Category::Cpu}; SwitchableSetting cpu_accuracy{linkage, CpuAccuracy::Auto, "cpu_accuracy", Category::Cpu}; + SwitchableSetting relocate_36bit_address_space{ + linkage, false, "relocate_36bit_address_space", Category::Cpu}; SwitchableSetting cpu_clock{linkage, CpuClock::Normal, "fast_cpu_time", @@ -553,22 +554,6 @@ struct Values { "accelerate_astc", Category::RendererAdvanced}; - SwitchableSetting frame_pacing_mode{linkage, - FramePacingMode::Target_Auto, - FramePacingMode::Target_Auto, - FramePacingMode::Target_120, - "frame_pacing_mode", - Category::RendererAdvanced, - Specialization::Default, - true, - true}; - - SwitchableSetting astc_recompression{linkage, - AstcRecompression::Uncompressed, - "astc_recompression", - Category::RendererAdvanced}; - - SwitchableSetting sync_memory_operations{linkage, false, "sync_memory_operations", @@ -576,6 +561,8 @@ struct Values { Specialization::Default, true, true}; + SwitchableSetting stall_on_gpu_fence_wait{linkage, true, "stall_on_gpu_fence_wait", + Category::RendererAdvanced}; SwitchableSetting renderer_force_max_clock{linkage, false, "force_max_clock", Category::RendererAdvanced}; @@ -602,8 +589,13 @@ struct Values { "use_reactive_flushing", Category::RendererAdvanced}; - SwitchableSetting barrier_feedback_loops{linkage, true, "barrier_feedback_loops", - Category::RendererAdvanced}; + SwitchableSetting barrier_feedback_loops{linkage, + true, + "barrier_feedback_loops", + Category::RendererAdvanced, + Specialization::Default, + true, + true}; SwitchableSetting enable_buffer_history{linkage, false, @@ -641,7 +633,7 @@ struct Values { true}; SwitchableSetting async_presentation{linkage, #ifdef __ANDROID__ - false, + true, #else false, #endif @@ -666,26 +658,8 @@ struct Values { SwitchableSetting use_asynchronous_shaders{linkage, false, "use_asynchronous_shaders", Category::RendererHacks}; - SwitchableSetting gpu_unswizzle_texture_size{linkage, - GpuUnswizzleSize::Large, - "gpu_unswizzle_texture_size", - Category::RendererHacks, - Specialization::Default}; - - SwitchableSetting gpu_unswizzle_stream_size{linkage, - GpuUnswizzle::Medium, - "gpu_unswizzle_stream_size", - Category::RendererHacks, - Specialization::Default}; - - SwitchableSetting gpu_unswizzle_chunk_size{linkage, - GpuUnswizzleChunk::Medium, - "gpu_unswizzle_chunk_size", - Category::RendererHacks, - Specialization::Default}; - - SwitchableSetting gpu_unswizzle_enabled{linkage, false, "gpu_unswizzle_enabled", - Category::RendererHacks}; + SwitchableSetting use_unified_memory{linkage, false, "use_unified_memory", + Category::RendererHacks}; SwitchableSetting dyna_state{linkage, #if defined(__ANDROID__) diff --git a/src/common/settings_enums.h b/src/common/settings_enums.h index 4b406f4c35..60e2b55742 100644 --- a/src/common/settings_enums.h +++ b/src/common/settings_enums.h @@ -130,8 +130,6 @@ ENUM(TimeZone, Auto, Default, Cet, Cst6Cdt, Cuba, Eet, Egypt, Eire, Est, Est5Edt Roc, Rok, Singapore, Turkey, Uct, Universal, Utc, WSu, Wet, Zulu); ENUM(AnisotropyMode, Automatic, Default, X2, X4, X8, X16); ENUM(AstcDecodeMode, Cpu, Gpu, CpuAsynchronous); -ENUM(AstcRecompression, Uncompressed, Bc1, Bc3); -ENUM(FramePacingMode, Target_Auto, Target_30, Target_60, Target_90, Target_120); ENUM(VSyncMode, Immediate, Mailbox, Fifo, FifoRelaxed); ENUM(VramUsageMode, Conservative, Aggressive); ENUM(RendererBackend, OpenGL_GLSL, Vulkan, Null, OpenGL_GLASM, OpenGL_SPIRV); @@ -153,9 +151,6 @@ ENUM(ConsoleMode, Handheld, Docked); ENUM(AppletMode, HLE, LLE); ENUM(SpirvOptimizeMode, Never, OnLoad, Always); ENUM(GpuClock, Normal, Boost, Overclock) -ENUM(GpuUnswizzleSize, VerySmall, Small, Normal, Large, VeryLarge) -ENUM(GpuUnswizzle, VeryLow, Low, Normal, Medium, High) -ENUM(GpuUnswizzleChunk, VeryLow, Low, Normal, Medium, High) ENUM(TemperatureUnits, Celsius, Fahrenheit) ENUM(ExtendedDynamicState, Disabled, EDS1, EDS2, EDS3); ENUM(GpuLogLevel, Off, Errors, Standard, Verbose, All) diff --git a/src/common/sparse_large_vector.cpp b/src/common/sparse_large_vector.cpp deleted file mode 100644 index ce6455a235..0000000000 --- a/src/common/sparse_large_vector.cpp +++ /dev/null @@ -1,143 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -/* virtual_buffer.cpp */ -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#ifdef _WIN32 -#include -#include -#else -#include -#endif - -#include "common/alignment.h" -#include "common/assert.h" -#include "common/sparse_large_vector.h" - -namespace Common { - -#ifdef _WIN32 -static std::vector> vector_regions {}; - -// Workaround for handling non-commited memory accessed by Dynarmic; usually result of an error -static LONG WINAPI FakePageFaultHandler(PEXCEPTION_POINTERS info) { - DWORD code = info->ExceptionRecord->ExceptionCode; - u64 exception_addr = reinterpret_cast(info->ExceptionRecord->ExceptionAddress); - - if (code != EXCEPTION_ACCESS_VIOLATION) { - // Not our problem - return EXCEPTION_CONTINUE_SEARCH; - } - - u64 addr = 0, addr2 = 0; - - for (auto region: vector_regions) { - auto addr_shifted = exception_addr >> HostPageBits; - if (region.first <= addr_shifted && addr_shifted <= region.second) { - addr = addr_shifted; - } - - // Page-boundary accesses - if (auto addr_ = (exception_addr + 0x40) >> HostPageBits; addr_ != addr_shifted && region.first <= addr_ && addr_ <= region.second) { - addr2 = addr_; - } - - if (addr != 0 || addr2 != 0) { - break; - } - } - - if (addr == 0 && addr2 == 0) { - // Not our problem - return EXCEPTION_CONTINUE_SEARCH; - } - - LOG_ERROR(HW_Memory, "Accessing an unallocated region of a SparseLargeVector at {:#x}; this shouldn't happen and is likely a Dynarmic error!", exception_addr); - - // Commit this region - if (addr != 0) { - if (!CommitVectorPage(addr << HostPageBits, false)) { - return EXCEPTION_CONTINUE_SEARCH; - } - } - // Commit next region if needed - if (addr2 != 0) { - if (!CommitVectorPage(addr2 << HostPageBits, false)) { - return EXCEPTION_CONTINUE_SEARCH; - } - } - - return EXCEPTION_CONTINUE_EXECUTION; -} - -bool CommitVectorPage(uintptr_t addr, bool write) noexcept { - MEMORY_BASIC_INFORMATION info {}; - auto res = VirtualQuery(reinterpret_cast(addr), &info, sizeof(info)); - if (res == 0) { - LOG_CRITICAL(HW_Memory, "Failed to query large buffer region at {:#x} with error {}, will try committing anyway", addr, GetLastError()); - } else if (info.State != MEM_RESERVE) { - LOG_ERROR(HW_Memory, "Tried to commit an unreserved large buffer region at {:#x} that is not mapped or is already committed (state {:#x})", addr, info.State); - return false; - } - - auto perm = write ? PAGE_READWRITE : PAGE_READONLY; - void* res2 = VirtualAlloc(reinterpret_cast(addr), HostPageSize, MEM_COMMIT, perm); - if (res2 == nullptr) { - LOG_ERROR(HW_Memory, "Failed to commit large buffer region at {:#x}, error {}", addr, GetLastError()); - return false; - } - - return true; -} -#endif - -#ifndef MAP_NOCORE -#define MAP_NOCORE 0 -#endif - -void* AllocateMemoryPages(std::size_t size) noexcept { - if (auto page = HostPageSize; size % page != 0) { - LOG_WARNING(HW_Memory, "Allocating unaligned large vector with size {:#x}; aligning to {} page size", size, page); - size = AlignUp(size, page); - } - -#ifdef _WIN32 - // We will never use this memory entirely so instead of committing it up front let's just reserve it and commit each page individually - void* base = VirtualAlloc(nullptr, size, MEM_RESERVE, PAGE_READWRITE); - - if (base != nullptr) { - vector_regions.emplace_back(reinterpret_cast(base), reinterpret_cast(base) + size); - - static std::once_flag flag; - std::call_once(flag, []() { AddVectoredExceptionHandler(1, FakePageFaultHandler); }); - } else { - // Try committing everything instead?? - LOG_WARNING(HW_Memory, "Failed to reserve large vector region with error {}, trying to commit instead..", GetLastError()); - base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); - } - ASSERT_MSG(base, "Failed to reserve {:#x} sized region with error {}", size, GetLastError()); -#else - void* base = mmap(nullptr, size, PROT_READ, MAP_ANON | MAP_PRIVATE | MAP_NOCORE, -1, 0); - if (base == MAP_FAILED) - base = nullptr; - ASSERT_MSG(base, "Failed to allocate {:#x} sized region with error {}", size, strerror(errno)); -#endif - return base; -} - -void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { - if (auto page = HostPageSize; size % page != 0) { - size = AlignUp(size, page); - } - if (!base) - return; -#ifdef _WIN32 - ASSERT(VirtualFree(base, 0, MEM_RELEASE)); -#else - ASSERT(munmap(base, size) == 0); -#endif -} - -} // namespace Common diff --git a/src/common/sparse_large_vector.h b/src/common/sparse_large_vector.h deleted file mode 100644 index 827944fb08..0000000000 --- a/src/common/sparse_large_vector.h +++ /dev/null @@ -1,194 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -/* virtual_buffer.h */ -// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include -#include -#include -#include - -#ifndef _WIN32 -#include -#include -#endif - -#include "common/alignment.h" -#include "common/assert.h" - -namespace Common { - -#ifdef _WIN32 -constexpr u64 HostPageSize = 0x1000; -constexpr u64 HostPageBits = 12; -constexpr u64 HostPageMask = ~(HostPageSize - 1); -bool CommitVectorPage(uintptr_t addr, bool write) noexcept; -#else -const u64 HostPageSize = sysconf(_SC_PAGESIZE); -const u64 HostPageBits = std::countr_zero(HostPageSize); -const u64 HostPageMask = ~(HostPageSize - 1); -#endif - -void* AllocateMemoryPages(std::size_t size) noexcept; -void FreeMemoryPages(void* base, std::size_t size) noexcept; - -/// A large page-aligned buffer that has optimized memory usage for zero-writes. -template -requires std::is_trivially_copyable_v -class SparseLargeVector final { -public: - constexpr SparseLargeVector() = default; - - explicit SparseLargeVector(std::size_t count) noexcept - : alloc_size{count * sizeof(T)} - { - base_ptr = static_cast(AllocateMemoryPages(alloc_size)); - - // each item in vector holds information for 64 pages - auto denom = HostPageSize * 64; - committed_pages = std::vector>((alloc_size + denom - 1) / denom); - } - - ~SparseLargeVector() noexcept { - FreeMemoryPages(base_ptr, alloc_size); - } - - SparseLargeVector(const SparseLargeVector&) = delete; - SparseLargeVector& operator=(const SparseLargeVector&) = delete; - SparseLargeVector(SparseLargeVector&& other) = delete; - SparseLargeVector& operator=(SparseLargeVector&& other) = delete; - - void ResizeAndClear(std::size_t count) noexcept { - if (auto const new_size = count * sizeof(T); new_size != alloc_size) { - FreeMemoryPages(base_ptr, alloc_size); - alloc_size = new_size; - base_ptr = static_cast(AllocateMemoryPages(alloc_size)); - - auto denom = HostPageSize * 64; - committed_pages = std::vector>((alloc_size + denom - 1) / denom); - } - } - - /// Returns a reference to the value of the requested index and allocates memory if needed. - T& GetAndFault(std::size_t index) noexcept { - if (index > alloc_size / sizeof(T)) { - UNREACHABLE_MSG("Out of bounds RW access on SparseLargeVector @ {}", index); - } - - if (!IsCommittedPage(index)) { - CommitPage(index); - } - return base_ptr[index]; - } - - /// Returns a reference to the value of the requested index if initialized, or will otherwise return a zero-initialized object. - const T& GetOrDefault(std::size_t index) const { -#ifdef _WIN32 - if (!IsCommittedPage(index)) { - return *reinterpret_cast(&default_val); - } -#endif - // On non-Windows, OS page table should optimize this by pointing to a zero page if unallocated. - return base_ptr[index]; - } - - void Set(std::size_t index, const T& value) noexcept { - if (index > alloc_size / sizeof(T)) { - LOG_CRITICAL(Common_Memory, "Out of bounds write on SparseLargeVector @ {}", index); - return; - } - if (!IsCommittedPage(index)) - CommitPage(index); - base_ptr[index] = value; - } - - void ZeroRegion(std::size_t start, std::size_t end_) noexcept { - u64 base = reinterpret_cast(&base_ptr[start]); - const u64 end = reinterpret_cast(&base_ptr[end_]); - - const u64 end_page = AlignUp(base, HostPageSize); - const u64 first_size = (std::min)(end_page, end) - base; - - if (IsCommittedPage(start / sizeof(T))) { - std::memset(reinterpret_cast(base), 0, first_size); - } - - if (end <= end_page) - return; - - base = end_page; - - for (u64 page = base; page < end; page += HostPageSize) { - if (!IsCommittedPage((page - reinterpret_cast(base_ptr)) / sizeof(T))) { - continue; - } - - std::memset(reinterpret_cast(page), 0, (std::min)( HostPageSize, end - page)); - } - } - - constexpr void CommitRegion(size_t index, size_t end_) { - const u64 base = static_cast(index) * sizeof(T); - const u64 end = static_cast(end_) * sizeof(T); - - for (u64 page = AlignDown(base, HostPageSize); page < end; page += HostPageSize) { - if (!IsCommittedPage(page / sizeof(T))) { - CommitPage(page / sizeof(T)); - } - } - } - - constexpr T& GetUnchecked(size_t index) { - return base_ptr[index]; - } - - [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { - return GetOrDefault(index); - } - - [[nodiscard]] constexpr const T* data() const noexcept { - return base_ptr; - } - - [[nodiscard]] constexpr std::size_t size() const noexcept { - return alloc_size / sizeof(T); - } - -private: - [[nodiscard]] constexpr bool IsCommittedPage(std::size_t index) const noexcept { - if (index > alloc_size / sizeof(T)) { - LOG_CRITICAL(Common_Memory, "Out of bounds access on large vector @ {}", index); - return false; - } - - auto page = (index * sizeof(T)) >> HostPageBits; - auto val = committed_pages[page >> 6].load(std::memory_order_acquire); - return (val >> (page & 63)) & 1; - } - - constexpr void CommitPage(std::size_t index) noexcept { - auto page_index = (index * sizeof(T)) >> HostPageBits; - auto page = reinterpret_cast(base_ptr + index) & HostPageMask; -#if defined(_WIN32) - CommitVectorPage(page, true); -#else - mprotect(reinterpret_cast(page), HostPageSize, PROT_READ | PROT_WRITE); -#endif - - committed_pages[page_index >> 6].fetch_or(1ULL << (page_index & 63), std::memory_order_release); - } - - std::size_t alloc_size{}; - T* base_ptr{}; - - std::vector> committed_pages{}; -#ifdef _WIN32 - const std::array default_val{}; -#endif -}; - -} // namespace Common diff --git a/src/common/virtual_buffer.cpp b/src/common/virtual_buffer.cpp new file mode 100644 index 0000000000..8a4265e3e6 --- /dev/null +++ b/src/common/virtual_buffer.cpp @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#ifdef _WIN32 +#include +#else +#include +#endif + +#include "common/assert.h" +#include "common/virtual_buffer.h" + +namespace Common { + +void* AllocateMemoryPages(std::size_t size) noexcept { +#ifdef _WIN32 + void* base = VirtualAlloc(nullptr, size, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE); + if (base == nullptr) { + // Probably failing to reserve is less likely than failing to commit + base = VirtualAlloc(nullptr, size, MEM_COMMIT, PAGE_READWRITE); + } +#else + void* base = mmap(nullptr, size, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + if (base == MAP_FAILED) + base = nullptr; +#endif + ASSERT(base); + return base; +} + +void FreeMemoryPages(void* base, [[maybe_unused]] std::size_t size) noexcept { + if (!base) + return; +#ifdef _WIN32 + ASSERT(VirtualFree(base, 0, MEM_RELEASE)); +#else + ASSERT(munmap(base, size) == 0); +#endif +} + +} // namespace Common diff --git a/src/common/virtual_buffer.h b/src/common/virtual_buffer.h new file mode 100644 index 0000000000..d6386e2a4d --- /dev/null +++ b/src/common/virtual_buffer.h @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +// SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project +// SPDX-License-Identifier: GPL-2.0-or-later + +#pragma once + +#include + +namespace Common { + +void* AllocateMemoryPages(std::size_t size) noexcept; +void FreeMemoryPages(void* base, std::size_t size) noexcept; + +template +class VirtualBuffer final { +public: + // TODO: Uncomment this and change Common::PageTable::PageInfo to be trivially constructible + // using std::atomic_ref once libc++ has support for it + // static_assert( + // std::is_trivially_constructible_v, + // "T must be trivially constructible, as non-trivial constructors will not be executed " + // "with the current allocator"); + + constexpr VirtualBuffer() = default; + explicit VirtualBuffer(std::size_t count) noexcept + : alloc_size{count * sizeof(T)} + { + base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); + } + + ~VirtualBuffer() noexcept { + FreeMemoryPages(base_ptr, alloc_size); + } + + VirtualBuffer(const VirtualBuffer&) = delete; + VirtualBuffer& operator=(const VirtualBuffer&) = delete; + + VirtualBuffer(VirtualBuffer&& other) noexcept + : alloc_size{std::exchange(other.alloc_size, 0)} + , base_ptr{std::exchange(other.base_ptr, nullptr)} + {} + + VirtualBuffer& operator=(VirtualBuffer&& other) noexcept { + alloc_size = std::exchange(other.alloc_size, 0); + base_ptr = std::exchange(other.base_ptr, nullptr); + return *this; + } + + void resize(std::size_t count) noexcept { + if (auto const new_size = count * sizeof(T); new_size != alloc_size) { + FreeMemoryPages(base_ptr, alloc_size); + alloc_size = new_size; + base_ptr = reinterpret_cast(AllocateMemoryPages(alloc_size)); + } + } + + [[nodiscard]] constexpr const T& operator[](std::size_t index) const noexcept { + return base_ptr[index]; + } + + [[nodiscard]] constexpr T& operator[](std::size_t index) noexcept { + return base_ptr[index]; + } + + [[nodiscard]] constexpr T* data() noexcept { + return base_ptr; + } + + [[nodiscard]] constexpr const T* data() const noexcept { + return base_ptr; + } + + [[nodiscard]] constexpr std::size_t size() const noexcept { + return alloc_size / sizeof(T); + } + +private: + std::size_t alloc_size{}; + T* base_ptr{}; +}; + +} // namespace Common diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 74391b5e65..9cc2792a8c 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -1238,8 +1238,7 @@ if (HAS_NCE) arm/nce/interpreter_visitor.cpp arm/nce/interpreter_visitor.h arm/nce/patcher.cpp - arm/nce/patcher.h - arm/nce/visitor_base.h) + arm/nce/patcher.h) target_link_libraries(core PRIVATE merry::oaknut) endif() diff --git a/src/core/arm/dynarmic/arm_dynarmic_32.cpp b/src/core/arm/dynarmic/arm_dynarmic_32.cpp index 80e2b2dca9..97284efa4c 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_32.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_32.cpp @@ -172,12 +172,12 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { if (page_table) { constexpr size_t PageBits = 12; constexpr size_t NumPageTableEntries = 1 << (32 - PageBits); + constexpr size_t PageLog2Stride = 5; + static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); - // Dynarmic will not write to the page table, const_cast is safe here - config.page_table = reinterpret_cast*>( - const_cast(page_table->entries.data())); - config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK; - config.page_table_marked_bit = 0; + config.page_table = reinterpret_cast*>(page_table->entries.data()); + config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; + config.page_table_log2_stride = PageLog2Stride; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; config.only_detect_misalignment_via_page_table_on_page_boundary = true; @@ -188,13 +188,6 @@ void ArmDynarmic32::MakeJit(Common::PageTable* page_table) { config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt; config.recompile_on_exclusive_fastmem_failure = true; - - if (reinterpret_cast(m_system.DeviceMemory().buffer.BackingBasePointer() + - Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) { - // Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries, - // we have to manually sign extend when our actual pointer is negative. - config.page_table_sign_extension = Common::PageTable::SIGN_BIT; - } } // Multi-process state @@ -427,7 +420,6 @@ void ArmDynarmic32::SignalInterrupt(Kernel::KThread* thread) { } void ArmDynarmic32::ClearInstructionCache() { - m_cb->last_code_addr = u64(-1); m_jit->ClearCache(); } diff --git a/src/core/arm/dynarmic/arm_dynarmic_64.cpp b/src/core/arm/dynarmic/arm_dynarmic_64.cpp index 9662bf1712..0b7f6874a7 100644 --- a/src/core/arm/dynarmic/arm_dynarmic_64.cpp +++ b/src/core/arm/dynarmic/arm_dynarmic_64.cpp @@ -211,12 +211,13 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s // Memory if (page_table) { - // Dynarmic will not write to the page table, const_cast is safe here - config.page_table = reinterpret_cast( - const_cast(page_table->entries.data())); + constexpr size_t PageLog2Stride = 5; + static_assert(1 << PageLog2Stride == sizeof(Common::PageTable::PageEntryData)); + + config.page_table = reinterpret_cast(page_table->entries.data()); config.page_table_address_space_bits = std::uint32_t(address_space_bits); - config.page_table_pointer_mask = Common::PageTable::ATTRIBUTE_MASK; - config.page_table_marked_bit = 0; + config.page_table_pointer_mask_bits = Common::PageTable::ATTRIBUTE_BITS; + config.page_table_log2_stride = PageLog2Stride; config.silently_mirror_page_table = false; config.absolute_offset_page_table = true; config.detect_misaligned_access_via_page_table = 16 | 32 | 64 | 128; @@ -230,13 +231,6 @@ void ArmDynarmic64::MakeJit(Common::PageTable* page_table, std::size_t address_s config.fastmem_exclusive_access = config.fastmem_pointer != std::nullopt; config.recompile_on_exclusive_fastmem_failure = true; - - if (reinterpret_cast(m_system.DeviceMemory().buffer.BackingBasePointer() + - Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize()) < (1ULL << 39)) { - // Systems like FreeBSD allocate memory really low by default, and since we pack our page table entries, - // we have to manually sign extend when our actual pointer is negative. - config.page_table_sign_extension = Common::PageTable::SIGN_BIT; - } } // Multi-process state @@ -453,7 +447,6 @@ void ArmDynarmic64::SignalInterrupt(Kernel::KThread* thread) { } void ArmDynarmic64::ClearInstructionCache() { - m_cb->last_code_addr = u64(-1); m_jit->ClearCache(); } diff --git a/src/core/arm/nce/interpreter_visitor.cpp b/src/core/arm/nce/interpreter_visitor.cpp index 2ccc176323..11b759edaf 100644 --- a/src/core/arm/nce/interpreter_visitor.cpp +++ b/src/core/arm/nce/interpreter_visitor.cpp @@ -5,779 +5,387 @@ // SPDX-FileCopyrightText: Copyright 2023 merryhime // SPDX-License-Identifier: GPL-2.0-or-later -#include +#include + #include "core/arm/nce/interpreter_visitor.h" namespace Core { namespace { -// Prefetch tuning parameters -[[maybe_unused]] constexpr size_t CACHE_LINE_SIZE = 64; -[[maybe_unused]] constexpr size_t PREFETCH_STRIDE = 128; // 2 cache lines ahead -[[maybe_unused]] constexpr size_t SIMD_PREFETCH_THRESHOLD = 32; // Bytes + +u64 SignExtend(u64 value, size_t bitsize, size_t regsize) { + const u64 mask = u64{1} << (bitsize - 1); + const u64 extended = ((value & ((u64{1} << bitsize) - 1)) ^ mask) - mask; + if (regsize == 64) { + return extended; + } + return static_cast(extended); +} + +u128 VectorGetElement(u128 value, size_t bitsize) { + if (bitsize >= 128) { + return value; + } + if (bitsize >= 64) { + return {value[0], 0}; + } + return {value[0] & ((u64{1} << bitsize) - 1), 0}; +} + } // namespace -template -u64 SignExtendToLong(u64 value) { - u64 mask = 1ULL << (BitSize - 1); - value &= (1ULL << BitSize) - 1; - return (value ^ mask) - mask; -} - -static u64 SignExtendToLong(u64 value, u64 bitsize) { - switch (bitsize) { - case 8: - return SignExtendToLong<8>(value); - case 16: - return SignExtendToLong<16>(value); - case 32: - return SignExtendToLong<32>(value); - default: - return value; - } -} - -template -u32 SignExtendToWord(u32 value) { - u32 mask = 1ULL << (BitSize - 1); - value &= (1ULL << BitSize) - 1; - return (value ^ mask) - mask; -} - -static u32 SignExtendToWord(u32 value, u64 bitsize) { - switch (bitsize) { - case 8: - return SignExtendToWord<8>(value); - case 16: - return SignExtendToWord<16>(value); - default: - return value; - } -} - -static u64 SignExtend(u64 value, u64 bitsize, u64 regsize) { - if (regsize == 64) { - return SignExtendToLong(value, bitsize); - } else { - return SignExtendToWord(static_cast(value), bitsize); - } -} - -static u128 VectorGetElement(u128 value, u64 bitsize) { - switch (bitsize) { - case 8: - return {value[0] & ((1ULL << 8) - 1), 0}; - case 16: - return {value[0] & ((1ULL << 16) - 1), 0}; - case 32: - return {value[0] & ((1ULL << 32) - 1), 0}; - case 64: - return {value[0], 0}; - default: - return value; - } -} - -u64 InterpreterVisitor::ExtendReg(size_t bitsize, Reg reg, Imm<3> option, u8 shift) { - ASSERT(shift <= 4); - ASSERT(bitsize == 32 || bitsize == 64); +u64 InterpreterVisitor::ExtendReg(Reg reg, Imm<3> option, u8 shift) { + const size_t len = size_t{8} << option.Bits<0, 1, size_t>(); u64 val = this->GetReg(reg); - size_t len; - u64 extended; - bool signed_extend; - - switch (option.ZeroExtend()) { - case 0b000: { // UXTB - val &= ((1ULL << 8) - 1); - len = 8; - signed_extend = false; - break; + if (len < 64) { + val &= (u64{1} << len) - 1; + if (option.Bit<2>()) { + val = SignExtend(val, len, 64); + } } - case 0b001: { // UXTH - val &= ((1ULL << 16) - 1); - len = 16; - signed_extend = false; - break; - } - case 0b010: { // UXTW - val &= ((1ULL << 32) - 1); - len = 32; - signed_extend = false; - break; - } - case 0b011: { // UXTX - len = 64; - signed_extend = false; - break; - } - case 0b100: { // SXTB - val &= ((1ULL << 8) - 1); - len = 8; - signed_extend = true; - break; - } - case 0b101: { // SXTH - val &= ((1ULL << 16) - 1); - len = 16; - signed_extend = true; - break; - } - case 0b110: { // SXTW - val &= ((1ULL << 32) - 1); - len = 32; - signed_extend = true; - break; - } - case 0b111: { // SXTX - len = 64; - signed_extend = true; - break; - } - default: - UNREACHABLE(); - } - - if (len < bitsize && signed_extend) { - extended = SignExtend(val, len, bitsize); - } else { - extended = val; - } - - return extended << shift; + return val << shift; } -u128 InterpreterVisitor::GetVec(Vec v) { - return m_fpsimd_regs[static_cast(v)]; -} - -u64 InterpreterVisitor::GetReg(Reg r) { - return m_regs[static_cast(r)]; -} - -u64 InterpreterVisitor::GetSp() { - return m_sp; -} - -u64 InterpreterVisitor::GetPc() { - return m_pc; -} - -void InterpreterVisitor::SetVec(Vec v, u128 value) { - m_fpsimd_regs[static_cast(v)] = value; -} - -void InterpreterVisitor::SetReg(Reg r, u64 value) { - m_regs[static_cast(r)] = value; -} - -void InterpreterVisitor::SetSp(u64 value) { - m_sp = value; -} - -bool InterpreterVisitor::Ordered(size_t size, bool L, bool o0, Reg Rn, Reg Rt) { - const auto memop = L ? MemOp::Load : MemOp::Store; - const size_t elsize = 8 << size; - const size_t datasize = elsize; - const size_t dbytes = datasize / 8; - - u64 address = (Rn == Reg::SP) ? this->GetSp() : this->GetReg(Rn); - switch (memop) { - case MemOp::Store: { - std::atomic_thread_fence(std::memory_order_seq_cst); - u64 value = this->GetReg(Rt); - m_memory.WriteBlock(address, &value, dbytes); - std::atomic_thread_fence(std::memory_order_seq_cst); - break; - } - case MemOp::Load: { +bool InterpreterVisitor::Ordered(size_t size, bool load, Reg Rn, Reg Rt) { + const size_t dbytes = size_t{1} << size; + const u64 address = this->GetRegSp(Rn); + if (load) { u64 value = 0; m_memory.ReadBlock(address, &value, dbytes); this->SetReg(Rt, value); std::atomic_thread_fence(std::memory_order_seq_cst); - break; - } - default: - UNREACHABLE(); + return true; } + std::atomic_thread_fence(std::memory_order_seq_cst); + u64 value = this->GetReg(Rt); + m_memory.WriteBlock(address, &value, dbytes); + std::atomic_thread_fence(std::memory_order_seq_cst); return true; } -bool InterpreterVisitor::STLLR(Imm<2> sz, Reg Rn, Reg Rt) { - const size_t size = sz.ZeroExtend(); - const bool L = 0; - const bool o0 = 0; - return this->Ordered(size, L, o0, Rn, Rt); -} - -bool InterpreterVisitor::STLR(Imm<2> sz, Reg Rn, Reg Rt) { - const size_t size = sz.ZeroExtend(); - const bool L = 0; - const bool o0 = 1; - return this->Ordered(size, L, o0, Rn, Rt); -} - -bool InterpreterVisitor::LDLAR(Imm<2> sz, Reg Rn, Reg Rt) { - const size_t size = sz.ZeroExtend(); - const bool L = 1; - const bool o0 = 0; - return this->Ordered(size, L, o0, Rn, Rt); -} - -bool InterpreterVisitor::LDAR(Imm<2> sz, Reg Rn, Reg Rt) { - const size_t size = sz.ZeroExtend(); - const bool L = 1; - const bool o0 = 1; - return this->Ordered(size, L, o0, Rn, Rt); -} - -bool InterpreterVisitor::LDR_lit_gen(bool opc_0, Imm<19> imm19, Reg Rt) { - const size_t size = opc_0 == 0 ? 4 : 8; - const s64 offset = Dynarmic::concatenate(imm19, Imm<2>{0}).SignExtend(); - const u64 address = this->GetPc() + offset; - +bool InterpreterVisitor::LoadLiteral(bool wide, Imm<19> imm19, Reg Rt) { + size_t size = 4; + if (wide) { + size = 8; + } u64 data = 0; - m_memory.ReadBlock(address, &data, size); - + m_memory.ReadBlock(m_pc + (imm19.SignExtend() << 2), &data, size); this->SetReg(Rt, data); return true; } -bool InterpreterVisitor::LDR_lit_fpsimd(Imm<2> opc, Imm<19> imm19, Vec Vt) { +bool InterpreterVisitor::LoadLiteralSimd(Imm<2> opc, Imm<19> imm19, Vec Vt) { if (opc == 0b11) { - // Unallocated encoding return false; } - - // Size in bytes - const u64 size = 4 << opc.ZeroExtend(); - const u64 offset = imm19.SignExtend() << 2; - const u64 address = this->GetPc() + offset; - u128 data{}; - m_memory.ReadBlock(address, &data, size); + m_memory.ReadBlock(m_pc + (imm19.SignExtend() << 2), &data, + size_t{4} << opc.ZeroExtend()); this->SetVec(Vt, data); return true; } -bool InterpreterVisitor::STP_LDP_gen(Imm<2> opc, bool not_postindex, bool wback, Imm<1> L, - Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt) { - if ((L == 0 && opc.Bit<0>() == 1) || opc == 0b11) { - // Unallocated encoding +bool InterpreterVisitor::Pair(Imm<2> opc, bool not_postindex, bool wback, bool load, Imm<7> imm7, + Reg Rt2, Reg Rn, Reg Rt) { + if (!not_postindex && !wback) { + return false; + } + const bool signed_ = opc.Bit<0>(); + if (opc == 0b11 || (!load && signed_)) { + return false; + } + if (load && (Rt == Rt2 || (wback && (Rt == Rn || Rt2 == Rn) && Rn != Reg::R31))) { + return false; + } + if (!load && wback && (Rt == Rn || Rt2 == Rn) && Rn != Reg::R31) { return false; } - const auto memop = L == 1 ? MemOp::Load : MemOp::Store; - if (memop == MemOp::Load && wback && (Rt == Rn || Rt2 == Rn) && Rn != Reg::R31) { - // Unpredictable instruction - return false; - } - if (memop == MemOp::Store && wback && (Rt == Rn || Rt2 == Rn) && Rn != Reg::R31) { - // Unpredictable instruction - return false; - } - if (memop == MemOp::Load && Rt == Rt2) { - // Unpredictable instruction - return false; - } - - u64 address; - if (Rn == Reg::SP) { - address = this->GetSp(); - } else { - address = this->GetReg(Rn); - } - - const bool postindex = !not_postindex; - const bool signed_ = opc.Bit<0>() != 0; const size_t scale = 2 + opc.Bit<1>(); - const size_t datasize = 8 << scale; + const size_t dbytes = size_t{1} << scale; const u64 offset = imm7.SignExtend() << scale; - - if (!postindex) { + u64 address = this->GetRegSp(Rn); + if (not_postindex) { address += offset; } - - const size_t dbytes = datasize / 8; - switch (memop) { - case MemOp::Store: { - u64 data1 = this->GetReg(Rt); - u64 data2 = this->GetReg(Rt2); - m_memory.WriteBlock(address, &data1, dbytes); - m_memory.WriteBlock(address + dbytes, &data2, dbytes); - break; - } - case MemOp::Load: { + if (load) { u64 data1 = 0, data2 = 0; m_memory.ReadBlock(address, &data1, dbytes); m_memory.ReadBlock(address + dbytes, &data2, dbytes); if (signed_) { - this->SetReg(Rt, SignExtend(data1, datasize, 64)); - this->SetReg(Rt2, SignExtend(data2, datasize, 64)); - } else { - this->SetReg(Rt, data1); - this->SetReg(Rt2, data2); + data1 = SignExtend(data1, dbytes * 8, 64); + data2 = SignExtend(data2, dbytes * 8, 64); } - break; + this->SetReg(Rt, data1); + this->SetReg(Rt2, data2); + } else { + u64 data1 = this->GetReg(Rt); + u64 data2 = this->GetReg(Rt2); + m_memory.WriteBlock(address, &data1, dbytes); + m_memory.WriteBlock(address + dbytes, &data2, dbytes); } - default: - UNREACHABLE(); - } - if (wback) { - if (postindex) { + if (!not_postindex) { address += offset; } - - if (Rn == Reg::SP) { - this->SetSp(address); - } else { - this->SetReg(Rn, address); - } + this->SetRegSp(Rn, address); } - return true; } -bool InterpreterVisitor::STP_LDP_fpsimd(Imm<2> opc, bool not_postindex, bool wback, Imm<1> L, - Imm<7> imm7, Vec Vt2, Reg Rn, Vec Vt) { - if (opc == 0b11) { - // Unallocated encoding +bool InterpreterVisitor::PairSimd(Imm<2> opc, bool not_postindex, bool wback, bool load, + Imm<7> imm7, Vec Vt2, Reg Rn, Vec Vt) { + if (!not_postindex && !wback) { + return false; + } + if (opc == 0b11 || (load && Vt == Vt2)) { return false; } - const auto memop = L == 1 ? MemOp::Load : MemOp::Store; - if (memop == MemOp::Load && Vt == Vt2) { - // Unpredictable instruction - return false; - } - - u64 address; - if (Rn == Reg::SP) { - address = this->GetSp(); - } else { - address = this->GetReg(Rn); - } - - const bool postindex = !not_postindex; const size_t scale = 2 + opc.ZeroExtend(); - const size_t datasize = 8 << scale; + const size_t dbytes = size_t{1} << scale; const u64 offset = imm7.SignExtend() << scale; - const size_t dbytes = datasize / 8; - - if (!postindex) { + u64 address = this->GetRegSp(Rn); + if (not_postindex) { address += offset; } - - switch (memop) { - case MemOp::Store: { - u128 data1 = VectorGetElement(this->GetVec(Vt), datasize); - u128 data2 = VectorGetElement(this->GetVec(Vt2), datasize); - m_memory.WriteBlock(address, &data1, dbytes); - m_memory.WriteBlock(address + dbytes, &data2, dbytes); - break; - } - case MemOp::Load: { + if (load) { u128 data1{}, data2{}; m_memory.ReadBlock(address, &data1, dbytes); m_memory.ReadBlock(address + dbytes, &data2, dbytes); this->SetVec(Vt, data1); this->SetVec(Vt2, data2); - break; + } else { + u128 data1 = VectorGetElement(this->GetVec(Vt), dbytes * 8); + u128 data2 = VectorGetElement(this->GetVec(Vt2), dbytes * 8); + m_memory.WriteBlock(address, &data1, dbytes); + m_memory.WriteBlock(address + dbytes, &data2, dbytes); } - default: - UNREACHABLE(); + if (wback) { + if (!not_postindex) { + address += offset; + } + this->SetRegSp(Rn, address); + } + return true; +} + +bool InterpreterVisitor::RegisterImmediate(bool wback, bool postindex, u64 offset, Imm<2> size, + Imm<2> opc, Reg Rn, Reg Rt) { + const bool signed_ = opc.Bit<1>(); + if (signed_ && (size == 0b11 || (size == 0b10 && opc.Bit<0>()))) { + return false; } + const size_t datasize = size_t{8} << size.ZeroExtend(); + size_t regsize = 32; + if (size == 0b11 || (signed_ && !opc.Bit<0>())) { + regsize = 64; + } + u64 address = this->GetRegSp(Rn); + if (!postindex) { + address += offset; + } + if (signed_ || opc.Bit<0>()) { + u64 data = 0; + m_memory.ReadBlock(address, &data, datasize / 8); + if (signed_) { + data = SignExtend(data, datasize, regsize); + } + this->SetReg(Rt, data); + } else { + u64 data = this->GetReg(Rt); + m_memory.WriteBlock(address, &data, datasize / 8); + } if (wback) { if (postindex) { address += offset; } - - if (Rn == Reg::SP) { - this->SetSp(address); - } else { - this->SetReg(Rn, address); - } + this->SetRegSp(Rn, address); } - return true; } -bool InterpreterVisitor::RegisterImmediate(bool wback, bool postindex, size_t scale, u64 offset, - Imm<2> size, Imm<2> opc, Reg Rn, Reg Rt) { - MemOp memop; - bool signed_ = false; - size_t regsize = 0; - const size_t datasize = 8 << scale; - - if (opc.Bit<1>() == 0) { - memop = opc.Bit<0>() ? MemOp::Load : MemOp::Store; - regsize = size == 0b11 ? 64 : 32; - } else if (size == 0b11) { - memop = MemOp::Prefetch; - ASSERT(!opc.Bit<0>()); - } else { - memop = MemOp::Load; - ASSERT(!(size == 0b10 && opc.Bit<0>() == 1)); - regsize = opc.Bit<0>() ? 32 : 64; - signed_ = true; +bool InterpreterVisitor::RegisterOffset(bool S, Imm<2> size, Imm<1> opc_1, Imm<1> opc_0, Reg Rm, + Imm<3> option, Reg Rn, Reg Rt) { + if (!option.Bit<1>()) { + return false; + } + const bool high = opc_1 == 1; + const bool wide = size == 0b11; + if (high && opc_0 == 1 && (wide || size == 0b10)) { + return false; + } + if (high && wide) { + return true; } - u64 address = (Rn == Reg::SP) ? this->GetSp() : this->GetReg(Rn); - if (!postindex) - address += offset; - - switch (memop) { - case MemOp::Store: { - u64 data = this->GetReg(Rt); - m_memory.WriteBlock(address, &data, datasize / 8); - break; + const size_t scale = size.ZeroExtend(); + const size_t datasize = size_t{8} << scale; + size_t regsize = 32; + if (wide || (high && opc_0 == 0)) { + regsize = 64; } - case MemOp::Load: { + u8 shift = 0; + if (S) { + shift = static_cast(scale); + } + const u64 address = this->GetRegSp(Rn) + this->ExtendReg(Rm, option, shift); + if (high || opc_0 == 1) { u64 data = 0; m_memory.ReadBlock(address, &data, datasize / 8); - if (signed_) { - this->SetReg(Rt, SignExtend(data, datasize, regsize)); - } else { - this->SetReg(Rt, data); + if (high) { + data = SignExtend(data, datasize, regsize); } - break; - } - case MemOp::Prefetch: - break; - } - - if (wback) { - if (postindex) - address += offset; - if (Rn == Reg::SP) - this->SetSp(address); - else - this->SetReg(Rn, address); + this->SetReg(Rt, data); + return true; } + u64 data = this->GetReg(Rt); + m_memory.WriteBlock(address, &data, datasize / 8); return true; } -bool InterpreterVisitor::STRx_LDRx_imm_1(Imm<2> size, Imm<2> opc, Imm<9> imm9, bool not_postindex, - Reg Rn, Reg Rt) { - const bool wback = true; - const bool postindex = !not_postindex; - const size_t scale = size.ZeroExtend(); - const u64 offset = imm9.SignExtend(); +bool InterpreterVisitor::SimdImmediate(bool wback, bool postindex, size_t scale, u64 offset, + bool load, Reg Rn, Vec Vt) { + if (scale > 4) { + return false; + } - return this->RegisterImmediate(wback, postindex, scale, offset, size, opc, Rn, Rt); -} - -bool InterpreterVisitor::STRx_LDRx_imm_2(Imm<2> size, Imm<2> opc, Imm<12> imm12, Reg Rn, Reg Rt) { - const bool wback = false; - const bool postindex = false; - const size_t scale = size.ZeroExtend(); - const u64 offset = imm12.ZeroExtend() << scale; - - return this->RegisterImmediate(wback, postindex, scale, offset, size, opc, Rn, Rt); -} - -bool InterpreterVisitor::STURx_LDURx(Imm<2> size, Imm<2> opc, Imm<9> imm9, Reg Rn, Reg Rt) { - const bool wback = false; - const bool postindex = false; - const size_t scale = size.ZeroExtend(); - const u64 offset = imm9.SignExtend(); - - return this->RegisterImmediate(wback, postindex, scale, offset, size, opc, Rn, Rt); -} - -bool InterpreterVisitor::SIMDImmediate(bool wback, bool postindex, size_t scale, u64 offset, - MemOp memop, Reg Rn, Vec Vt) { - const size_t datasize = 8 << scale; - u64 address = (Rn == Reg::SP) ? this->GetSp() : this->GetReg(Rn); - if (!postindex) + const size_t dbytes = size_t{1} << scale; + u64 address = this->GetRegSp(Rn); + if (!postindex) { address += offset; - - switch (memop) { - case MemOp::Store: { - u128 data = VectorGetElement(this->GetVec(Vt), datasize); - m_memory.WriteBlock(address, &data, datasize / 8); - break; } - case MemOp::Load: { + if (load) { u128 data{}; - m_memory.ReadBlock(address, &data, datasize / 8); + m_memory.ReadBlock(address, &data, dbytes); this->SetVec(Vt, data); - break; + } else { + u128 data = VectorGetElement(this->GetVec(Vt), dbytes * 8); + m_memory.WriteBlock(address, &data, dbytes); } - default: - UNREACHABLE(); - } - if (wback) { - if (postindex) + if (postindex) { address += offset; - if (Rn == Reg::SP) - this->SetSp(address); - else - this->SetReg(Rn, address); + } + this->SetRegSp(Rn, address); } return true; } -bool InterpreterVisitor::STR_imm_fpsimd_1(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, - bool not_postindex, Reg Rn, Vec Vt) { - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding +bool InterpreterVisitor::SimdOffset(size_t scale, bool S, bool load, Reg Rm, Imm<3> option, Reg Rn, + Vec Vt) { + if (scale > 4 || !option.Bit<1>()) { return false; } - const bool wback = true; - const bool postindex = !not_postindex; - const u64 offset = imm9.SignExtend(); - - return this->SIMDImmediate(wback, postindex, scale, offset, MemOp::Store, Rn, Vt); -} - -bool InterpreterVisitor::STR_imm_fpsimd_2(Imm<2> size, Imm<1> opc_1, Imm<12> imm12, Reg Rn, - Vec Vt) { - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; + const size_t dbytes = size_t{1} << scale; + u8 shift = 0; + if (S) { + shift = static_cast(scale); } - - const bool wback = false; - const bool postindex = false; - const u64 offset = imm12.ZeroExtend() << scale; - - return this->SIMDImmediate(wback, postindex, scale, offset, MemOp::Store, Rn, Vt); -} - -bool InterpreterVisitor::LDR_imm_fpsimd_1(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, - bool not_postindex, Reg Rn, Vec Vt) { - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; - } - - const bool wback = true; - const bool postindex = !not_postindex; - const u64 offset = imm9.SignExtend(); - - return this->SIMDImmediate(wback, postindex, scale, offset, MemOp::Load, Rn, Vt); -} - -bool InterpreterVisitor::LDR_imm_fpsimd_2(Imm<2> size, Imm<1> opc_1, Imm<12> imm12, Reg Rn, - Vec Vt) { - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; - } - - const bool wback = false; - const bool postindex = false; - const u64 offset = imm12.ZeroExtend() << scale; - - return this->SIMDImmediate(wback, postindex, scale, offset, MemOp::Load, Rn, Vt); -} - -bool InterpreterVisitor::STUR_fpsimd(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, Reg Rn, Vec Vt) { - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; - } - - const bool wback = false; - const bool postindex = false; - const u64 offset = imm9.SignExtend(); - - return this->SIMDImmediate(wback, postindex, scale, offset, MemOp::Store, Rn, Vt); -} - -bool InterpreterVisitor::LDUR_fpsimd(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, Reg Rn, Vec Vt) { - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; - } - - const bool wback = false; - const bool postindex = false; - const u64 offset = imm9.SignExtend(); - - return this->SIMDImmediate(wback, postindex, scale, offset, MemOp::Load, Rn, Vt); -} - -bool InterpreterVisitor::RegisterOffset(size_t scale, u8 shift, Imm<2> size, Imm<1> opc_1, - Imm<1> opc_0, Reg Rm, Imm<3> option, Reg Rn, Reg Rt) { - MemOp memop; - size_t regsize = 64; - bool signed_ = false; - - if (opc_1 == 0) { - memop = opc_0 == 1 ? MemOp::Load : MemOp::Store; - regsize = size == 0b11 ? 64 : 32; - signed_ = false; - } else if (size == 0b11) { - memop = MemOp::Prefetch; - if (opc_0 == 1) { - // Unallocated encoding - return false; - } - } else { - memop = MemOp::Load; - if (size == 0b10 && opc_0 == 1) { - // Unallocated encoding - return false; - } - regsize = opc_0 == 1 ? 32 : 64; - signed_ = true; - } - - const size_t datasize = 8 << scale; - - // Operation - const u64 offset = this->ExtendReg(64, Rm, option, shift); - - u64 address; - if (Rn == Reg::SP) { - address = this->GetSp(); - } else { - address = this->GetReg(Rn); - } - address += offset; - - switch (memop) { - case MemOp::Store: { - u64 data = this->GetReg(Rt); - m_memory.WriteBlock(address, &data, datasize / 8); - break; - } - case MemOp::Load: { - u64 data = 0; - m_memory.ReadBlock(address, &data, datasize / 8); - if (signed_) { - this->SetReg(Rt, SignExtend(data, datasize, regsize)); - } else { - this->SetReg(Rt, data); - } - break; - } - case MemOp::Prefetch: - break; - } - - return true; -} - -bool InterpreterVisitor::STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Reg Rt) { - const Imm<1> opc_0{0}; - const size_t scale = size.ZeroExtend(); - const u8 shift = S ? static_cast(scale) : 0; - if (!option.Bit<1>()) { - // Unallocated encoding - return false; - } - return this->RegisterOffset(scale, shift, size, opc_1, opc_0, Rm, option, Rn, Rt); -} - -bool InterpreterVisitor::LDRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Reg Rt) { - const Imm<1> opc_0{1}; - const size_t scale = size.ZeroExtend(); - const u8 shift = S ? static_cast(scale) : 0; - if (!option.Bit<1>()) { - // Unallocated encoding - return false; - } - return this->RegisterOffset(scale, shift, size, opc_1, opc_0, Rm, option, Rn, Rt); -} - -bool InterpreterVisitor::SIMDOffset(size_t scale, u8 shift, Imm<1> opc_0, Reg Rm, Imm<3> option, - Reg Rn, Vec Vt) { - const auto memop = opc_0 == 1 ? MemOp::Load : MemOp::Store; - const size_t datasize = 8 << scale; - - // Operation - const u64 offset = this->ExtendReg(64, Rm, option, shift); - - u64 address; - if (Rn == Reg::SP) { - address = this->GetSp(); - } else { - address = this->GetReg(Rn); - } - address += offset; - - switch (memop) { - case MemOp::Store: { - u128 data = VectorGetElement(this->GetVec(Vt), datasize); - m_memory.WriteBlock(address, &data, datasize / 8); - break; - } - case MemOp::Load: { + const u64 address = this->GetRegSp(Rn) + this->ExtendReg(Rm, option, shift); + if (load) { u128 data{}; - m_memory.ReadBlock(address, &data, datasize / 8); + m_memory.ReadBlock(address, &data, dbytes); this->SetVec(Vt, data); - break; + return true; } - default: - UNREACHABLE(); - } - + u128 data = VectorGetElement(this->GetVec(Vt), dbytes * 8); + m_memory.WriteBlock(address, &data, dbytes); return true; } -bool InterpreterVisitor::STR_reg_fpsimd(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, - Reg Rn, Vec Vt) { - const Imm<1> opc_0{0}; - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; +bool InterpreterVisitor::Execute(u32 inst) { + const Imm<2> size{inst >> 30}; + const Imm<2> opc{(inst >> 22) & 3}; + const Imm<3> option{(inst >> 13) & 7}; + const Imm<19> imm19{(inst >> 5) & 0x7FFFF}; + const Reg Rn = static_cast((inst >> 5) & 31); + const Reg Rt = static_cast(inst & 31); + const Vec Vt = static_cast(inst & 31); + const bool load = (inst & (1U << 22)) != 0; + const bool not_postindex = (inst & (1U << 11)) != 0; + const bool scaled = (inst & (1U << 12)) != 0; + const size_t vscale = ((inst >> 21) & 4) | (inst >> 30); + const u64 imm9 = Imm<9>{(inst >> 12) & 0x1FF}.SignExtend(); + + switch ((inst >> 24) & 0x3F) { + case 0b001000: + if ((inst & 0x00BF7C00) != 0x009F7C00) { + return false; + } + return this->Ordered(size.ZeroExtend(), load, Rn, Rt); + case 0b011000: + if ((inst & 0x80000000) != 0) { + return false; + } + return this->LoadLiteral((inst & 0x40000000) != 0, imm19, Rt); + case 0b011100: + return this->LoadLiteralSimd(size, imm19, Vt); + case 0b101000: + case 0b101001: + return this->Pair(size, (inst & (1U << 24)) != 0, (inst & (1U << 23)) != 0, load, + Imm<7>{(inst >> 15) & 0x7F}, static_cast((inst >> 10) & 31), Rn, Rt); + case 0b101100: + case 0b101101: + return this->PairSimd(size, (inst & (1U << 24)) != 0, (inst & (1U << 23)) != 0, load, + Imm<7>{(inst >> 15) & 0x7F}, static_cast((inst >> 10) & 31), Rn, + Vt); + case 0b111000: + if ((inst & (1U << 21)) != 0) { + if ((inst & 0x00000C00) != 0x00000800) { + return false; + } + return this->RegisterOffset(scaled, size, Imm<1>{(inst >> 23) & 1}, + Imm<1>{(inst >> 22) & 1}, + static_cast((inst >> 16) & 31), option, Rn, Rt); + } + if ((inst & (1U << 10)) != 0) { + return this->RegisterImmediate(true, !not_postindex, imm9, size, opc, Rn, Rt); + } + if (not_postindex) { + return false; + } + return this->RegisterImmediate(false, false, imm9, size, opc, Rn, Rt); + case 0b111001: + return this->RegisterImmediate(false, false, + u64{(inst >> 10) & 0xFFF} << size.ZeroExtend(), size, + opc, Rn, Rt); + case 0b111100: + if ((inst & (1U << 21)) != 0) { + if ((inst & 0x00000C00) != 0x00000800) { + return false; + } + return this->SimdOffset(vscale, scaled, load, static_cast((inst >> 16) & 31), + option, Rn, Vt); + } + if ((inst & (1U << 10)) != 0) { + return this->SimdImmediate(true, !not_postindex, vscale, imm9, load, Rn, Vt); + } + if (not_postindex) { + return false; + } + return this->SimdImmediate(false, false, vscale, imm9, load, Rn, Vt); + case 0b111101: + return this->SimdImmediate(false, false, vscale, u64{(inst >> 10) & 0xFFF} << vscale, load, + Rn, Vt); } - const u8 shift = S ? static_cast(scale) : 0; - if (!option.Bit<1>()) { - // Unallocated encoding - return false; - } - return this->SIMDOffset(scale, shift, opc_0, Rm, option, Rn, Vt); + return false; } -bool InterpreterVisitor::LDR_reg_fpsimd(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, - Reg Rn, Vec Vt) { - const Imm<1> opc_0{1}; - const size_t scale = Dynarmic::concatenate(opc_1, size).ZeroExtend(); - if (scale > 4) { - // Unallocated encoding - return false; - } - const u8 shift = S ? static_cast(scale) : 0; - if (!option.Bit<1>()) { - // Unallocated encoding - return false; - } - return this->SIMDOffset(scale, shift, opc_0, Rm, option, Rn, Vt); -} - -std::optional MatchAndExecuteOneInstruction(Core::Memory::Memory& memory, mcontext_t* context, fpsimd_context* fpsimd_context) { +std::optional MatchAndExecuteOneInstruction(Core::Memory::Memory& memory, mcontext_t* context, + fpsimd_context* fpsimd_context) { std::span regs(reinterpret_cast(context->regs), 31); std::span vregs(reinterpret_cast(fpsimd_context->vregs), 32); u64& sp = *reinterpret_cast(&context->sp); const u64& pc = *reinterpret_cast(&context->pc); InterpreterVisitor visitor(memory, regs, vregs, sp, pc); - u32 instruction = memory.Read32(pc); - bool was_executed = false; - - auto decoder = Dynarmic::A64::Decode(visitor, instruction); - if (decoder) { - was_executed = *decoder; - } else { - was_executed = false; + if (!visitor.Execute(memory.Read32(pc))) { + return std::nullopt; } - return was_executed ? std::optional(pc + 4) : std::nullopt; + return pc + 4; } } // namespace Core diff --git a/src/core/arm/nce/interpreter_visitor.h b/src/core/arm/nce/interpreter_visitor.h index 131b69cd7d..667ea24ac8 100644 --- a/src/core/arm/nce/interpreter_visitor.h +++ b/src/core/arm/nce/interpreter_visitor.h @@ -7,97 +7,77 @@ #pragma once -#include -#include -#include +#include #include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wshadow" +#include +#include +#pragma GCC diagnostic pop -#include "core/hle/kernel/k_thread.h" #include "core/memory.h" -#include "common/logging.h" -#include "core/arm/nce/visitor_base.h" namespace Core { -namespace Memory { -class Memory; -} - -class InterpreterVisitor final : public VisitorBase { +class InterpreterVisitor { public: explicit InterpreterVisitor(Core::Memory::Memory& memory, std::span regs, std::span fpsimd_regs, u64& sp, const u64& pc) : m_memory(memory), m_regs(regs), m_fpsimd_regs(fpsimd_regs), m_sp(sp), m_pc(pc) {} - ~InterpreterVisitor() override = default; - enum class MemOp { - Load, - Store, - Prefetch, - }; - - u128 GetVec(Vec v); - u64 GetReg(Reg r); - u64 GetSp(); - u64 GetPc(); - - void SetVec(Vec v, u128 value); - void SetReg(Reg r, u64 value); - void SetSp(u64 value); - - u64 ExtendReg(size_t bitsize, Reg reg, Imm<3> option, u8 shift); - - // Loads and stores - Load/Store Exclusive - bool Ordered(size_t size, bool L, bool o0, Reg Rn, Reg Rt); - bool STLLR(Imm<2> size, Reg Rn, Reg Rt) override; - bool STLR(Imm<2> size, Reg Rn, Reg Rt) override; - bool LDLAR(Imm<2> size, Reg Rn, Reg Rt) override; - bool LDAR(Imm<2> size, Reg Rn, Reg Rt) override; - - // Loads and stores - Load register (literal) - bool LDR_lit_gen(bool opc_0, Imm<19> imm19, Reg Rt) override; - bool LDR_lit_fpsimd(Imm<2> opc, Imm<19> imm19, Vec Vt) override; - - // Loads and stores - Load/Store register pair - bool STP_LDP_gen(Imm<2> opc, bool not_postindex, bool wback, Imm<1> L, Imm<7> imm7, Reg Rt2, - Reg Rn, Reg Rt) override; - bool STP_LDP_fpsimd(Imm<2> opc, bool not_postindex, bool wback, Imm<1> L, Imm<7> imm7, Vec Vt2, - Reg Rn, Vec Vt) override; - - // Loads and stores - Load/Store register (immediate) - bool RegisterImmediate(bool wback, bool postindex, size_t scale, u64 offset, Imm<2> size, - Imm<2> opc, Reg Rn, Reg Rt); - bool STRx_LDRx_imm_1(Imm<2> size, Imm<2> opc, Imm<9> imm9, bool not_postindex, Reg Rn, - Reg Rt) override; - bool STRx_LDRx_imm_2(Imm<2> size, Imm<2> opc, Imm<12> imm12, Reg Rn, Reg Rt) override; - bool STURx_LDURx(Imm<2> size, Imm<2> opc, Imm<9> imm9, Reg Rn, Reg Rt) override; - - bool SIMDImmediate(bool wback, bool postindex, size_t scale, u64 offset, MemOp memop, Reg Rn, - Vec Vt); - bool STR_imm_fpsimd_1(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, bool not_postindex, Reg Rn, - Vec Vt) override; - bool STR_imm_fpsimd_2(Imm<2> size, Imm<1> opc_1, Imm<12> imm12, Reg Rn, Vec Vt) override; - bool LDR_imm_fpsimd_1(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, bool not_postindex, Reg Rn, - Vec Vt) override; - bool LDR_imm_fpsimd_2(Imm<2> size, Imm<1> opc_1, Imm<12> imm12, Reg Rn, Vec Vt) override; - bool STUR_fpsimd(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, Reg Rn, Vec Vt) override; - bool LDUR_fpsimd(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, Reg Rn, Vec Vt) override; - - // Loads and stores - Load/Store register (register offset) - bool RegisterOffset(size_t scale, u8 shift, Imm<2> size, Imm<1> opc_1, Imm<1> opc_0, Reg Rm, - Imm<3> option, Reg Rn, Reg Rt); - bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Reg Rt) override; - bool LDRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Reg Rt) override; - - bool SIMDOffset(size_t scale, u8 shift, Imm<1> opc_0, Reg Rm, Imm<3> option, Reg Rn, Vec Vt); - bool STR_reg_fpsimd(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Vec Vt) override; - bool LDR_reg_fpsimd(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Vec Vt) override; + bool Execute(u32 inst); private: + template + using Imm = Dynarmic::Imm; + using Reg = Dynarmic::A64::Reg; + using Vec = Dynarmic::A64::Vec; + + u128 GetVec(Vec v) const { + return m_fpsimd_regs[static_cast(v)]; + } + void SetVec(Vec v, u128 value) { + m_fpsimd_regs[static_cast(v)] = value; + } + u64 GetReg(Reg r) const { + return m_regs[static_cast(r)]; + } + void SetReg(Reg r, u64 value) { + m_regs[static_cast(r)] = value; + } + u64 GetRegSp(Reg r) const { + if (r == Reg::SP) { + return m_sp; + } + return m_regs[static_cast(r)]; + } + void SetRegSp(Reg r, u64 value) { + if (r == Reg::SP) { + m_sp = value; + return; + } + m_regs[static_cast(r)] = value; + } + + u64 ExtendReg(Reg reg, Imm<3> option, u8 shift); + + bool Ordered(size_t size, bool load, Reg Rn, Reg Rt); + bool LoadLiteral(bool wide, Imm<19> imm19, Reg Rt); + bool LoadLiteralSimd(Imm<2> opc, Imm<19> imm19, Vec Vt); + bool Pair(Imm<2> opc, bool not_postindex, bool wback, bool load, Imm<7> imm7, Reg Rt2, Reg Rn, + Reg Rt); + bool PairSimd(Imm<2> opc, bool not_postindex, bool wback, bool load, Imm<7> imm7, Vec Vt2, + Reg Rn, Vec Vt); + bool RegisterImmediate(bool wback, bool postindex, u64 offset, Imm<2> size, Imm<2> opc, Reg Rn, + Reg Rt); + bool RegisterOffset(bool S, Imm<2> size, Imm<1> opc_1, Imm<1> opc_0, Reg Rm, Imm<3> option, + Reg Rn, Reg Rt); + bool SimdImmediate(bool wback, bool postindex, size_t scale, u64 offset, bool load, Reg Rn, + Vec Vt); + bool SimdOffset(size_t scale, bool S, bool load, Reg Rm, Imm<3> option, Reg Rn, Vec Vt); + Core::Memory::Memory& m_memory; std::span m_regs; std::span m_fpsimd_regs; diff --git a/src/core/arm/nce/patcher.cpp b/src/core/arm/nce/patcher.cpp index 7d2041d4d2..b1569ceee6 100644 --- a/src/core/arm/nce/patcher.cpp +++ b/src/core/arm/nce/patcher.cpp @@ -371,14 +371,9 @@ size_t Patcher::GetPreSectionSize() const noexcept { } void Patcher::WriteLoadContext(oaknut::VectorCodeGenerator& cg) { - // This function was called, which modifies X30, so use that as a scratch register. - // SP contains the guest X30, so save our return X30 to SP + 8, since we have allocated 16 bytes - // of stack. cg.STR(X30, SP, 8); - cg.MRS(X30, oaknut::SystemReg::TPIDR_EL0); - cg.LDR(X30, X30, offsetof(NativeExecutionParameters, native_context)); + cg.LDR(X30, SP, 0); - // Load system registers. cg.LDR(W0, X30, offsetof(GuestContext, fpsr)); cg.MSR(oaknut::SystemReg::FPSR, X0); cg.LDR(W0, X30, offsetof(GuestContext, fpcr)); @@ -386,7 +381,9 @@ void Patcher::WriteLoadContext(oaknut::VectorCodeGenerator& cg) { cg.LDR(W0, X30, offsetof(GuestContext, nzcv)); cg.MSR(oaknut::SystemReg::NZCV, X0); - // Load all vector registers. + cg.LDR(X0, X30, 8 * 30); + cg.STR(X0, SP, 0); + static constexpr size_t VEC_OFF = offsetof(GuestContext, vector_registers); for (int i = 0; i <= 30; i += 2) { cg.LDP(oaknut::QReg{i}, oaknut::QReg{i + 1}, X30, VEC_OFF + 16 * i); @@ -436,7 +433,7 @@ void Patcher::WriteSaveContext(oaknut::VectorCodeGenerator& cg) { cg.STR(W0, X30, offsetof(GuestContext, nzcv)); cg.LDR(X0, SP, POST_INDEXED, 16); - // Reload our return X30 from the stack, and return. + cg.MOV(X1, X30); cg.LDR(X30, SP, 8); cg.RET(); } @@ -457,8 +454,6 @@ void Patcher::WriteSvcTrampoline(ModuleDestLabel module_dest, u32 svc_id, oaknut // Now that we've saved all registers, we can use any registers as scratch. // Store PC + 4 to arm interface, since we know the instruction offset from the entry point. oaknut::Label pc_after_svc; - cg.MRS(X1, oaknut::SystemReg::TPIDR_EL0); - cg.LDR(X1, X1, offsetof(NativeExecutionParameters, native_context)); cg.LDR(X2, pc_after_svc); cg.STR(X2, X1, offsetof(GuestContext, pc)); @@ -514,24 +509,13 @@ void Patcher::WriteSvcTrampoline(ModuleDestLabel module_dest, u32 svc_id, oaknut // Host called this location. Save the return address so we can // unwind the stack properly when jumping back. - cg.MRS(X2, oaknut::SystemReg::TPIDR_EL0); - cg.LDR(X2, X2, offsetof(NativeExecutionParameters, native_context)); - cg.ADD(X0, X2, offsetof(GuestContext, host_ctx)); + cg.ADD(X0, X1, offsetof(GuestContext, host_ctx)); cg.STR(X30, X0, offsetof(HostContext, host_saved_regs) + 11 * sizeof(u64)); - // Reload all guest registers except X30 and PC. - // The function also expects 16 bytes of stack already allocated. - cg.STR(X30, SP, PRE_INDEXED, -16); + cg.STR(X1, SP, PRE_INDEXED, -16); cg.BL(load_ctx); cg.LDR(X30, SP, POST_INDEXED, 16); - // Use X1 as a scratch register to restore X30. - cg.STR(X1, SP, PRE_INDEXED, -16); - cg.MRS(X1, oaknut::SystemReg::TPIDR_EL0); - cg.LDR(X1, X1, offsetof(NativeExecutionParameters, native_context)); - cg.LDR(X30, X1, offsetof(GuestContext, cpu_registers) + sizeof(u64) * 30); - cg.LDR(X1, SP, POST_INDEXED, 16); - // Unlock the context. this->UnlockContext(cg); diff --git a/src/core/arm/nce/visitor_base.h b/src/core/arm/nce/visitor_base.h deleted file mode 100644 index 6a2be3d9bc..0000000000 --- a/src/core/arm/nce/visitor_base.h +++ /dev/null @@ -1,2783 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project -// SPDX-FileCopyrightText: Copyright 2023 merryhime -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wshadow" - -#include -#include -#include - -#pragma GCC diagnostic pop - -namespace Core { - -class VisitorBase { -public: - using instruction_return_type = bool; - - template - using Imm = Dynarmic::Imm; - using Reg = Dynarmic::A64::Reg; - using Vec = Dynarmic::A64::Vec; - using Cond = Dynarmic::A64::Cond; - - virtual ~VisitorBase() {} - - virtual bool UnallocatedEncoding() { - return false; - } - - // Data processing - Immediate - PC relative addressing - virtual bool ADR(Imm<2> immlo, Imm<19> immhi, Reg Rd) { - return false; - } - virtual bool ADRP(Imm<2> immlo, Imm<19> immhi, Reg Rd) { - return false; - } - - // Data processing - Immediate - Add/Sub (with tag) - virtual bool ADDG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUBG(Imm<6> offset_imm, Imm<4> tag_offset, Reg Rn, Reg Rd) { - return false; - } - - // Data processing - Immediate - Add/Sub - virtual bool ADD_imm(bool sf, Imm<2> shift, Imm<12> imm12, Reg Rn, Reg Rd) { - return false; - } - virtual bool ADDS_imm(bool sf, Imm<2> shift, Imm<12> imm12, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUB_imm(bool sf, Imm<2> shift, Imm<12> imm12, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUBS_imm(bool sf, Imm<2> shift, Imm<12> imm12, Reg Rn, Reg Rd) { - return false; - } - - // Data processing - Immediate - Logical - virtual bool AND_imm(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - virtual bool ORR_imm(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - virtual bool EOR_imm(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - virtual bool ANDS_imm(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - - // Data processing - Immediate - Move Wide - virtual bool MOVN(bool sf, Imm<2> hw, Imm<16> imm16, Reg Rd) { - return false; - } - virtual bool MOVZ(bool sf, Imm<2> hw, Imm<16> imm16, Reg Rd) { - return false; - } - virtual bool MOVK(bool sf, Imm<2> hw, Imm<16> imm16, Reg Rd) { - return false; - } - - // Data processing - Immediate - Bitfield - virtual bool SBFM(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - virtual bool BFM(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - virtual bool UBFM(bool sf, bool N, Imm<6> immr, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - virtual bool ASR_1(Imm<5> immr, Reg Rn, Reg Rd) { - return false; - } - virtual bool ASR_2(Imm<6> immr, Reg Rn, Reg Rd) { - return false; - } - virtual bool SXTB_1(Reg Rn, Reg Rd) { - return false; - } - virtual bool SXTB_2(Reg Rn, Reg Rd) { - return false; - } - virtual bool SXTH_1(Reg Rn, Reg Rd) { - return false; - } - virtual bool SXTH_2(Reg Rn, Reg Rd) { - return false; - } - virtual bool SXTW(Reg Rn, Reg Rd) { - return false; - } - - // Data processing - Immediate - Extract - virtual bool EXTR(bool sf, bool N, Reg Rm, Imm<6> imms, Reg Rn, Reg Rd) { - return false; - } - - // Conditional branch - virtual bool B_cond(Imm<19> imm19, Cond cond) { - return false; - } - - // Exception generation - virtual bool SVC(Imm<16> imm16) { - return false; - } - virtual bool HVC(Imm<16> imm16) { - return false; - } - virtual bool SMC(Imm<16> imm16) { - return false; - } - virtual bool BRK(Imm<16> imm16) { - return false; - } - virtual bool HLT(Imm<16> imm16) { - return false; - } - virtual bool DCPS1(Imm<16> imm16) { - return false; - } - virtual bool DCPS2(Imm<16> imm16) { - return false; - } - virtual bool DCPS3(Imm<16> imm16) { - return false; - } - - // System - virtual bool MSR_imm(Imm<3> op1, Imm<4> CRm, Imm<3> op2) { - return false; - } - virtual bool HINT(Imm<4> CRm, Imm<3> op2) { - return false; - } - virtual bool NOP() { - return false; - } - virtual bool YIELD() { - return false; - } - virtual bool WFE() { - return false; - } - virtual bool WFI() { - return false; - } - virtual bool SEV() { - return false; - } - virtual bool SEVL() { - return false; - } - virtual bool XPAC_1(bool D, Reg Rd) { - return false; - } - virtual bool XPAC_2() { - return false; - } - virtual bool PACIA_1(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool PACIA_2() { - return false; - } - virtual bool PACIB_1(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool PACIB_2() { - return false; - } - virtual bool AUTIA_1(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool AUTIA_2() { - return false; - } - virtual bool AUTIB_1(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool AUTIB_2() { - return false; - } - virtual bool BTI(Imm<2> upper_op2) { - return false; - } - virtual bool ESB() { - return false; - } - virtual bool PSB() { - return false; - } - virtual bool TSB() { - return false; - } - virtual bool CSDB() { - return false; - } - virtual bool CLREX(Imm<4> CRm) { - return false; - } - virtual bool DSB(Imm<4> CRm) { - return false; - } - virtual bool SSBB() { - return false; - } - virtual bool PSSBB() { - return false; - } - virtual bool DMB(Imm<4> CRm) { - return false; - } - virtual bool ISB(Imm<4> CRm) { - return false; - } - virtual bool SYS(Imm<3> op1, Imm<4> CRn, Imm<4> CRm, Imm<3> op2, Reg Rt) { - return false; - } - virtual bool SB() { - return false; - } - virtual bool MSR_reg(Imm<1> o0, Imm<3> op1, Imm<4> CRn, Imm<4> CRm, Imm<3> op2, Reg Rt) { - return false; - } - virtual bool SYSL(Imm<3> op1, Imm<4> CRn, Imm<4> CRm, Imm<3> op2, Reg Rt) { - return false; - } - virtual bool MRS(Imm<1> o0, Imm<3> op1, Imm<4> CRn, Imm<4> CRm, Imm<3> op2, Reg Rt) { - return false; - } - - // System - Flag manipulation instructions - virtual bool CFINV() { - return false; - } - virtual bool RMIF(Imm<6> lsb, Reg Rn, Imm<4> mask) { - return false; - } - virtual bool SETF8(Reg Rn) { - return false; - } - virtual bool SETF16(Reg Rn) { - return false; - } - - // System - Flag format instructions - virtual bool XAFlag() { - return false; - } - virtual bool AXFlag() { - return false; - } - - // SYS: Data Cache - virtual bool DC_IVAC(Reg Rt) { - return false; - } - virtual bool DC_ISW(Reg Rt) { - return false; - } - virtual bool DC_CSW(Reg Rt) { - return false; - } - virtual bool DC_CISW(Reg Rt) { - return false; - } - virtual bool DC_ZVA(Reg Rt) { - return false; - } - virtual bool DC_CVAC(Reg Rt) { - return false; - } - virtual bool DC_CVAU(Reg Rt) { - return false; - } - virtual bool DC_CVAP(Reg Rt) { - return false; - } - virtual bool DC_CIVAC(Reg Rt) { - return false; - } - - // SYS: Instruction Cache - virtual bool IC_IALLU() { - return false; - } - virtual bool IC_IALLUIS() { - return false; - } - virtual bool IC_IVAU(Reg Rt) { - return false; - } - - // Unconditional branch (Register) - virtual bool BR(Reg Rn) { - return false; - } - virtual bool BRA(bool Z, bool M, Reg Rn, Reg Rm) { - return false; - } - virtual bool BLR(Reg Rn) { - return false; - } - virtual bool BLRA(bool Z, bool M, Reg Rn, Reg Rm) { - return false; - } - virtual bool RET(Reg Rn) { - return false; - } - virtual bool RETA(bool M) { - return false; - } - virtual bool ERET() { - return false; - } - virtual bool ERETA(bool M) { - return false; - } - virtual bool DRPS() { - return false; - } - - // Unconditional branch (immediate) - virtual bool B_uncond(Imm<26> imm26) { - return false; - } - virtual bool BL(Imm<26> imm26) { - return false; - } - - // Compare and branch (immediate) - virtual bool CBZ(bool sf, Imm<19> imm19, Reg Rt) { - return false; - } - virtual bool CBNZ(bool sf, Imm<19> imm19, Reg Rt) { - return false; - } - virtual bool TBZ(Imm<1> b5, Imm<5> b40, Imm<14> imm14, Reg Rt) { - return false; - } - virtual bool TBNZ(Imm<1> b5, Imm<5> b40, Imm<14> imm14, Reg Rt) { - return false; - } - - // Loads and stores - Advanced SIMD Load/Store multiple structures - virtual bool STx_mult_1(bool Q, Imm<4> opcode, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool STx_mult_2(bool Q, Reg Rm, Imm<4> opcode, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LDx_mult_1(bool Q, Imm<4> opcode, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LDx_mult_2(bool Q, Reg Rm, Imm<4> opcode, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - - // Loads and stores - Advanced SIMD Load/Store single structures - virtual bool ST1_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool ST1_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool ST3_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool ST3_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool ST2_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool ST2_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool ST4_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool ST4_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool LD1_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD1_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool LD3_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD3_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool LD1R_1(bool Q, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD1R_2(bool Q, Reg Rm, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD3R_1(bool Q, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD3R_2(bool Q, Reg Rm, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD2_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD2_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool LD4_sngl_1(bool Q, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD4_sngl_2(bool Q, Reg Rm, Imm<2> upper_opcode, bool S, Imm<2> size, Reg Rn, - Vec Vt) { - return false; - } - virtual bool LD2R_1(bool Q, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD2R_2(bool Q, Reg Rm, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD4R_1(bool Q, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - virtual bool LD4R_2(bool Q, Reg Rm, Imm<2> size, Reg Rn, Vec Vt) { - return false; - } - - // Loads and stores - Load/Store Exclusive - virtual bool STXR(Imm<2> size, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool STLXR(Imm<2> size, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool STXP(Imm<1> size, Reg Rs, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool STLXP(Imm<1> size, Reg Rs, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDXR(Imm<2> size, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDAXR(Imm<2> size, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDXP(Imm<1> size, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDAXP(Imm<1> size, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool STLLR(Imm<2> size, Reg Rn, Reg Rt) { - return false; - } - virtual bool STLR(Imm<2> size, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDLAR(Imm<2> size, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDAR(Imm<2> size, Reg Rn, Reg Rt) { - return false; - } - virtual bool CASP(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { - return false; - } - virtual bool CASB(bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { - return false; - } - virtual bool CASH(bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { - return false; - } - virtual bool CAS(bool sz, bool L, Reg Rs, bool o0, Reg Rn, Reg Rt) { - return false; - } - - // Loads and stores - Load register (literal) - virtual bool LDR_lit_gen(bool opc_0, Imm<19> imm19, Reg Rt) { - return false; - } - virtual bool LDR_lit_fpsimd(Imm<2> opc, Imm<19> imm19, Vec Vt) { - return false; - } - virtual bool LDRSW_lit(Imm<19> imm19, Reg Rt) { - return false; - } - virtual bool PRFM_lit(Imm<19> imm19, Imm<5> prfop) { - return false; - } - - // Loads and stores - Load/Store no-allocate pair - virtual bool STNP_LDNP_gen(Imm<1> upper_opc, Imm<1> L, Imm<7> imm7, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool STNP_LDNP_fpsimd(Imm<2> opc, Imm<1> L, Imm<7> imm7, Vec Vt2, Reg Rn, Vec Vt) { - return false; - } - - // Loads and stores - Load/Store register pair - virtual bool STP_LDP_gen(Imm<2> opc, bool not_postindex, bool wback, Imm<1> L, Imm<7> imm7, - Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool STP_LDP_fpsimd(Imm<2> opc, bool not_postindex, bool wback, Imm<1> L, Imm<7> imm7, - Vec Vt2, Reg Rn, Vec Vt) { - return false; - } - virtual bool STGP_1(Imm<7> offset_imm, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool STGP_2(Imm<7> offset_imm, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - virtual bool STGP_3(Imm<7> offset_imm, Reg Rt2, Reg Rn, Reg Rt) { - return false; - } - - // Loads and stores - Load/Store register (immediate) - virtual bool STRx_LDRx_imm_1(Imm<2> size, Imm<2> opc, Imm<9> imm9, bool not_postindex, Reg Rn, - Reg Rt) { - return false; - } - virtual bool STRx_LDRx_imm_2(Imm<2> size, Imm<2> opc, Imm<12> imm12, Reg Rn, Reg Rt) { - return false; - } - virtual bool STURx_LDURx(Imm<2> size, Imm<2> opc, Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool PRFM_imm(Imm<12> imm12, Reg Rn, Reg Rt) { - return false; - } - virtual bool PRFM_unscaled_imm(Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool STR_imm_fpsimd_1(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, bool not_postindex, - Reg Rn, Vec Vt) { - return false; - } - virtual bool STR_imm_fpsimd_2(Imm<2> size, Imm<1> opc_1, Imm<12> imm12, Reg Rn, Vec Vt) { - return false; - } - virtual bool LDR_imm_fpsimd_1(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, bool not_postindex, - Reg Rn, Vec Vt) { - return false; - } - virtual bool LDR_imm_fpsimd_2(Imm<2> size, Imm<1> opc_1, Imm<12> imm12, Reg Rn, Vec Vt) { - return false; - } - virtual bool STUR_fpsimd(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, Reg Rn, Vec Vt) { - return false; - } - virtual bool LDUR_fpsimd(Imm<2> size, Imm<1> opc_1, Imm<9> imm9, Reg Rn, Vec Vt) { - return false; - } - - // Loads and stores - Load/Store register (unprivileged) - virtual bool STTRB(Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDTRB(Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDTRSB(Imm<2> opc, Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool STTRH(Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDTRH(Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDTRSH(Imm<2> opc, Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool STTR(Imm<2> size, Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDTR(Imm<2> size, Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDTRSW(Imm<9> imm9, Reg Rn, Reg Rt) { - return false; - } - - // Loads and stores - Atomic memory options - virtual bool LDADDB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDCLRB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDEORB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSETB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSMAXB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSMINB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDUMAXB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDUMINB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool SWPB(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDAPRB(Reg Rn, Reg Rt) { - return false; - } - virtual bool LDADDH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDCLRH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDEORH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSETH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSMAXH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSMINH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDUMAXH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDUMINH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool SWPH(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDAPRH(Reg Rn, Reg Rt) { - return false; - } - virtual bool LDADD(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDCLR(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDEOR(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSET(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSMAX(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDSMIN(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDUMAX(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDUMIN(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool SWP(bool A, bool R, Reg Rs, Reg Rn, Reg Rt) { - return false; - } - virtual bool LDAPR(Reg Rn, Reg Rt) { - return false; - } - - // Loads and stores - Load/Store register (register offset) - virtual bool STRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Reg Rt) { - return false; - } - virtual bool LDRx_reg(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Reg Rt) { - return false; - } - virtual bool STR_reg_fpsimd(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Vec Vt) { - return false; - } - virtual bool LDR_reg_fpsimd(Imm<2> size, Imm<1> opc_1, Reg Rm, Imm<3> option, bool S, Reg Rn, - Vec Vt) { - return false; - } - - // Loads and stores - Load/Store memory tags - virtual bool STG_1(Imm<9> imm9, Reg Rn) { - return false; - } - virtual bool STG_2(Imm<9> imm9, Reg Rn) { - return false; - } - virtual bool STG_3(Imm<9> imm9, Reg Rn) { - return false; - } - virtual bool LDG(Imm<9> offset_imm, Reg Rn, Reg Rt) { - return false; - } - virtual bool STZG_1(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool STZG_2(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool STZG_3(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool ST2G_1(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool ST2G_2(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool ST2G_3(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool STGV(Reg Rn, Reg Rt) { - return false; - } - virtual bool STZ2G_1(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool STZ2G_2(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool STZ2G_3(Imm<9> offset_imm, Reg Rn) { - return false; - } - virtual bool LDGV(Reg Rn, Reg Rt) { - return false; - } - - // Loads and stores - Load/Store register (pointer authentication) - virtual bool LDRA(bool M, bool S, Imm<9> imm9, bool W, Reg Rn, Reg Rt) { - return false; - } - - // Data Processing - Register - 2 source - virtual bool UDIV(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool SDIV(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool LSLV(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool LSRV(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool ASRV(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool RORV(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool CRC32(bool sf, Reg Rm, Imm<2> sz, Reg Rn, Reg Rd) { - return false; - } - virtual bool CRC32C(bool sf, Reg Rm, Imm<2> sz, Reg Rn, Reg Rd) { - return false; - } - virtual bool PACGA(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUBP(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool IRG(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool GMI(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUBPS(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - 1 source - virtual bool RBIT_int(bool sf, Reg Rn, Reg Rd) { - return false; - } - virtual bool REV16_int(bool sf, Reg Rn, Reg Rd) { - return false; - } - virtual bool REV(bool sf, bool opc_0, Reg Rn, Reg Rd) { - return false; - } - virtual bool CLZ_int(bool sf, Reg Rn, Reg Rd) { - return false; - } - virtual bool CLS_int(bool sf, Reg Rn, Reg Rd) { - return false; - } - virtual bool REV32_int(Reg Rn, Reg Rd) { - return false; - } - virtual bool PACDA(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool PACDB(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool AUTDA(bool Z, Reg Rn, Reg Rd) { - return false; - } - virtual bool AUTDB(bool Z, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - Logical (shifted register) - virtual bool AND_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool BIC_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool ORR_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool ORN_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool EOR_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool EON(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool ANDS_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool BICS(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - Add/Sub (shifted register) - virtual bool ADD_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool ADDS_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUB_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUBS_shift(bool sf, Imm<2> shift, Reg Rm, Imm<6> imm6, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - Add/Sub (shifted register) - virtual bool ADD_ext(bool sf, Reg Rm, Imm<3> option, Imm<3> imm3, Reg Rn, Reg Rd) { - return false; - } - virtual bool ADDS_ext(bool sf, Reg Rm, Imm<3> option, Imm<3> imm3, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUB_ext(bool sf, Reg Rm, Imm<3> option, Imm<3> imm3, Reg Rn, Reg Rd) { - return false; - } - virtual bool SUBS_ext(bool sf, Reg Rm, Imm<3> option, Imm<3> imm3, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - Add/Sub (with carry) - virtual bool ADC(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool ADCS(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool SBC(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool SBCS(bool sf, Reg Rm, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - Conditional compare - virtual bool CCMN_reg(bool sf, Reg Rm, Cond cond, Reg Rn, Imm<4> nzcv) { - return false; - } - virtual bool CCMP_reg(bool sf, Reg Rm, Cond cond, Reg Rn, Imm<4> nzcv) { - return false; - } - virtual bool CCMN_imm(bool sf, Imm<5> imm5, Cond cond, Reg Rn, Imm<4> nzcv) { - return false; - } - virtual bool CCMP_imm(bool sf, Imm<5> imm5, Cond cond, Reg Rn, Imm<4> nzcv) { - return false; - } - - // Data Processing - Register - Conditional select - virtual bool CSEL(bool sf, Reg Rm, Cond cond, Reg Rn, Reg Rd) { - return false; - } - virtual bool CSINC(bool sf, Reg Rm, Cond cond, Reg Rn, Reg Rd) { - return false; - } - virtual bool CSINV(bool sf, Reg Rm, Cond cond, Reg Rn, Reg Rd) { - return false; - } - virtual bool CSNEG(bool sf, Reg Rm, Cond cond, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - Register - 3 source - virtual bool MADD(bool sf, Reg Rm, Reg Ra, Reg Rn, Reg Rd) { - return false; - } - virtual bool MSUB(bool sf, Reg Rm, Reg Ra, Reg Rn, Reg Rd) { - return false; - } - virtual bool SMADDL(Reg Rm, Reg Ra, Reg Rn, Reg Rd) { - return false; - } - virtual bool SMSUBL(Reg Rm, Reg Ra, Reg Rn, Reg Rd) { - return false; - } - virtual bool SMULH(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - virtual bool UMADDL(Reg Rm, Reg Ra, Reg Rn, Reg Rd) { - return false; - } - virtual bool UMSUBL(Reg Rm, Reg Ra, Reg Rn, Reg Rd) { - return false; - } - virtual bool UMULH(Reg Rm, Reg Rn, Reg Rd) { - return false; - } - - // Data Processing - FP and SIMD - AES - virtual bool AESE(Vec Vn, Vec Vd) { - return false; - } - virtual bool AESD(Vec Vn, Vec Vd) { - return false; - } - virtual bool AESMC(Vec Vn, Vec Vd) { - return false; - } - virtual bool AESIMC(Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SHA - virtual bool SHA1C(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA1P(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA1M(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA1SU0(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA256H(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA256H2(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA256SU1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA1H(Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA1SU1(Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA256SU0(Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Scalar copy - virtual bool DUP_elt_1(Imm<5> imm5, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Scalar three - virtual bool FMULX_vec_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMULX_vec_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_reg_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPS_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPS_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTS_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTS_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_reg_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGE_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGE_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABD_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABD_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_reg_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_reg_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGT_1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGT_2(bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Two register misc FP16 - virtual bool FCVTNS_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMS_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAS_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool SCVTF_int_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_zero_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_zero_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLT_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPS_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZS_int_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPE_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPX_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTNU_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMU_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAU_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool UCVTF_int_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_zero_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLE_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPU_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZU_int_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTE_1(Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Two register misc - virtual bool FCVTNS_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMS_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAS_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool SCVTF_int_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_zero_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_zero_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLT_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPS_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZS_int_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPE_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPX_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTNU_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMU_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAU_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool UCVTF_int_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_zero_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLE_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPU_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZU_int_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTE_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Scalar two register misc FP16 - virtual bool FCVTNS_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMS_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAS_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool SCVTF_int_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_zero_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_zero_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLT_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPS_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZS_int_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPE_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTNU_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMU_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAU_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool UCVTF_int_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_zero_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLE_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPU_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZU_int_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTE_3(bool Q, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Scalar two register misc - virtual bool FCVTNS_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMS_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAS_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool SCVTF_int_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_zero_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_zero_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLT_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPS_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZS_int_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPE_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTNU_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTMU_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTAU_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool UCVTF_int_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_zero_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLE_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTPU_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZU_int_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTE_4(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Scalar three same extra - virtual bool SQRDMLAH_vec_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMLAH_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMLSH_vec_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMLSH_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Scalar two-register misc - virtual bool SUQADD_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQABS_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGT_zero_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMEQ_zero_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMLT_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool ABS_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQXTN_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool USQADD_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQNEG_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGE_zero_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMLE_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool NEG_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQXTUN_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQXTN_1(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTXN_1(bool sz, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Scalar pairwise - virtual bool ADDP_pair(Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNMP_pair_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNMP_pair_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FADDP_pair_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FADDP_pair_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXP_pair_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXP_pair_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNMP_pair_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNMP_pair_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINP_pair_1(Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINP_pair_2(bool sz, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Scalar three different - virtual bool SQDMLAL_vec_1(Imm<2> size, Reg Rm, Reg Rn, Vec Vd) { - return false; - } - virtual bool SQDMLSL_vec_1(Imm<2> size, Reg Rm, Reg Rn, Vec Vd) { - return false; - } - virtual bool SQDMULL_vec_1(Imm<2> size, Reg Rm, Reg Rn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Scalar three same - virtual bool SQADD_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSUB_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGT_reg_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGE_reg_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSHL_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHL_reg_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRSHL_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRSHL_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ADD_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMTST_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQDMULH_vec_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQADD_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSUB_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMHI_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMHS_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool USHL_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSHL_reg_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSHL_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQRSHL_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SUB_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMEQ_reg_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMULH_vec_1(Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Scalar shift by immediate - virtual bool SSHR_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSRA_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRSHR_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRSRA_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHL_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHL_imm_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHRN_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRSHRN_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SCVTF_fix_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZS_fix_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool USHR_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool USRA_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSHR_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSRA_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRI_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SLI_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHLU_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSHL_imm_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHRUN_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRSHRUN_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSHRN_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQRSHRN_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UCVTF_fix_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZU_fix_1(Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Scalar x indexed element - virtual bool SQDMLAL_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQDMLSL_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQDMULL_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQDMULH_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQRDMULH_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLA_elt_1(Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLA_elt_2(bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLS_elt_1(Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLS_elt_2(bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMUL_elt_1(Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMUL_elt_2(bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMLAH_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQRDMLSH_elt_1(Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMULX_elt_1(Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMULX_elt_2(bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Table Lookup - virtual bool TBL(bool Q, Vec Vm, Imm<2> len, size_t Vn, Vec Vd) { - return false; - } - virtual bool TBX(bool Q, Vec Vm, Imm<2> len, size_t Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Permute - virtual bool UZP1(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool TRN1(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ZIP1(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UZP2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool TRN2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ZIP2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Extract - virtual bool EXT(bool Q, Vec Vm, Imm<4> imm4, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Copy - virtual bool DUP_elt_2(bool Q, Imm<5> imm5, Vec Vn, Vec Vd) { - return false; - } - virtual bool DUP_gen(bool Q, Imm<5> imm5, Reg Rn, Vec Vd) { - return false; - } - virtual bool SMOV(bool Q, Imm<5> imm5, Vec Vn, Reg Rd) { - return false; - } - virtual bool UMOV(bool Q, Imm<5> imm5, Vec Vn, Reg Rd) { - return false; - } - virtual bool INS_gen(Imm<5> imm5, Reg Rn, Vec Vd) { - return false; - } - virtual bool INS_elt(Imm<5> imm5, Imm<4> imm4, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Three same - virtual bool FMULX_vec_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPS_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTS_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGE_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABD_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_reg_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGT_3(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNM_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLA_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FADD_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAX_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNM_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLS_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FSUB_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMIN_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNMP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FADDP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMUL_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FDIV_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNMP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINP_vec_1(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Three same extra - virtual bool SDOT_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UDOT_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMLA_vec(bool Q, Imm<2> size, Vec Vm, Imm<2> rot, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCADD_vec(bool Q, Imm<2> size, Vec Vm, Imm<1> rot, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Two register misc - virtual bool REV64_asimd(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool REV16_asimd(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SADDLP(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CLS_asimd(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CNT(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SADALP(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool XTN(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTN(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTL(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool URECPE(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool REV32_asimd(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool UADDLP(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CLZ_asimd(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool UADALP(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHLL(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool NOT(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool RBIT_asimd(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSQRTE(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool SUQADD_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQABS_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGT_zero_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMEQ_zero_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMLT_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool ABS_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQXTN_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool USQADD_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQNEG_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGE_zero_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMLE_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool NEG_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQXTUN_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQXTN_2(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTXN_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTN_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTN_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTM_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTM_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABS_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABS_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTP_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTP_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTZ_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTZ_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTA_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTA_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTX_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTX_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FNEG_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FNEG_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTI_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTI_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FSQRT_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FSQRT_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT32X_1(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT64X_1(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT32Z_1(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT64Z_1(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD across lanes - virtual bool SADDLV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMAXV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMINV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool ADDV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNMV_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNMV_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXV_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXV_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNMV_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNMV_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINV_1(bool Q, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINV_2(bool Q, bool sz, Vec Vn, Vec Vd) { - return false; - } - virtual bool UADDLV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMAXV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMINV(bool Q, Imm<2> size, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD three different - virtual bool SADDL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SADDW(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSUBL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSUBW(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ADDHN(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SABAL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SUBHN(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SABDL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMLAL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMLSL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMULL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool PMULL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UADDL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UADDW(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool USUBL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool USUBW(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool RADDHN(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UABAL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool RSUBHN(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UABDL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMLAL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMLSL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMULL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQDMLAL_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQDMLSL_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQDMULL_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD three same - virtual bool SHADD(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRHADD(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHSUB(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMAX(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMIN(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SABD(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SABA(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool MLA_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool MUL_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMAXP(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SMINP(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ADDP_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLAL_vec_1(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLAL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool AND_asimd(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool BIC_asimd_reg(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLSL_vec_1(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLSL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ORR_asimd_reg(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ORN_asimd(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UHADD(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool URHADD(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UHSUB(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMAX(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMIN(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UABD(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UABA(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool MLS_vec(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool PMUL(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMAXP(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UMINP(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool EOR_asimd(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool BSL(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool BIT(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool BIF(bool Q, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLA_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FADD_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAX_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNM_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLS_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FSUB_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMIN_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FADDP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMUL_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FDIV_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNMP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINP_vec_2(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMULX_vec_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMEQ_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRECPS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRSQRTS_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGE_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGE_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABD_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCMGT_reg_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FACGT_4(bool Q, bool sz, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQADD_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSUB_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGT_reg_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMGE_reg_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSHL_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHL_reg_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRSHL_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRSHL_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool ADD_vector(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMTST_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQDMULH_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQADD_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSUB_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMHI_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMHS_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool USHL_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSHL_reg_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSHL_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQRSHL_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SUB_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool CMEQ_reg_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMULH_vec_2(bool Q, Imm<2> size, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD modified immediate - virtual bool MOVI(bool Q, bool op, Imm<1> a, Imm<1> b, Imm<1> c, Imm<4> cmode, Imm<1> d, - Imm<1> e, Imm<1> f, Imm<1> g, Imm<1> h, Vec Vd) { - return false; - } - virtual bool FMOV_2(bool Q, bool op, Imm<1> a, Imm<1> b, Imm<1> c, Imm<1> d, Imm<1> e, Imm<1> f, - Imm<1> g, Imm<1> h, Vec Vd) { - return false; - } - virtual bool FMOV_3(bool Q, Imm<1> a, Imm<1> b, Imm<1> c, Imm<1> d, Imm<1> e, Imm<1> f, - Imm<1> g, Imm<1> h, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD Shift by immediate - virtual bool SSHR_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSRA_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRSHR_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRSRA_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHL_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHL_imm_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHRN(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool RSHRN(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHRN_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRSHRN_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SSHLL(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SCVTF_fix_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZS_fix_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool USHR_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool USRA_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSHR_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool URSRA_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SRI_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SLI_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHLU_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSHL_imm_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQSHRUN_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRSHRUN_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQSHRN_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UQRSHRN_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool USHLL(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool UCVTF_fix_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVTZU_fix_2(bool Q, Imm<4> immh, Imm<3> immb, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SIMD vector x indexed element - virtual bool SMLAL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQDMLAL_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool SMLSL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQDMLSL_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool MUL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SMULL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vm, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQDMULL_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool SQDMULH_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool SQRDMULH_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool SDOT_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLA_elt_3(bool Q, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLA_elt_4(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLS_elt_3(bool Q, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMLS_elt_4(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMUL_elt_3(bool Q, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMUL_elt_4(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLAL_elt_1(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLAL_elt_2(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLSL_elt_1(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FMLSL_elt_2(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool MLA_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool UMLAL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool MLS_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool UMLSL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool UMULL_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQRDMLAH_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool UDOT_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool SQRDMLSH_elt_2(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, - Vec Vn, Vec Vd) { - return false; - } - virtual bool FMULX_elt_3(bool Q, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMULX_elt_4(bool Q, bool sz, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<1> H, Vec Vn, - Vec Vd) { - return false; - } - virtual bool FCMLA_elt(bool Q, Imm<2> size, Imm<1> L, Imm<1> M, Imm<4> Vmlo, Imm<2> rot, - Imm<1> H, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Cryptographic three register - virtual bool SM3TT1A(Vec Vm, Imm<2> imm2, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM3TT1B(Vec Vm, Imm<2> imm2, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM3TT2A(Vec Vm, Imm<2> imm2, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM3TT2B(Vec Vm, Imm<2> imm2, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SHA512 three register - virtual bool SHA512H(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA512H2(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SHA512SU1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool RAX1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool XAR(Vec Vm, Imm<6> imm6, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM3PARTW1(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM3PARTW2(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM4EKEY(Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Cryptographic four register - virtual bool EOR3(Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } - virtual bool BCAX(Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } - virtual bool SM3SS1(Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - SHA512 two register - virtual bool SHA512SU0(Vec Vn, Vec Vd) { - return false; - } - virtual bool SM4E(Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Conversion between floating point and fixed point - virtual bool SCVTF_float_fix(bool sf, Imm<2> type, Imm<6> scale, Reg Rn, Vec Vd) { - return false; - } - virtual bool UCVTF_float_fix(bool sf, Imm<2> type, Imm<6> scale, Reg Rn, Vec Vd) { - return false; - } - virtual bool FCVTZS_float_fix(bool sf, Imm<2> type, Imm<6> scale, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTZU_float_fix(bool sf, Imm<2> type, Imm<6> scale, Vec Vn, Reg Rd) { - return false; - } - - // Data Processing - FP and SIMD - Conversion between floating point and integer - virtual bool FCVTNS_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTNU_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool SCVTF_float_int(bool sf, Imm<2> type, Reg Rn, Vec Vd) { - return false; - } - virtual bool UCVTF_float_int(bool sf, Imm<2> type, Reg Rn, Vec Vd) { - return false; - } - virtual bool FCVTAS_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTAU_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FMOV_float_gen(bool sf, Imm<2> type, Imm<1> rmode_0, Imm<1> opc_0, size_t n, - size_t d) { - return false; - } - virtual bool FCVTPS_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTPU_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTMS_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTMU_float(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTZS_float_int(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FCVTZU_float_int(bool sf, Imm<2> type, Vec Vn, Reg Rd) { - return false; - } - virtual bool FJCVTZS(Vec Vn, Reg Rd) { - return false; - } - - // Data Processing - FP and SIMD - Floating point data processing - virtual bool FMOV_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FABS_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FNEG_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FSQRT_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FCVT_float(Imm<2> type, Imm<2> opc, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTN_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTP_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTM_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTZ_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTA_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTX_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINTI_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT32X_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT64X_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT32Z_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - virtual bool FRINT64Z_float(Imm<2> type, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Floating point compare - virtual bool FCMP_float(Imm<2> type, Vec Vm, Vec Vn, bool cmp_with_zero) { - return false; - } - virtual bool FCMPE_float(Imm<2> type, Vec Vm, Vec Vn, bool cmp_with_zero) { - return false; - } - - // Data Processing - FP and SIMD - Floating point immediate - virtual bool FMOV_float_imm(Imm<2> type, Imm<8> imm8, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Floating point conditional compare - virtual bool FCCMP_float(Imm<2> type, Vec Vm, Cond cond, Vec Vn, Imm<4> nzcv) { - return false; - } - virtual bool FCCMPE_float(Imm<2> type, Vec Vm, Cond cond, Vec Vn, Imm<4> nzcv) { - return false; - } - - // Data Processing - FP and SIMD - Floating point data processing two register - virtual bool FMUL_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FDIV_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FADD_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FSUB_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAX_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMIN_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMAXNM_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMINNM_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - virtual bool FNMUL_float(Imm<2> type, Vec Vm, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Floating point conditional select - virtual bool FCSEL_float(Imm<2> type, Vec Vm, Cond cond, Vec Vn, Vec Vd) { - return false; - } - - // Data Processing - FP and SIMD - Floating point data processing three register - virtual bool FMADD_float(Imm<2> type, Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } - virtual bool FMSUB_float(Imm<2> type, Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } - virtual bool FNMADD_float(Imm<2> type, Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } - virtual bool FNMSUB_float(Imm<2> type, Vec Vm, Vec Va, Vec Vn, Vec Vd) { - return false; - } -}; - -} // namespace Core diff --git a/src/core/device_memory.h b/src/core/device_memory.h index b5103e23bc..7603e94698 100644 --- a/src/core/device_memory.h +++ b/src/core/device_memory.h @@ -29,11 +29,8 @@ public: template Common::PhysicalAddress GetPhysicalAddr(const T* ptr) const { - return GetPhysicalAddr(reinterpret_cast(ptr)); - } - - Common::PhysicalAddress GetPhysicalAddr(uintptr_t ptr) const { - return (ptr - reinterpret_cast(buffer.BackingBasePointer())) + + return (reinterpret_cast(ptr) - + reinterpret_cast(buffer.BackingBasePointer())) + DramMemoryMap::Base; } diff --git a/src/core/device_memory_manager.h b/src/core/device_memory_manager.h index b4b3b46088..72c8b7787f 100644 --- a/src/core/device_memory_manager.h +++ b/src/core/device_memory_manager.h @@ -18,7 +18,11 @@ #include "common/common_types.h" #include "common/range_mutex.h" #include "common/scratch_buffer.h" -#include "common/sparse_large_vector.h" +#include "common/virtual_buffer.h" + +namespace Common { +class HostMemory; +} namespace Core { @@ -126,6 +130,10 @@ public: // New batch API to update multiple ranges with a single lock acquisition. void UpdatePagesCachedBatch(std::span> ranges, s32 delta); + const Common::HostMemory& GetHostMemory() const noexcept { + return host_memory; + } + private: struct TranslationEntry { DAddr guest_page{}; @@ -171,6 +179,7 @@ private: std::unique_ptr> impl; const uintptr_t physical_base; + const Common::HostMemory& host_memory; DeviceInterface* device_inter; struct TrackedEntry { @@ -178,8 +187,8 @@ private: u32 continuity_tracker; u32 compressed_physical_ptr; }; - Common::SparseLargeVector compressed_device_addr; - Common::SparseLargeVector tracked_entries; + Common::VirtualBuffer compressed_device_addr; + Common::VirtualBuffer tracked_entries; // Process memory interfaces @@ -200,8 +209,8 @@ private: return std::make_pair(asid, address); } - constexpr void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { - tracked_entries.GetUnchecked(page_index).cpu_backing_address = address | (asid.id << asid_start_bit); + void InsertCPUBacking(size_t page_index, VAddr address, Asid asid) { + tracked_entries[page_index].cpu_backing_address = address | (asid.id << asid_start_bit); } std::array t_slot{}; diff --git a/src/core/device_memory_manager.inc b/src/core/device_memory_manager.inc index 97c2706abb..8f3ed53285 100644 --- a/src/core/device_memory_manager.inc +++ b/src/core/device_memory_manager.inc @@ -171,12 +171,24 @@ struct DeviceMemoryManagerAllocator { template DeviceMemoryManager::DeviceMemoryManager(const DeviceMemory& device_memory_) : physical_base{uintptr_t(device_memory_.buffer.BackingBasePointer())} + , host_memory{device_memory_.buffer} , device_inter{nullptr} , compressed_device_addr(1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS)) , tracked_entries(device_as_size >> Memory::YUZU_PAGEBITS) { impl = std::make_unique>(); cached_pages = std::make_unique(); + + const size_t total_virtual = device_as_size >> Memory::YUZU_PAGEBITS; + for (size_t i = 0; i < total_virtual; i++) { + tracked_entries[i].compressed_physical_ptr = 0; + tracked_entries[i].continuity_tracker = 1; + tracked_entries[i].cpu_backing_address = 0; + } + const size_t total_phys = 1ULL << ((Settings::values.memory_layout_mode.GetValue() == Settings::MemoryLayout::Memory_4Gb ? physical_min_bits : physical_max_bits) - Memory::YUZU_PAGEBITS); + for (size_t i = 0; i < total_phys; i++) { + compressed_device_addr[i] = 0; + } } template @@ -209,28 +221,26 @@ void DeviceMemoryManager::Map(DAddr address, VAddr virtual_address, size size_t start_page_d = address >> Memory::YUZU_PAGEBITS; size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; std::scoped_lock lk(mapping_guard); - - tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = 0; i < num_pages; i++) { const VAddr new_vaddress = virtual_address + i * Memory::YUZU_PAGESIZE; auto* ptr = process_memory->GetPointerSilent(Common::ProcessAddress(new_vaddress)); if (ptr == nullptr) [[unlikely]] { - tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = 0; + tracked_entries[start_page_d + i].compressed_physical_ptr = 0; continue; } auto phys_addr = static_cast(GetRawPhysicalAddr(ptr) >> Memory::YUZU_PAGEBITS) + 1U; - tracked_entries.GetUnchecked(start_page_d + i).compressed_physical_ptr = phys_addr; + tracked_entries[start_page_d + i].compressed_physical_ptr = phys_addr; InsertCPUBacking(start_page_d + i, new_vaddress, asid); const u32 base_dev = compressed_device_addr[phys_addr - 1U]; const u32 new_dev = static_cast(start_page_d + i); if (base_dev == 0) [[likely]] { - compressed_device_addr.GetAndFault(phys_addr - 1U) = new_dev; + compressed_device_addr[phys_addr - 1U] = new_dev; continue; } u32 start_id = base_dev & MULTI_MASK; if ((base_dev >> MULTI_FLAG_BITS) == 0) { start_id = impl->multi_dev_address.Register(base_dev); - compressed_device_addr.GetAndFault(phys_addr - 1U) = MULTI_FLAG | start_id; + compressed_device_addr[phys_addr - 1U] = MULTI_FLAG | start_id; } impl->multi_dev_address.Register(new_dev, start_id); } @@ -246,26 +256,24 @@ void DeviceMemoryManager::Unmap(DAddr address, size_t size) { size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; device_inter->InvalidateRegion(address, size); std::scoped_lock lk(mapping_guard); - - tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); // should already be committed, but just in case for (size_t i = 0; i < num_pages; i++) { - auto& entry = tracked_entries.GetUnchecked(start_page_d + i); - auto phys_addr = entry.compressed_physical_ptr; - entry.compressed_physical_ptr = 0; - entry.cpu_backing_address = 0; + auto phys_addr = tracked_entries[start_page_d + i].compressed_physical_ptr; + tracked_entries[start_page_d + i].compressed_physical_ptr = 0; + tracked_entries[start_page_d + i].cpu_backing_address = 0; if (phys_addr != 0) [[likely]] { - u32& base_dev = compressed_device_addr.GetAndFault(phys_addr - 1U); + const u32 base_dev = compressed_device_addr[phys_addr - 1U]; if ((base_dev >> MULTI_FLAG_BITS) == 0) [[likely]] { - base_dev = 0; + compressed_device_addr[phys_addr - 1] = 0; continue; } const auto [more_entries, new_start] = impl->multi_dev_address.Unregister( static_cast(start_page_d + i), base_dev & MULTI_MASK); if (!more_entries) { - base_dev = impl->multi_dev_address.ReleaseEntry(new_start); + compressed_device_addr[phys_addr - 1] = + impl->multi_dev_address.ReleaseEntry(new_start); continue; } - base_dev = new_start | MULTI_FLAG; + compressed_device_addr[phys_addr - 1] = new_start | MULTI_FLAG; } } t_slot = {}; @@ -278,8 +286,6 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu size_t num_pages = Common::AlignUp(size, Memory::YUZU_PAGESIZE) >> Memory::YUZU_PAGEBITS; uintptr_t last_ptr = 0; size_t page_count = 1; - - tracked_entries.CommitRegion(start_page_d, start_page_d + num_pages); for (size_t i = num_pages; i > 0; i--) { size_t index = i - 1; const VAddr new_vaddress = virtual_address + index * Memory::YUZU_PAGESIZE; @@ -291,14 +297,14 @@ void DeviceMemoryManager::TrackContinuityImpl(DAddr address, VAddr virtu page_count = 1; } last_ptr = new_ptr; - tracked_entries.GetUnchecked(start_page_d + index).continuity_tracker = static_cast(page_count) - 1; + tracked_entries[start_page_d + index].continuity_tracker = static_cast(page_count); } } template u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) { size_t page_index = src_addr >> page_bits; size_t subbits = src_addr & page_mask; - if ((static_cast(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) { + if ((static_cast(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) { return GetPointer(src_addr); } return nullptr; @@ -308,7 +314,7 @@ template const u8* DeviceMemoryManager::GetSpan(const DAddr src_addr, const std::size_t size) const { size_t page_index = src_addr >> page_bits; size_t subbits = src_addr & page_mask; - if ((static_cast(tracked_entries[page_index].continuity_tracker+1) << page_bits) >= size + subbits) { + if ((static_cast(tracked_entries[page_index].continuity_tracker) << page_bits) >= size + subbits) { return GetPointer(src_addr); } return nullptr; @@ -378,7 +384,7 @@ void DeviceMemoryManager::WalkBlock(DAddr addr, std::size_t size, auto o std::size_t page_index = addr >> Memory::YUZU_PAGEBITS; std::size_t page_offset = addr & Memory::YUZU_PAGEMASK; while (remaining_size) { - const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker+1); + const size_t next_pages = std::size_t(tracked_entries[page_index].continuity_tracker); const std::size_t copy_amount = (std::min)((next_pages << Memory::YUZU_PAGEBITS) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << Memory::YUZU_PAGEBITS) + page_offset); SCOPE_EXIT{ diff --git a/src/core/file_sys/program_metadata.cpp b/src/core/file_sys/program_metadata.cpp index 63ca9d3fe3..d84c24f286 100644 --- a/src/core/file_sys/program_metadata.cpp +++ b/src/core/file_sys/program_metadata.cpp @@ -147,6 +147,10 @@ ProgramAddressSpaceType ProgramMetadata::GetAddressSpaceType() const { return npdm_header.address_space_type; } +void ProgramMetadata::SetAddressSpaceType(ProgramAddressSpaceType address_space) { + npdm_header.address_space_type.Assign(address_space); +} + u8 ProgramMetadata::GetMainThreadPriority() const { return npdm_header.main_thread_priority; } diff --git a/src/core/file_sys/program_metadata.h b/src/core/file_sys/program_metadata.h index 115e6d6cd1..7f5853567e 100644 --- a/src/core/file_sys/program_metadata.h +++ b/src/core/file_sys/program_metadata.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -73,6 +76,7 @@ public: bool Is64BitProgram() const; ProgramAddressSpaceType GetAddressSpaceType() const; + void SetAddressSpaceType(ProgramAddressSpaceType address_space); u8 GetMainThreadPriority() const; u8 GetMainThreadCore() const; u32 GetMainThreadStackSize() const; diff --git a/src/core/frontend/emu_window.h b/src/core/frontend/emu_window.h index ee7a7693b2..e495e8978f 100644 --- a/src/core/frontend/emu_window.h +++ b/src/core/frontend/emu_window.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: 2014 Citra Emulator Project @@ -74,7 +74,7 @@ public: }; /// Called from GPU thread when a frame is displayed. - virtual void OnFrameDisplayed() {} + virtual void OnFrameDisplayed([[maybe_unused]] u32 presented_frames) {} /** * Returns a GraphicsContext that the frontend provides to be used for rendering. diff --git a/src/core/hle/kernel/k_page_table_base.cpp b/src/core/hle/kernel/k_page_table_base.cpp index 2a0a7d49d5..a727355a9f 100644 --- a/src/core/hle/kernel/k_page_table_base.cpp +++ b/src/core/hle/kernel/k_page_table_base.cpp @@ -635,36 +635,6 @@ Result KPageTableBase::CheckMemoryState(const KMemoryInfo& info, KMemoryState st R_SUCCEED(); } -bool KPageTableBase::BeginTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, TraversalContext *out_context, - Common::ProcessAddress address) const { - out_context->next_offset = GetInteger(address); - out_context->next_page = GetInteger(address) >> PageBits; - - return ContinueTraversal(impl, out_entry, out_context); -} - -bool KPageTableBase::ContinueTraversal(const Common::PageTable &impl, TraversalEntry *out_entry, - TraversalContext *context) const { - // Setup invalid defaults. - out_entry->phys_addr = 0; - out_entry->block_size = PageSize; - // Validate that we can read the actual entry. - if (auto const page = context->next_page; page < impl.entries.size()) { - // Validate that the entry is mapped. - if (auto const paddr = impl.entries[page].Pointer(true); paddr != 0) { - // Populate the results and return true - out_entry->phys_addr = GetInteger(m_system.DeviceMemory().GetPhysicalAddr(paddr + context->next_offset)); - context->next_page += 1; - context->next_offset += PageSize; - return true; - } - } - context->next_page += 1; - context->next_offset += PageSize; - // Otherwise return false - return false; -} - Result KPageTableBase::CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, @@ -970,7 +940,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a size_t tot_size = 0; next_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), region_start); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), region_start); next_entry.block_size = (next_entry.block_size - (GetInteger(region_start) & (next_entry.block_size - 1))); @@ -1006,7 +976,7 @@ Result KPageTableBase::QueryMappingImpl(KProcessAddress* out, KPhysicalAddress a break; } - next_valid = ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + next_valid = impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); } // Check the last entry. @@ -1784,7 +1754,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_ // Begin traversal. TraversalContext context; TraversalEntry next_entry; - R_UNLESS(BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr), + R_UNLESS(impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr), ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -1794,7 +1764,7 @@ Result KPageTableBase::MakePageGroup(KPageGroup& pg, KProcessAddress addr, size_ // Iterate, adding to group as we go. while (tot_size < size) { - R_UNLESS(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)), + R_UNLESS(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)), ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -1858,7 +1828,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr // Begin traversal. TraversalContext context; TraversalEntry next_entry; - if (!BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), addr)) { + if (!impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), addr)) { return false; } @@ -1869,7 +1839,7 @@ bool KPageTableBase::IsValidPageGroup(const KPageGroup& pg, KProcessAddress addr // Iterate, comparing expected to actual. while (tot_size < size) { - if (!ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))) { + if (!impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))) { return false; } @@ -1926,7 +1896,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState( // Begin a traversal. TraversalContext context; TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0}; - R_UNLESS(BeginTraversal(impl, std::addressof(cur_entry), std::addressof(context), address), + R_UNLESS(impl.BeginTraversal(std::addressof(cur_entry), std::addressof(context), address), ResultInvalidCurrentMemory); // Traverse until we have enough size or we aren't contiguous any more. @@ -1935,7 +1905,7 @@ Result KPageTableBase::GetContiguousMemoryRangeWithState( for (contig_size = cur_entry.block_size - (GetInteger(phys_address) & (cur_entry.block_size - 1)); contig_size < size; contig_size += cur_entry.block_size) { - if (!ContinueTraversal(impl, std::addressof(cur_entry), std::addressof(context))) { + if (!impl.ContinueTraversal(std::addressof(cur_entry), std::addressof(context))) { break; } if (cur_entry.phys_addr != phys_address + contig_size) { @@ -2364,7 +2334,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(m_impl, std::addressof(next_entry), std::addressof(context), virt_addr); + m_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), virt_addr); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Set tracking variables. @@ -2375,7 +2345,7 @@ Result KPageTableBase::QueryPhysicalAddress(Svc::lp64::PhysicalMemoryInfo* out, while (true) { // Continue the traversal. traverse_valid = - ContinueTraversal(m_impl, std::addressof(next_entry), std::addressof(context)); + m_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); if (!traverse_valid) { break; } @@ -2597,7 +2567,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre TraversalContext context; TraversalEntry next_entry; ASSERT( - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address)); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address)); // Check that the physical region matches. R_UNLESS(next_entry.phys_addr == phys_addr, ResultInvalidMemoryRegion); @@ -2607,7 +2577,7 @@ Result KPageTableBase::UnmapIoRegion(KProcessAddress dst_address, KPhysicalAddre next_entry.block_size - (GetInteger(phys_addr) & (next_entry.block_size - 1)); checked_size < size; checked_size += next_entry.block_size) { // Continue the traversal. - ASSERT(ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context))); + ASSERT(impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context))); // Check that the physical region matches. R_UNLESS(next_entry.phys_addr == phys_addr + checked_size, ResultInvalidMemoryRegion); @@ -3059,7 +3029,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_ TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), address); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3071,7 +3041,7 @@ Result KPageTableBase::InvalidateProcessDataCache(KProcessAddress address, size_ while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3159,7 +3129,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_address); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3197,7 +3167,7 @@ Result KPageTableBase::ReadDebugMemory(KProcessAddress dst_address, KProcessAddr while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3255,7 +3225,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_address); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_address); R_UNLESS(traverse_valid, ResultInvalidCurrentMemory); // Prepare tracking variables. @@ -3297,7 +3267,7 @@ Result KPageTableBase::WriteDebugMemory(KProcessAddress dst_address, KProcessAdd while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3758,7 +3728,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3798,7 +3768,7 @@ Result KPageTableBase::CopyMemoryFromLinearToUser( while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3852,7 +3822,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), src_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), src_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3875,7 +3845,7 @@ Result KPageTableBase::CopyMemoryFromLinearToKernel( while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -3932,7 +3902,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear( TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -3971,7 +3941,7 @@ Result KPageTableBase::CopyMemoryFromUserToLinear( while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -4027,7 +3997,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si TraversalContext context; TraversalEntry next_entry; bool traverse_valid = - BeginTraversal(impl, std::addressof(next_entry), std::addressof(context), dst_addr); + impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), dst_addr); ASSERT(traverse_valid); // Prepare tracking variables. @@ -4050,7 +4020,7 @@ Result KPageTableBase::CopyMemoryFromKernelToLinear(KProcessAddress dst_addr, si while (tot_size < size) { // Continue the traversal. traverse_valid = - ContinueTraversal(impl, std::addressof(next_entry), std::addressof(context)); + impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); if (next_entry.phys_addr != (cur_addr + cur_size)) { @@ -4119,10 +4089,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( bool traverse_valid; // Begin traversal. - traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry), std::addressof(src_context), src_addr); ASSERT(traverse_valid); - traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry), std::addressof(dst_context), dst_addr); ASSERT(traverse_valid); @@ -4157,7 +4127,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( if (ofs + cur_copy_size != size) { if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) { // Continue the src traversal. - traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry), std::addressof(src_context)); ASSERT(traverse_valid); @@ -4168,7 +4138,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeap( if (cur_dst_addr + cur_min_size == dst_next_entry.phys_addr + dst_next_entry.block_size) { // Continue the dst traversal. - traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry), std::addressof(dst_context)); ASSERT(traverse_valid); @@ -4253,10 +4223,10 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( bool traverse_valid; // Begin traversal. - traverse_valid = BeginTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.BeginTraversal(std::addressof(src_next_entry), std::addressof(src_context), src_addr); ASSERT(traverse_valid); - traverse_valid = BeginTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.BeginTraversal(std::addressof(dst_next_entry), std::addressof(dst_context), dst_addr); ASSERT(traverse_valid); @@ -4291,7 +4261,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( if (ofs + cur_copy_size != size) { if (cur_src_addr + cur_min_size == cur_src_block_addr + cur_src_size) { // Continue the src traversal. - traverse_valid = ContinueTraversal(src_impl, std::addressof(src_next_entry), + traverse_valid = src_impl.ContinueTraversal(std::addressof(src_next_entry), std::addressof(src_context)); ASSERT(traverse_valid); @@ -4302,7 +4272,7 @@ Result KPageTableBase::CopyMemoryFromHeapToHeapWithoutCheckDestination( if (cur_dst_addr + cur_min_size == dst_next_entry.phys_addr + dst_next_entry.block_size) { // Continue the dst traversal. - traverse_valid = ContinueTraversal(dst_impl, std::addressof(dst_next_entry), + traverse_valid = dst_impl.ContinueTraversal(std::addressof(dst_next_entry), std::addressof(dst_context)); ASSERT(traverse_valid); @@ -4577,7 +4547,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, // Begin traversal. TraversalContext context; TraversalEntry next_entry; - bool traverse_valid = BeginTraversal(src_impl, std::addressof(next_entry), + bool traverse_valid = src_impl.BeginTraversal(std::addressof(next_entry), std::addressof(context), aligned_src_start); ASSERT(traverse_valid); @@ -4627,7 +4597,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, // If the block's size was one page, we may need to continue traversal. if (cur_block_size == 0 && aligned_src_size > PageSize) { traverse_valid = - ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); + src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); cur_block_addr = next_entry.phys_addr; @@ -4640,7 +4610,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, while (aligned_src_start + tot_block_size < mapping_src_end) { // Continue the traversal. traverse_valid = - ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); + src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); // Process the block. @@ -4683,7 +4653,7 @@ Result KPageTableBase::SetupForIpcServer(KProcessAddress* out_addr, size_t size, if (mapped_block_end + cur_block_size < aligned_src_end && cur_block_size == last_block_size) { traverse_valid = - ContinueTraversal(src_impl, std::addressof(next_entry), std::addressof(context)); + src_impl.ContinueTraversal(std::addressof(next_entry), std::addressof(context)); ASSERT(traverse_valid); cur_block_addr = next_entry.phys_addr; @@ -5631,7 +5601,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si ContiguousRangeInfo(KPageTableBase& pt, KProcessAddress address, size_t size) : m_pt(pt), m_remaining_size(size) { // Begin a traversal. - ASSERT(m_pt.BeginTraversal(m_pt.GetImpl(), std::addressof(m_entry), + ASSERT(m_pt.GetImpl().BeginTraversal(std::addressof(m_entry), std::addressof(m_context), address)); // Setup tracking fields. @@ -5662,7 +5632,7 @@ Result KPageTableBase::UnmapProcessMemory(KProcessAddress dst_address, size_t si void DetermineContiguousBlockExtents() { // Continue traversing until we're not contiguous, or we have enough. while (m_cur_size < m_remaining_size) { - ASSERT(m_pt.ContinueTraversal(m_pt.GetImpl(), std::addressof(m_entry), + ASSERT(m_pt.GetImpl().ContinueTraversal(std::addressof(m_entry), std::addressof(m_context))); // If we're not contiguous, we're done. diff --git a/src/core/hle/kernel/k_page_table_base.h b/src/core/hle/kernel/k_page_table_base.h index 0c222910fc..1d4fa85f04 100644 --- a/src/core/hle/kernel/k_page_table_base.h +++ b/src/core/hle/kernel/k_page_table_base.h @@ -370,10 +370,6 @@ private: size_t num_pages, size_t alignment, size_t offset, size_t guard_pages) const; - bool BeginTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* out_context, - Common::ProcessAddress address) const; - bool ContinueTraversal(const Common::PageTable& impl, TraversalEntry* out_entry, TraversalContext* context) const; - Result CheckMemoryStateContiguous(size_t* out_blocks_needed, KProcessAddress addr, size_t size, KMemoryState state_mask, KMemoryState state, KMemoryPermission perm_mask, KMemoryPermission perm, @@ -478,14 +474,7 @@ private: // Validate pre-conditions. ASSERT(this->IsLockedByCurrentThread()); - if (virt_addr > (1ULL << m_address_space_width)) { - return false; - } - - *out = m_system.DeviceMemory().GetPhysicalAddr( - this->GetImpl().entries[GetInteger(virt_addr) >> PageBits].Pointer(true) + GetInteger(virt_addr)); - - return true; + return this->GetImpl().GetPhysicalAddress(out, virt_addr); } public: diff --git a/src/core/hle/service/nvdrv/core/nvmap.cpp b/src/core/hle/service/nvdrv/core/nvmap.cpp index e2bf49da62..b2cbfbd686 100644 --- a/src/core/hle/service/nvdrv/core/nvmap.cpp +++ b/src/core/hle/service/nvdrv/core/nvmap.cpp @@ -211,14 +211,19 @@ DAddr NvMap::PinHandle(NvMap::Handle::Id handle, bool low_area_pin) { while ((address = smmu.Allocate(aligned_up)) == 0) { // Free handles until the allocation succeeds std::scoped_lock queueLock(unmap_queue_lock); - if (auto freeHandleDesc{unmap_queue.front()}) { - // Handles in the unmap queue are guaranteed not to be pinned so don't bother - // checking if they are before unmapping - std::scoped_lock freeLock(freeHandleDesc->mutex); - if (handle_description->d_address) - UnmapHandle(*freeHandleDesc); - } else { + if (unmap_queue.empty()) { LOG_CRITICAL(Service_NVDRV, "Ran out of SMMU address space!"); + return 0; + } + // Handles in the unmap queue are guaranteed not to be pinned so don't bother + // checking if they are before unmapping + const std::shared_ptr freeHandleDesc = unmap_queue.front(); + std::scoped_lock freeLock(freeHandleDesc->mutex); + if (freeHandleDesc->d_address) { + UnmapHandle(*freeHandleDesc); + } else { + unmap_queue.pop_front(); + freeHandleDesc->unmap_queue_entry.reset(); } } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp index 3c9e42f602..f486817c8d 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.cpp @@ -233,9 +233,9 @@ NvResult nvhost_as_gpu::FreeSpace(IoctlFreeSpace& params) { if (allocation.page_size != params.page_size || allocation.size != (u64(params.pages) * params.page_size)) return NvResult::BadValue; - for (const auto mapping_offset : allocation.mappings) - if (!FreeMappingLocked(mapping_offset)) - return NvResult::BadValue; + for (const auto mapping_offset : allocation.mappings) { + void(FreeMappingLocked(mapping_offset)); + } // Unset sparse flag if required if (allocation.sparse) @@ -375,39 +375,16 @@ NvResult nvhost_as_gpu::MapBufferEx(IoctlMapBufferEx& params) { mapping_map.insert_or_assign(params.offset, Mapping(params.handle, device_address, params.offset, size, false, big_page, false)); } - map_buffer_offsets.insert(params.offset); - return NvResult::Success; } NvResult nvhost_as_gpu::UnmapBuffer(IoctlUnmapBuffer& params) { + LOG_DEBUG(Service_NVDRV, "called, offset={:#x}", params.offset); std::scoped_lock lock(mutex); - if (auto const offset_it = map_buffer_offsets.find(params.offset); offset_it != map_buffer_offsets.end()) { - LOG_DEBUG(Service_NVDRV, "called, offset={:#x}", params.offset); - if (!vm.initialised) { - return NvResult::BadValue; - } - - auto const it = mapping_map.find(params.offset); - auto const mapping = it->second; - if (!mapping.fixed) { - auto& allocator{mapping.big_page ? *vm.big_page_allocator : *vm.small_page_allocator}; - u32 page_size_bits{mapping.big_page ? vm.big_page_size_bits : VM::PAGE_SIZE_BITS}; - allocator.Free(u32(mapping.offset >> page_size_bits), u32(mapping.size >> page_size_bits)); - } - - // Sparse mappings shouldn't be fully unmapped, just returned to their sparse state - // Only FreeSpace can unmap them fully - if (mapping.sparse_alloc) { - gmmu->MapSparse(params.offset, mapping.size, mapping.big_page); - } else { - gmmu->Unmap(params.offset, mapping.size); - } - - nvmap.UnpinHandle(mapping.handle); - mapping_map.erase(params.offset); - map_buffer_offsets.erase(params.offset); + if (!vm.initialised) { + return NvResult::BadValue; } + void(FreeMappingLocked(params.offset)); return NvResult::Success; } diff --git a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h index b13b1c9efa..f66dcf0dc2 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h +++ b/src/core/hle/service/nvdrv/devices/nvhost_as_gpu.h @@ -113,8 +113,6 @@ private: }; static_assert(sizeof(IoctlRemapEntry) == 20, "IoctlRemapEntry is incorrect size"); - ::Common::unordered_set map_buffer_offsets{}; - struct IoctlMapBufferEx { MappingFlags flags{}; // bit0: fixed_offset, bit2: cacheable u32_le kind{}; // -1 is default diff --git a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp index 6ef32b890f..7db706c5e7 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_ctrl.cpp @@ -13,6 +13,7 @@ #include "common/assert.h" #include "common/logging.h" #include "common/scope_exit.h" +#include "common/settings.h" #include "core/core.h" #include "core/hle/kernel/k_event.h" #include "core/hle/service/nvdrv/core/container.h" @@ -147,9 +148,12 @@ NvResult nvhost_ctrl::IocCtrlEventWait(IocCtrlEventWaitParams& params, bool is_a const auto check_failing = [&]() { if (events[slot].fails > 2) { - { - auto lk = system.StallApplication(); - host1x_syncpoint_manager.WaitHost(fence_id, target_value); + std::unique_lock stall; + if (Settings::values.stall_on_gpu_fence_wait.GetValue()) { + stall = system.StallApplication(); + } + host1x_syncpoint_manager.WaitHost(fence_id, target_value); + if (stall.owns_lock()) { system.UnstallApplication(); } params.value.raw = target_value; diff --git a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp index 358679b831..193b28573e 100644 --- a/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp +++ b/src/core/hle/service/nvdrv/devices/nvhost_gpu.cpp @@ -312,19 +312,17 @@ static boost::container::small_vector BuildWaitComman static boost::container::small_vector BuildIncrementCommandList( NvFence fence) { - boost::container::small_vector result{ + const Tegra::CommandHeader increment = + BuildFenceAction(Tegra::Engines::Puller::FenceOperation::Increment, fence.id); + return { Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointPayload, 1, Tegra::SubmissionMode::Increasing), - {}}; - - for (u32 count = 0; count < 2; ++count) { - result.push_back(Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 1, - Tegra::SubmissionMode::Increasing)); - result.push_back( - BuildFenceAction(Tegra::Engines::Puller::FenceOperation::Increment, fence.id)); - } - - return result; + {}, + Tegra::BuildCommandHeader(Tegra::BufferMethods::SyncpointOperation, 2, + Tegra::SubmissionMode::NonIncreasing), + increment, + increment, + }; } static boost::container::small_vector BuildIncrementWithWfiCommandList( @@ -370,17 +368,14 @@ NvResult nvhost_gpu::SubmitGPFIFOImpl(IoctlSubmitGpfifo& params, Tegra::CommandL u32 increment{(flags.fence_increment.Value() != 0 ? 2 : 0) + (flags.increment_value.Value() != 0 ? params.fence.value : 0)}; params.fence.value = syncpoint_manager.IncrementSyncpointMaxExt(channel_syncpoint, increment); - gpu.PushGPUEntries(bind_id, std::move(entries)); - if (flags.fence_increment.Value()) { if (flags.suppress_wfi.Value()) { - gpu.PushGPUEntries(bind_id, - Tegra::CommandList{BuildIncrementCommandList(params.fence)}); + entries.prefetch_command_list = BuildIncrementCommandList(params.fence); } else { - gpu.PushGPUEntries(bind_id, - Tegra::CommandList{BuildIncrementWithWfiCommandList(params.fence)}); + entries.prefetch_command_list = BuildIncrementWithWfiCommandList(params.fence); } } + gpu.PushGPUEntries(bind_id, std::move(entries)); flags.raw = 0; diff --git a/src/core/hle/service/nvdrv/nvdrv.cpp b/src/core/hle/service/nvdrv/nvdrv.cpp index b252a2efa5..dd82e76d9e 100644 --- a/src/core/hle/service/nvdrv/nvdrv.cpp +++ b/src/core/hle/service/nvdrv/nvdrv.cpp @@ -165,13 +165,22 @@ Module::Module(Core::System& system) Module::~Module() {} +std::shared_ptr Module::FindDevice(DeviceFD fd) const { + std::scoped_lock lock(open_files_mutex); + const auto itr = open_files.find(fd); + if (itr == open_files.end()) { + return nullptr; + } + return itr->second; +} + NvResult Module::VerifyFD(DeviceFD fd) const { if (fd < 0) { LOG_ERROR(Service_NVDRV, "Invalid DeviceFD={}!", fd); return NvResult::InvalidState; } - if (open_files.find(fd) == open_files.end()) { + if (!FindDevice(fd)) { LOG_ERROR(Service_NVDRV, "Could not find DeviceFD={}!", fd); return NvResult::NotImplemented; } @@ -187,8 +196,11 @@ DeviceFD Module::Open(const std::string& device_name, NvCore::SessionId session_ } const DeviceFD fd = next_fd++; - auto& builder = it->second; - auto device = builder(fd)->second; + std::shared_ptr device; + { + std::scoped_lock lock(open_files_mutex); + device = it->second(fd)->second; + } device->OnOpen(session_id, fd); @@ -202,14 +214,13 @@ NvResult Module::Ioctl1(DeviceFD fd, Ioctl command, std::span input, return NvResult::InvalidState; } - const auto itr = open_files.find(fd); - - if (itr == open_files.end()) { + const auto device = FindDevice(fd); + if (!device) { LOG_ERROR(Service_NVDRV, "Could not find DeviceFD={}!", fd); return NvResult::NotImplemented; } - return itr->second->Ioctl1(fd, command, input, output); + return device->Ioctl1(fd, command, input, output); } NvResult Module::Ioctl2(DeviceFD fd, Ioctl command, std::span input, @@ -219,14 +230,13 @@ NvResult Module::Ioctl2(DeviceFD fd, Ioctl command, std::span input, return NvResult::InvalidState; } - const auto itr = open_files.find(fd); - - if (itr == open_files.end()) { + const auto device = FindDevice(fd); + if (!device) { LOG_ERROR(Service_NVDRV, "Could not find DeviceFD={}!", fd); return NvResult::NotImplemented; } - return itr->second->Ioctl2(fd, command, input, inline_input, output); + return device->Ioctl2(fd, command, input, inline_input, output); } NvResult Module::Ioctl3(DeviceFD fd, Ioctl command, std::span input, std::span output, @@ -236,14 +246,13 @@ NvResult Module::Ioctl3(DeviceFD fd, Ioctl command, std::span input, s return NvResult::InvalidState; } - const auto itr = open_files.find(fd); - - if (itr == open_files.end()) { + const auto device = FindDevice(fd); + if (!device) { LOG_ERROR(Service_NVDRV, "Could not find DeviceFD={}!", fd); return NvResult::NotImplemented; } - return itr->second->Ioctl3(fd, command, input, output, inline_output); + return device->Ioctl3(fd, command, input, output, inline_output); } NvResult Module::Close(DeviceFD fd) { @@ -252,16 +261,17 @@ NvResult Module::Close(DeviceFD fd) { return NvResult::InvalidState; } - const auto itr = open_files.find(fd); - - if (itr == open_files.end()) { + const auto device = FindDevice(fd); + if (!device) { LOG_ERROR(Service_NVDRV, "Could not find DeviceFD={}!", fd); return NvResult::NotImplemented; } - itr->second->OnClose(fd); - - open_files.erase(itr); + { + std::scoped_lock lock(open_files_mutex); + open_files.erase(fd); + } + device->OnClose(fd); return NvResult::Success; } @@ -272,14 +282,13 @@ NvResult Module::QueryEvent(DeviceFD fd, u32 event_id, Kernel::KEvent*& event) { return NvResult::InvalidState; } - const auto itr = open_files.find(fd); - - if (itr == open_files.end()) { + const auto device = FindDevice(fd); + if (!device) { LOG_ERROR(Service_NVDRV, "Could not find DeviceFD={}!", fd); return NvResult::NotImplemented; } - event = itr->second->QueryEvent(event_id); + event = device->QueryEvent(event_id); if (!event) { return NvResult::BadParameter; } diff --git a/src/core/hle/service/nvdrv/nvdrv.h b/src/core/hle/service/nvdrv/nvdrv.h index f503f108c3..4d8873c28f 100644 --- a/src/core/hle/service/nvdrv/nvdrv.h +++ b/src/core/hle/service/nvdrv/nvdrv.h @@ -65,10 +65,7 @@ public: /// Returns a pointer to one of the available devices, identified by its name. template std::shared_ptr GetDevice(DeviceFD fd) { - auto itr = open_files.find(fd); - if (itr == open_files.end()) - return nullptr; - return std::static_pointer_cast(itr->second); + return std::static_pointer_cast(FindDevice(fd)); } NvResult VerifyFD(DeviceFD fd) const; @@ -97,6 +94,8 @@ public: private: friend class EventInterface; + std::shared_ptr FindDevice(DeviceFD fd) const; + /// Manages syncpoints on the host NvCore::Container container; @@ -106,6 +105,7 @@ private: using FilesContainerType = ::Common::unordered_map>; /// Mapping of file descriptors to the devices they reference. FilesContainerType open_files; + mutable std::mutex open_files_mutex; KernelHelpers::ServiceContext service_context; diff --git a/src/core/loader/deconstructed_rom_directory.cpp b/src/core/loader/deconstructed_rom_directory.cpp index 5e2ec7e0b9..b04da7e84f 100644 --- a/src/core/loader/deconstructed_rom_directory.cpp +++ b/src/core/loader/deconstructed_rom_directory.cpp @@ -182,6 +182,12 @@ AppLoader_DeconstructedRomDirectory::LoadResult AppLoader_DeconstructedRomDirect } metadata.Print(); + if (Settings::values.relocate_36bit_address_space.GetValue() && + Settings::values.cpu_backend.GetValue() == Settings::CpuBackend::Nce && + metadata.GetAddressSpaceType() == FileSys::ProgramAddressSpaceType::Is36Bit) { + metadata.SetAddressSpaceType(FileSys::ProgramAddressSpaceType::Is39Bit); + } + // Enable NCE only for applications with 39-bit address space. const bool is_39bit = metadata.GetAddressSpaceType() == FileSys::ProgramAddressSpaceType::Is39Bit; diff --git a/src/core/memory.cpp b/src/core/memory.cpp index 837016de1d..a5e6b8592c 100644 --- a/src/core/memory.cpp +++ b/src/core/memory.cpp @@ -101,10 +101,8 @@ struct Memory::Impl { } u64 protect_bytes = 0, protect_begin = 0; - - current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr + size) >> YUZU_PAGEBITS); for (u64 addr = vaddr; addr < vaddr + size; addr += YUZU_PAGESIZE) { - const Common::PageType page_type = current_page_table->entries.GetUnchecked(addr >> YUZU_PAGEBITS).Type(); + const Common::PageType page_type = current_page_table->entries[addr >> YUZU_PAGEBITS].ptr.Type(); switch (page_type) { case Common::PageType::RasterizerCachedMemory: if (protect_bytes > 0) { @@ -125,14 +123,16 @@ struct Memory::Impl { } [[nodiscard]] u8* GetPointerFromRasterizerCachedMemory(u64 vaddr) const { - if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr) - return reinterpret_cast(paddr) + vaddr; + Common::PhysicalAddress const paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr; + if (paddr) + return system.DeviceMemory().GetPointer(paddr + vaddr); return {}; } [[nodiscard]] u8* GetPointerFromDebugMemory(u64 vaddr) const { - if (u64 paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Pointer(true); paddr) - return reinterpret_cast(paddr) + vaddr; + const Common::PhysicalAddress paddr = current_page_table->entries[vaddr >> YUZU_PAGEBITS].addr; + if (paddr != 0) + return system.DeviceMemory().GetPointer(paddr + vaddr); return {}; } @@ -243,12 +243,10 @@ struct Memory::Impl { std::size_t page_index = addr >> YUZU_PAGEBITS; std::size_t page_offset = addr & YUZU_PAGEMASK; bool user_accessible = true; - - current_page_table->entries.CommitRegion(page_index, page_index + (size >> YUZU_PAGEBITS) + 1); while (remaining_size != 0) { const std::size_t copy_amount = (std::min)(std::size_t(YUZU_PAGESIZE) - page_offset, remaining_size); const auto current_vaddr = u64((page_index << YUZU_PAGEBITS) + page_offset); - const auto [pointer, type, _] = current_page_table->entries.GetUnchecked(page_index).PointerTypeBlock(); + const auto [pointer, type] = current_page_table->entries[page_index].ptr.PointerType(); switch (type) { case Common::PageType::Unmapped: { user_accessible = false; @@ -299,10 +297,10 @@ struct Memory::Impl { } [[nodiscard]] inline const u8* GetSpan(const VAddr addr, const std::size_t size) const noexcept { - return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr; + return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr; } [[nodiscard]] inline u8* GetSpan(const VAddr addr, const std::size_t size) noexcept { - return (current_page_table->entries[addr >> YUZU_PAGEBITS].Block() == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].Block()) ? GetPointerSilent(addr) : nullptr; + return (current_page_table->entries[addr >> YUZU_PAGEBITS].block == current_page_table->entries[(addr + size) >> YUZU_PAGEBITS].block) ? GetPointerSilent(addr) : nullptr; } bool WriteBlockImpl(const Common::ProcessAddress addr, const void* buffer, const std::size_t size, bool unsafe) { @@ -406,14 +404,11 @@ struct Memory::Impl { // The region is at a granularity of CPU pages. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; - - current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); - const auto [pointer, type, block] = entry.PointerTypeBlock(true); + const Common::PageType page_type = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); if (debug) { // Switch page type to debug if now debug - switch (type) { + switch (page_type) { case Common::PageType::Unmapped: ASSERT(false && "Attempted to mark unmapped pages as debug"); break; @@ -422,14 +417,14 @@ struct Memory::Impl { // Page is already marked. break; case Common::PageType::Memory: - entry.MarkDebug(pointer, block); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::DebugMemory); break; default: UNREACHABLE(); } } else { // Switch page type to non-debug if now non-debug - switch (type) { + switch (page_type) { case Common::PageType::Unmapped: ASSERT(false && "Attempted to mark unmapped pages as non-debug"); break; @@ -438,7 +433,8 @@ struct Memory::Impl { // Don't mess with already non-debug or rasterizer memory. break; case Common::PageType::DebugMemory: { - entry.Store(false, Common::PageType::Memory, block, pointer); + u8* const pointer = GetPointerFromDebugMemory(vaddr & ~YUZU_PAGEMASK); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); break; } default: @@ -470,10 +466,8 @@ struct Memory::Impl { // is different). This assumes the specified GPU address region is contiguous as well. const u64 num_pages = ((vaddr + size - 1) >> YUZU_PAGEBITS) - (vaddr >> YUZU_PAGEBITS) + 1; - current_page_table->entries.CommitRegion(vaddr >> YUZU_PAGEBITS, (vaddr >> YUZU_PAGEBITS) + num_pages); for (u64 i = 0; i < num_pages; ++i, vaddr += YUZU_PAGESIZE) { - auto& entry = current_page_table->entries.GetUnchecked(vaddr >> YUZU_PAGEBITS); - const Common::PageType page_type = entry.Type(); + const Common::PageType page_type= current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Type(); if (cached) { // Switch page type to cached if now cached switch (page_type) { @@ -483,7 +477,7 @@ struct Memory::Impl { break; case Common::PageType::DebugMemory: case Common::PageType::Memory: - entry.MarkRasterizerCached(); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::RasterizerCachedMemory); break; case Common::PageType::RasterizerCachedMemory: // There can be more than one GPU region mapped per CPU region, so it's common @@ -505,13 +499,13 @@ struct Memory::Impl { // that this area is already unmarked as cached. break; case Common::PageType::RasterizerCachedMemory: { - if (auto [ptr, _, block] = entry.PointerTypeBlock(true); ptr == 0) { + if (u8* const pointer = GetPointerFromRasterizerCachedMemory(vaddr & ~YUZU_PAGEMASK); pointer == nullptr) { // It's possible that this function has been called while updating the // pagetable after unmapping a VMA. In that case the underlying VMA will no // longer exist, and we should just leave the pagetable entry blank. - entry.Store(false, Common::PageType::Unmapped, block, 0); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(0, Common::PageType::Unmapped); } else { - entry.Store(false, Common::PageType::Memory, block, ptr); + current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Store(uintptr_t(pointer) - (vaddr & ~YUZU_PAGEMASK), Common::PageType::Memory); } break; } @@ -545,18 +539,22 @@ struct Memory::Impl { ASSERT_MSG(type != Common::PageType::Memory, "Mapping memory page without a pointer @ {:016x}", base * YUZU_PAGESIZE); - page_table.entries.ZeroRegion(base, end); - } else { - auto current_block = block_count.fetch_add(1, std::memory_order_relaxed); - ASSERT(current_block != 65535); - - page_table.entries.CommitRegion(base, end); while (base != end) { - auto host_ptr = reinterpret_cast(system.DeviceMemory().GetPointer(target)) - (base << YUZU_PAGEBITS);; - auto& entry = page_table.entries.GetUnchecked(base); + page_table.entries[base].ptr.Store(0, type); + page_table.entries[base].addr = 0; + page_table.entries[base].block = 0; + base += 1; + } + } else { + auto orig_base = base; + while (base != end) { + auto host_ptr = uintptr_t(system.DeviceMemory().GetPointer(target)) - (base << YUZU_PAGEBITS); + auto backing = GetInteger(target) - (base << YUZU_PAGEBITS); + page_table.entries[base].ptr.Store(host_ptr, type); + page_table.entries[base].addr = backing; + page_table.entries[base].block = orig_base << YUZU_PAGEBITS; - entry.Store(false, type, current_block, host_ptr); - ASSERT_MSG(page_table.entries[base].Pointer(), + ASSERT_MSG(page_table.entries[base].ptr.Pointer(), "memory mapping base yield a nullptr within the table"); base += 1; @@ -571,11 +569,11 @@ struct Memory::Impl { vaddr &= 0xffffffffffffULL; if (AddressSpaceContains(*current_page_table, vaddr, 1)) [[likely]] { // Avoid adding any extra logic to this fast-path block - const auto raw = current_page_table->entries[vaddr >> YUZU_PAGEBITS].Raw(); - if (auto pointer = Common::PageTable::PageEntryData::ExtractPointer(raw); pointer) [[likely]] { + const uintptr_t raw_pointer = current_page_table->entries[vaddr >> YUZU_PAGEBITS].ptr.Raw(); + if (const uintptr_t pointer = Common::PageTable::PageInfo::ExtractPointer(raw_pointer)) [[likely]] { return reinterpret_cast(pointer + vaddr); } else { - switch (static_cast(raw.type)) { + switch (Common::PageTable::PageInfo::ExtractType(raw_pointer)) { case Common::PageType::Memory: ASSERT_MSG(false, "Mapped memory page without a pointer @ {:#016x}", vaddr); return nullptr; @@ -775,7 +773,6 @@ struct Memory::Impl { #else Common::HostMemory* host_buffer{}; #endif - std::atomic block_count = 0; }; Memory::Memory(Core::System& system_) : system{system_} { @@ -814,7 +811,7 @@ bool Memory::IsValidVirtualAddress(const Common::ProcessAddress vaddr) const { if (page >= page_table.entries.size()) { return false; } - const auto [pointer, type, _] = page_table.entries[page].PointerTypeBlock(); + const auto [pointer, type] = page_table.entries[page].ptr.PointerType(); return pointer != 0 || type == Common::PageType::RasterizerCachedMemory || type == Common::PageType::DebugMemory; } diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp index 8b6ec2506e..da51220e9c 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a32_address_space.cpp @@ -371,10 +371,8 @@ EmitConfig A32AddressSpace::GetEmitConfig() { .page_table_pointer = std::bit_cast(conf.page_table), .page_table_address_space_bits = 32, - .page_table_pointer_mask = conf.page_table_pointer_mask, + .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, .page_table_log2_stride = conf.page_table_log2_stride, - .page_table_marked_bit = conf.page_table_marked_bit, - .page_table_sign_extension = conf.page_table_sign_extension, .silently_mirror_page_table = true, .absolute_offset_page_table = conf.absolute_offset_page_table, .detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp index 2dbb7d2e1d..2c71ffe282 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/a64_address_space.cpp @@ -545,10 +545,8 @@ EmitConfig A64AddressSpace::GetEmitConfig() { .page_table_pointer = std::bit_cast(conf.page_table), .page_table_address_space_bits = conf.page_table_address_space_bits, - .page_table_pointer_mask = conf.page_table_pointer_mask, + .page_table_pointer_mask_bits = conf.page_table_pointer_mask_bits, .page_table_log2_stride = conf.page_table_log2_stride, - .page_table_marked_bit = conf.page_table_marked_bit, - .page_table_sign_extension = conf.page_table_sign_extension, .silently_mirror_page_table = conf.silently_mirror_page_table, .absolute_offset_page_table = conf.absolute_offset_page_table, .detect_misaligned_access_via_page_table = conf.detect_misaligned_access_via_page_table, diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h index 693f71ea39..f2c89bbfc0 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64.h @@ -128,10 +128,8 @@ struct EmitConfig { // Page table u64 page_table_pointer; std::size_t page_table_address_space_bits; - u64 page_table_pointer_mask; + int page_table_pointer_mask_bits; std::size_t page_table_log2_stride; - std::optional page_table_marked_bit; - std::optional page_table_sign_extension; bool silently_mirror_page_table; bool absolute_offset_page_table; u8 detect_misaligned_access_via_page_table; diff --git a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp index 170fcf9613..143244d60b 100644 --- a/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp +++ b/src/dynarmic/src/dynarmic/backend/arm64/emit_arm64_memory.cpp @@ -273,18 +273,9 @@ std::pair InlinePageTableEmitVAddrLookup(oaknut::Cod // load x0 = *<(u8*)pagetable + index> code.LDR(Xscratch0, Xpagetable, Xscratch0); - if (ctx.conf.page_table_marked_bit) { - code.TST(Xscratch0, 1ULL << *ctx.conf.page_table_marked_bit); - code.B(NE, *fallback); - } - - if (ctx.conf.page_table_pointer_mask != 0) { - code.AND(Xscratch0, Xscratch0, ctx.conf.page_table_pointer_mask); - } - - // TODO: combine this with page_table_pointer_mask - if (ctx.conf.page_table_sign_extension) { - code.SBFM(Xscratch0, Xscratch0, 0, *ctx.conf.page_table_sign_extension); + if (ctx.conf.page_table_pointer_mask_bits != 0) { + const u64 mask = u64(~u64(0)) << ctx.conf.page_table_pointer_mask_bits; + code.AND(Xscratch0, Xscratch0, mask); } code.CBZ(Xscratch0, *fallback); diff --git a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h index 54ad638ada..3ac078f1d7 100644 --- a/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h +++ b/src/dynarmic/src/dynarmic/backend/x64/emit_x64_memory.h @@ -9,7 +9,6 @@ #pragma once #include -#include #include "dynarmic/backend/x64/xbyak.h" #include "dynarmic/backend/x64/a32_emit_x64.h" @@ -79,46 +78,27 @@ Xbyak::RegExp EmitVAddrLookup(BlockOfCode& code, EmitContext& ctx, size_t bitsiz template<> [[maybe_unused]] Xbyak::RegExp EmitVAddrLookup(BlockOfCode& code, A32EmitContext& ctx, size_t bitsize, Xbyak::Label& abort, Xbyak::Reg64 vaddr) { const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code); - const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code); + const Xbyak::Reg32 tmp = ctx.conf.absolute_offset_page_table ? page.cvt32() : ctx.reg_alloc.ScratchGpr(code).cvt32(); - EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp); + EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp.cvt64()); - code.mov(tmp, vaddr); + // TODO: This code assumes vaddr has been zext from 32-bits to 64-bits. + + code.mov(tmp, vaddr.cvt32()); code.shr(tmp, int(page_table_const_bits)); - - if (ctx.conf.page_table_log2_stride > 3) { - code.shl(tmp, int(ctx.conf.page_table_log2_stride)); - code.mov(page, qword[r14 + tmp.cvt64()]); - } else { - code.mov(page, qword[r14 + tmp.cvt64() * int(ctx.conf.page_table_log2_stride)]); - } - - // check for marked bit, use as unmapped if marked - if (ctx.conf.page_table_marked_bit) { - code.bt(page, *ctx.conf.page_table_marked_bit); - code.jc(abort, code.T_NEAR); - } - // mask away attributes - if (ctx.conf.page_table_pointer_mask == 0) { + code.shl(tmp, int(ctx.conf.page_table_log2_stride)); + code.mov(page, qword[r14 + tmp.cvt64()]); + if (ctx.conf.page_table_pointer_mask_bits == 0) { code.test(page, page); - } else if (std::in_range(ctx.conf.page_table_pointer_mask)) { - code.and_(page, ctx.conf.page_table_pointer_mask); } else { - code.mov(tmp, ctx.conf.page_table_pointer_mask); - code.and_(page, tmp); + code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits); } - // check for sign bit, apply sign extension as needed - if (ctx.conf.page_table_sign_extension) { - code.shl(page, 63 - int(*ctx.conf.page_table_sign_extension)); - code.sar(page, 63 - int(*ctx.conf.page_table_sign_extension)); - } - code.jz(abort, code.T_NEAR); if (ctx.conf.absolute_offset_page_table) { return page + vaddr; } - code.mov(tmp, vaddr); - code.and_(tmp, u32(page_table_const_mask)); + code.mov(tmp, vaddr.cvt32()); + code.and_(tmp, static_cast(page_table_const_mask)); return page + tmp.cvt64(); } @@ -128,7 +108,7 @@ template<> const size_t unused_top_bits = 64 - ctx.conf.page_table_address_space_bits; const Xbyak::Reg64 page = ctx.reg_alloc.ScratchGpr(code); - const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table && ctx.conf.page_table_pointer_mask == 0 ? page : ctx.reg_alloc.ScratchGpr(code); + const Xbyak::Reg64 tmp = ctx.conf.absolute_offset_page_table ? page : ctx.reg_alloc.ScratchGpr(code); EmitDetectMisalignedVAddr(code, ctx, bitsize, abort, vaddr, tmp); @@ -163,26 +143,11 @@ template<> code.shl(tmp, int(ctx.conf.page_table_log2_stride)); code.mov(page, qword[r14 + tmp]); - - // check for marked bit, use as unmapped if marked - if (ctx.conf.page_table_marked_bit) { - code.bt(page, *ctx.conf.page_table_marked_bit); - code.jc(abort, code.T_NEAR); - } - // mask away attributes - if (ctx.conf.page_table_pointer_mask == 0) { + if (ctx.conf.page_table_pointer_mask_bits == 0) { code.test(page, page); - } else if (std::in_range(ctx.conf.page_table_pointer_mask)) { - code.and_(page, ctx.conf.page_table_pointer_mask); } else { - code.mov(tmp, ctx.conf.page_table_pointer_mask); - code.and_(page, tmp); + code.and_(page, ~u32(0) << ctx.conf.page_table_pointer_mask_bits); } - if (ctx.conf.page_table_sign_extension) { - code.shl(page, *ctx.conf.page_table_sign_extension); - code.sar(page, *ctx.conf.page_table_sign_extension); - } - code.jz(abort, code.T_NEAR); if (ctx.conf.absolute_offset_page_table) { return page + vaddr; diff --git a/src/dynarmic/src/dynarmic/interface/A32/config.h b/src/dynarmic/src/dynarmic/interface/A32/config.h index b93b116164..5a97fb69f3 100644 --- a/src/dynarmic/src/dynarmic/interface/A32/config.h +++ b/src/dynarmic/src/dynarmic/interface/A32/config.h @@ -159,23 +159,14 @@ struct UserConfig { /// Maximum size is limited by the maximum length of a x86_64 / arm64 jump. std::uint32_t code_cache_size = 128 * 1024 * 1024; // bytes - /// Applies a bit mask to the bits in host pointers from the page table. + /// Masks out the first N bits in host pointers from the page table. /// The intention behind this is to allow users of Dynarmic to pack attributes in the /// same integer and update the pointer attribute pair atomically. - /// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes. - std::uint64_t page_table_pointer_mask = 0; + /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. + std::int32_t page_table_pointer_mask_bits = 0; - /// Log2 of the size per page entry, value should be either 3 or 4 - std::uint32_t page_table_log2_stride = 3; - - /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. - /// If the bit is set to 1, Dynarmic will treat it as unmapped. - /// This bit should be included as part of `page_table_pointer_mask_bits`. - std::optional page_table_marked_bit = std::nullopt; - - /// If this value is set, Dynarmic will sign extend the page table pointer by this bit. - /// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`. - std::optional page_table_sign_extension = std::nullopt; + // Log2 of the size per page entry, value should be either 3 or 4 + std::size_t page_table_log2_stride = 3; /// Select the architecture version to use. /// There are minor behavioural differences between versions. diff --git a/src/dynarmic/src/dynarmic/interface/A64/config.h b/src/dynarmic/src/dynarmic/interface/A64/config.h index 818aed73e7..83c1593fd8 100644 --- a/src/dynarmic/src/dynarmic/interface/A64/config.h +++ b/src/dynarmic/src/dynarmic/interface/A64/config.h @@ -173,23 +173,14 @@ struct UserConfig { /// This is only used if page_table is not nullptr. std::uint32_t page_table_address_space_bits = 36; - /// Applies a bit mask to the bits in host pointers from the page table. + /// Masks out the first N bits in host pointers from the page table. /// The intention behind this is to allow users of Dynarmic to pack attributes in the /// same integer and update the pointer attribute pair atomically. - /// If the configured value is ~(0b111ULL), all pointers will be forcefully aligned to 8 bytes. - std::uint64_t page_table_pointer_mask = 0; + /// If the configured value is 3, all pointers will be forcefully aligned to 8 bytes. + std::int32_t page_table_pointer_mask_bits = 0; - /// Log2 of the size per page entry, value should be either 3 or 4 - std::uint32_t page_table_log2_stride = 3; - - /// Setting this value has Dynarmic check the specified bit of the page pointer provided by page table. - /// If the bit is set to 1, Dynarmic will treat it as unmapped. - /// This bit should be included as part of `page_table_pointer_mask`. - std::optional page_table_marked_bit = std::nullopt; - - /// If this value is set, Dynarmic will sign extend the page table pointer by this bit. - /// Useful for compacting bits into the page table and should be used as part of `page_table_pointer_mask`. - std::optional page_table_sign_extension = std::nullopt; + // Log2 of the size per page entry, value should be either 3 or 4 + std::size_t page_table_log2_stride = 3; /// Counter-timer frequency register. The value of the register is not interpreted by /// dynarmic. diff --git a/src/qt_common/config/shared_translation.cpp b/src/qt_common/config/shared_translation.cpp index 4ef97c602a..0562749073 100644 --- a/src/qt_common/config/shared_translation.cpp +++ b/src/qt_common/config/shared_translation.cpp @@ -154,6 +154,25 @@ std::unique_ptr InitializeTranslations(QObject* parent) { INSERT(Settings, post_shader_preset, QString(), QString()); INSERT(Settings, post_shader_enabled, tr("Enable post-processing effects"), tr("Applies post-processing effects to the final image.")); + INSERT(Settings, frame_gen, tr("Frame generation (Lossless Scaling)"), + tr("Generates intermediate frames with the Lossless Scaling frame generation " + "shaders.\nRequires your own legal copy of Lossless.dll and forces VSync (FIFO).")); + INSERT(Settings, frame_gen_multiplier, tr("Frame generation multiplier:"), + tr("How many frames are shown for each frame the game renders.\nOnly used when the " + "target frame rate is 0.")); + INSERT(Settings, frame_gen_target_rate, tr("Frame generation target rate:"), + tr("Pick the rate your display can actually show. The multiplier then rises or falls " + "on its own to hold it.\n0 uses the fixed multiplier.")); + INSERT(Settings, frame_gen_flow_scale_auto, tr("Match motion estimation to the game"), + tr("Estimate motion at the resolution the game actually renders instead of the " + "upscaled output.")); + INSERT(Settings, frame_gen_flow_scale, tr("Motion estimation resolution:"), + tr("Resolution of the optical flow pass, as a fraction of the game's render " + "resolution.\nLowering it is the cheapest way to reclaim performance.")); + INSERT(Settings, frame_gen_queue_target, tr("Frame generation queue target:"), + tr("How many finished frames may wait ahead of the display.\nLarger queues absorb GPU " + "spikes at the cost of input latency.")); + INSERT(Settings, frame_gen_dump_flow, QString(), QString()); INSERT(Settings, fullscreen_mode, tr("Fullscreen Mode:"), tr("The method used to render the window in fullscreen.\nBorderless offers the best " "compatibility with the on-screen keyboard that some games request for " @@ -179,14 +198,6 @@ std::unique_ptr InitializeTranslations(QObject* parent) { "GPU: Use the GPU's compute shaders to decode ASTC textures (recommended).\n" "CPU Asynchronously: Use the CPU to decode ASTC textures on demand. Eliminates" "ASTC decoding\nstuttering but may present artifacts.")); - INSERT(Settings, astc_recompression, tr("ASTC Recompression Method:"), - tr("Most GPUs lack support for ASTC textures and must decompress to an" - "intermediate format: RGBA8.\n" - "BC1/BC3: The intermediate format will be recompressed to BC1 or BC3 format,\n" - " saving VRAM but degrading image quality.")); - INSERT(Settings, frame_pacing_mode, tr("Frame Pacing Mode (Vulkan only)"), - tr("Controls how the emulator manages frame pacing to reduce stuttering and make the " - "frame rate smoother and more consistent.")); INSERT(Settings, vram_usage_mode, tr("VRAM Usage Mode:"), tr("Selects whether the emulator should prefer to conserve memory or make maximum usage " "of available video memory for performance.\nAggressive mode may impact performance " @@ -211,6 +222,10 @@ std::unique_ptr InitializeTranslations(QObject* parent) { tr("Ensures data consistency between compute and memory operations.\nThis option fixes " "issues in games, but may degrade performance.\nUnreal Engine 4 games often see the " "most significant changes thereof.")); + INSERT(Settings, stall_on_gpu_fence_wait, tr("Pause CPU on GPU fence timeouts"), + tr("When a game keeps timing out while waiting for the GPU, pauses every CPU thread " + "until the GPU catches up.\nDisabling it keeps the other threads running, which can " + "reduce stutter but may break games that rely on it.")); INSERT(Settings, async_presentation, tr("Enable asynchronous presentation (Vulkan only)"), tr("Slightly improves performance by moving presentation to a separate CPU thread.")); INSERT( @@ -234,23 +249,6 @@ std::unique_ptr InitializeTranslations(QObject* parent) { INSERT(Settings, gpu_clock, tr("GPU Clocks"), tr("Makes the game believe GPU work finishes faster than it does, so it stops lowering " "resolution and render distance to fit the Switch's clocks.")); - INSERT(Settings, gpu_unswizzle_enabled, tr("GPU Unswizzle"), - tr("Accelerates BCn 3D texture decoding using GPU compute.\n" - "Disable if experiencing crashes or graphical glitches.")); - INSERT(Settings, gpu_unswizzle_texture_size, tr("GPU Unswizzle Max Texture Size"), - tr("Sets the maximum size (MiB) for GPU-based texture unswizzling.\n" - "While the GPU is faster for medium and large textures, the CPU may be more " - "efficient for very small ones.\n" - "Adjust this to find the balance between GPU acceleration and CPU overhead.")); - INSERT(Settings, gpu_unswizzle_stream_size, tr("GPU Unswizzle Stream Size"), - tr("Sets the maximum amount of texture data (in MiB) processed per frame.\n" - "Higher values can reduce stutter during texture loading but may impact frame " - "consistency.")); - INSERT(Settings, gpu_unswizzle_chunk_size, tr("GPU Unswizzle Chunk Size"), - tr("Determines the number of depth slices processed in a single dispatch.\n" - "Increasing this can improve throughput on high-end GPUs but may cause TDR or driver " - "timeouts on weaker hardware.")); - INSERT(Settings, use_vulkan_driver_pipeline_cache, tr("Use Vulkan pipeline cache"), tr("Enables GPU vendor-specific pipeline cache.\nThis option can improve shader loading " "time significantly in cases where the Vulkan driver does not store pipeline cache " @@ -396,21 +394,6 @@ std::unique_ptr ComboboxEnumeration(QObject* parent) { PAIR(AstcDecodeMode, Gpu, tr("GPU")), PAIR(AstcDecodeMode, CpuAsynchronous, tr("CPU Asynchronous")), }}); - translations->insert( - {Settings::EnumMetadata::Index(), - { - PAIR(AstcRecompression, Uncompressed, tr("Uncompressed (Best quality)")), - PAIR(AstcRecompression, Bc1, tr("BC1 (Low quality)")), - PAIR(AstcRecompression, Bc3, tr("BC3 (Medium quality)")), - }}); - translations->insert({Settings::EnumMetadata::Index(), - { - PAIR(FramePacingMode, Target_Auto, tr("Auto")), - PAIR(FramePacingMode, Target_30, tr("30 FPS")), - PAIR(FramePacingMode, Target_60, tr("60 FPS")), - PAIR(FramePacingMode, Target_90, tr("90 FPS")), - PAIR(FramePacingMode, Target_120, tr("120 FPS")), - }}); translations->insert({Settings::EnumMetadata::Index(), { PAIR(VramUsageMode, Conservative, tr("Conservative")), @@ -654,30 +637,6 @@ std::unique_ptr ComboboxEnumeration(QObject* parent) { PAIR(GpuClock, Boost, tr("Boost")), PAIR(GpuClock, Overclock, tr("Overclock")), }}); - translations->insert({Settings::EnumMetadata::Index(), - { - PAIR(GpuUnswizzleSize, VerySmall, tr("Very Small (16 MB)")), - PAIR(GpuUnswizzleSize, Small, tr("Small (32 MB)")), - PAIR(GpuUnswizzleSize, Normal, tr("Normal (128 MB)")), - PAIR(GpuUnswizzleSize, Large, tr("Large (256 MB)")), - PAIR(GpuUnswizzleSize, VeryLarge, tr("Very Large (512 MB)")), - }}); - translations->insert({Settings::EnumMetadata::Index(), - { - PAIR(GpuUnswizzle, VeryLow, tr("Very Low (4 MB)")), - PAIR(GpuUnswizzle, Low, tr("Low (8 MB)")), - PAIR(GpuUnswizzle, Normal, tr("Normal (16 MB)")), - PAIR(GpuUnswizzle, Medium, tr("Medium (32 MB)")), - PAIR(GpuUnswizzle, High, tr("High (64 MB)")), - }}); - translations->insert({Settings::EnumMetadata::Index(), - { - PAIR(GpuUnswizzleChunk, VeryLow, tr("Very Low (32)")), - PAIR(GpuUnswizzleChunk, Low, tr("Low (64)")), - PAIR(GpuUnswizzleChunk, Normal, tr("Normal (128)")), - PAIR(GpuUnswizzleChunk, Medium, tr("Medium (256)")), - PAIR(GpuUnswizzleChunk, High, tr("High (512)")), - }}); translations->insert({Settings::EnumMetadata::Index(), { diff --git a/src/qt_common/config/uisettings.h b/src/qt_common/config/uisettings.h index 794d1a029b..58a538367a 100644 --- a/src/qt_common/config/uisettings.h +++ b/src/qt_common/config/uisettings.h @@ -262,6 +262,5 @@ Q_DECLARE_METATYPE(Settings::ResolutionSetup); Q_DECLARE_METATYPE(Settings::ScalingFilter); Q_DECLARE_METATYPE(Settings::AntiAliasing); Q_DECLARE_METATYPE(Settings::RendererBackend); -Q_DECLARE_METATYPE(Settings::AstcRecompression); Q_DECLARE_METATYPE(Settings::AstcDecodeMode); Q_DECLARE_METATYPE(Settings::SpirvOptimizeMode); diff --git a/src/shader_recompiler/CMakeLists.txt b/src/shader_recompiler/CMakeLists.txt index 957ed5fb55..cc9ce95d92 100644 --- a/src/shader_recompiler/CMakeLists.txt +++ b/src/shader_recompiler/CMakeLists.txt @@ -172,6 +172,7 @@ add_library(shader_recompiler STATIC ir_opt/constant_propagation_pass.cpp ir_opt/dead_code_elimination_pass.cpp ir_opt/dual_vertex_pass.cpp + ir_opt/geometry_compaction_pass.cpp ir_opt/global_memory_to_storage_buffer_pass.cpp ir_opt/identity_removal_pass.cpp ir_opt/layer_pass.cpp diff --git a/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp b/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp index 9319ea007f..ee39ec2a47 100644 --- a/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp +++ b/src/shader_recompiler/backend/glasm/emit_glasm_memory.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -32,10 +35,10 @@ void GlobalStorageOp(EmitContext& ctx, Register address, bool pointer_based, std std::string_view else_expr = {}) { const size_t num_buffers{ctx.info.storage_buffers_descriptors.size()}; for (size_t index = 0; index < num_buffers; ++index) { - if (!ctx.info.nvn_buffer_used[index]) { + const auto& ssbo{ctx.info.storage_buffers_descriptors[index]}; + if (!ssbo.is_global_fallback) { continue; } - const auto& ssbo{ctx.info.storage_buffers_descriptors[index]}; const u64 ssbo_align_mask{~(ctx.profile.min_ssbo_alignment - 1U)}; ctx.Add("LDC.U64 DC.x,c{}[{}];" // unaligned_ssbo_addr "AND.U64 DC.x,DC.x,{};" // ssbo_addr = unaligned_ssbo_addr & ssbo_align_mask @@ -65,9 +68,10 @@ void GlobalStorageOp(EmitContext& ctx, Register address, bool pointer_based, std if (!else_expr.empty()) { ctx.Add("{}", else_expr); } - const size_t num_used_buffers{ctx.info.nvn_buffer_used.count()}; - for (size_t index = 0; index < num_used_buffers; ++index) { - ctx.Add("ENDIF;"); + for (const auto& ssbo : ctx.info.storage_buffers_descriptors) { + if (ssbo.is_global_fallback) { + ctx.Add("ENDIF;"); + } } } diff --git a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp index a2df6159fb..4f4d420bd7 100644 --- a/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp +++ b/src/shader_recompiler/backend/glsl/glsl_emit_context.cpp @@ -633,7 +633,7 @@ std::string EmitContext::DefineGlobalMemoryFunctions() { std::string load_func_128{"uvec4 LoadGlobal128(uint64_t addr){"}; const size_t num_buffers{info.storage_buffers_descriptors.size()}; for (size_t index = 0; index < num_buffers; ++index) { - if (!info.nvn_buffer_used[index]) { + if (!info.storage_buffers_descriptors[index].is_global_fallback) { continue; } define_body(write_func, index, "{0}[uint(addr-{1})>>2]=data;return;}}"); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv.cpp b/src/shader_recompiler/backend/spirv/emit_spirv.cpp index c221888c09..a903480c78 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include #include #include @@ -439,10 +440,13 @@ void SetupCapabilities(const Profile& profile, const Info& info, EmitContext& ct ctx.AddCapability(spv::Capability::DrawParameters); } if ((info.uses_subgroup_vote || info.uses_subgroup_invocation_id || - info.uses_subgroup_shuffles) && + info.uses_subgroup_shuffles || info.uses_subgroup_mask) && profile.support_vote && profile.SupportsSubgroupStage(ctx.stage)) { ctx.AddCapability(spv::Capability::GroupNonUniformBallot); ctx.AddCapability(spv::Capability::GroupNonUniformShuffle); + if (info.uses_subgroup_shuffles && profile.support_shuffle_relative) { + ctx.AddCapability(spv::Capability::GroupNonUniformShuffleRelative); + } if (!profile.warp_size_potentially_larger_than_guest) { // vote ops are only used when not taking the long path ctx.AddCapability(spv::Capability::GroupNonUniformVote); @@ -521,6 +525,29 @@ void PatchPhiNodes(IR::Program& program, EmitContext& ctx) { return { ctx.Def(phi->Arg(phi_arg)), parent }; }); } + +void RewriteOpcodes(std::vector& code, std::span> rewrites) { + if (rewrites.empty()) { + return; + } + size_t offset = 5; + while (offset + 2 < code.size()) { + const u32 word_count{code[offset] >> 16}; + const auto opcode{static_cast(code[offset] & 0xFFFFu)}; + if (word_count == 0) { + return; + } + if (opcode == spv::Op::OpGroupNonUniformShuffleXor || + opcode == spv::Op::OpGroupNonUniformQuadBroadcast) { + const auto it{std::ranges::find(rewrites, code[offset + 2], + &std::pair::first)}; + if (it != rewrites.end()) { + code[offset] = (code[offset] & 0xFFFF0000u) | static_cast(it->second); + } + } + offset += word_count; + } +} } // Anonymous namespace std::vector EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_info, IR::Program& program, Bindings& bindings) { @@ -535,7 +562,9 @@ std::vector EmitSPIRV(const Profile& profile, const RuntimeInfo& runtime_in SetupCapabilities(profile, program.info, ctx); SetupTransformFeedbackCapabilities(ctx, main); PatchPhiNodes(program, ctx); - return ctx.Assemble(); + std::vector code{ctx.Assemble()}; + RewriteOpcodes(code, ctx.opcode_rewrites); + return code; } Id EmitPhi(EmitContext& ctx, IR::Inst* inst) { diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp index d1c08c1131..465c40277a 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_atomic.cpp @@ -417,48 +417,48 @@ Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const I return ctx.OpPackHalf2x16(ctx.U32[1], result); } -Id EmitGlobalAtomicIAdd32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicIAdd32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicSMin32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicSMin32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicUMin32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicUMin32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicSMax32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicSMax32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicUMax32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicUMax32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicInc32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicInc32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicInc32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicDec32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicDec32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicDec32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicAnd32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicAnd32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicOr32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicOr32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicXor32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicXor32, ctx.U32[1], address, value); } -Id EmitGlobalAtomicExchange32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicExchange32, ctx.U32[1], address, value); } Id EmitGlobalAtomicIAdd64(EmitContext&) { @@ -549,32 +549,32 @@ Id EmitGlobalAtomicExchange32x2(EmitContext&) { throw NotImplementedException("SPIR-V Instruction"); } -Id EmitGlobalAtomicAddF32(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicAddF32, ctx.F32[1], address, value); } -Id EmitGlobalAtomicAddF16x2(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicAddF16x2(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicAddF16x2, ctx.U32[1], address, value); } -Id EmitGlobalAtomicAddF32x2(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicAddF32x2(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicAddF32x2, ctx.U32[1], address, value); } -Id EmitGlobalAtomicMinF16x2(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicMinF16x2(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicMinF16x2, ctx.U32[1], address, value); } -Id EmitGlobalAtomicMinF32x2(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicMinF32x2(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicMinF32x2, ctx.U32[1], address, value); } -Id EmitGlobalAtomicMaxF16x2(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicMaxF16x2(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicMaxF16x2, ctx.U32[1], address, value); } -Id EmitGlobalAtomicMaxF32x2(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitGlobalAtomicMaxF32x2(EmitContext& ctx, Id address, Id value) { + return ctx.CallGlobalMemory(IR::Opcode::GlobalAtomicMaxF32x2, ctx.U32[1], address, value); } } // namespace Shader::Backend::SPIRV diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h index 165ec3fe84..9d49289dd7 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h +++ b/src/shader_recompiler/backend/spirv/emit_spirv_instructions.h @@ -90,17 +90,17 @@ Id EmitUndefU8(EmitContext& ctx); Id EmitUndefU16(EmitContext& ctx); Id EmitUndefU32(EmitContext& ctx); Id EmitUndefU64(EmitContext& ctx); -void EmitLoadGlobalU8(EmitContext& ctx); -void EmitLoadGlobalS8(EmitContext& ctx); -void EmitLoadGlobalU16(EmitContext& ctx); -void EmitLoadGlobalS16(EmitContext& ctx); +Id EmitLoadGlobalU8(EmitContext& ctx, Id address); +Id EmitLoadGlobalS8(EmitContext& ctx, Id address); +Id EmitLoadGlobalU16(EmitContext& ctx, Id address); +Id EmitLoadGlobalS16(EmitContext& ctx, Id address); Id EmitLoadGlobal32(EmitContext& ctx, Id address); Id EmitLoadGlobal64(EmitContext& ctx, Id address); Id EmitLoadGlobal128(EmitContext& ctx, Id address); -void EmitWriteGlobalU8(EmitContext& ctx); -void EmitWriteGlobalS8(EmitContext& ctx); -void EmitWriteGlobalU16(EmitContext& ctx); -void EmitWriteGlobalS16(EmitContext& ctx); +void EmitWriteGlobalU8(EmitContext& ctx, Id address, Id value); +void EmitWriteGlobalS8(EmitContext& ctx, Id address, Id value); +void EmitWriteGlobalU16(EmitContext& ctx, Id address, Id value); +void EmitWriteGlobalS16(EmitContext& ctx, Id address, Id value); void EmitWriteGlobal32(EmitContext& ctx, Id address, Id value); void EmitWriteGlobal64(EmitContext& ctx, Id address, Id value); void EmitWriteGlobal128(EmitContext& ctx, Id address, Id value); @@ -415,17 +415,17 @@ Id EmitStorageAtomicMaxF16x2(EmitContext& ctx, const IR::Value& binding, const I Id value); Id EmitStorageAtomicMaxF32x2(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, Id value); -Id EmitGlobalAtomicIAdd32(EmitContext& ctx); -Id EmitGlobalAtomicSMin32(EmitContext& ctx); -Id EmitGlobalAtomicUMin32(EmitContext& ctx); -Id EmitGlobalAtomicSMax32(EmitContext& ctx); -Id EmitGlobalAtomicUMax32(EmitContext& ctx); -Id EmitGlobalAtomicInc32(EmitContext& ctx); -Id EmitGlobalAtomicDec32(EmitContext& ctx); -Id EmitGlobalAtomicAnd32(EmitContext& ctx); -Id EmitGlobalAtomicOr32(EmitContext& ctx); -Id EmitGlobalAtomicXor32(EmitContext& ctx); -Id EmitGlobalAtomicExchange32(EmitContext& ctx); +Id EmitGlobalAtomicIAdd32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicSMin32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicUMin32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicSMax32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicUMax32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicInc32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicDec32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicAnd32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicOr32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicXor32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicExchange32(EmitContext& ctx, Id address, Id value); Id EmitGlobalAtomicIAdd64(EmitContext& ctx); Id EmitGlobalAtomicSMin64(EmitContext& ctx); Id EmitGlobalAtomicUMin64(EmitContext& ctx); @@ -448,13 +448,13 @@ Id EmitGlobalAtomicAnd32x2(EmitContext& ctx); Id EmitGlobalAtomicOr32x2(EmitContext& ctx); Id EmitGlobalAtomicXor32x2(EmitContext& ctx); Id EmitGlobalAtomicExchange32x2(EmitContext& ctx); -Id EmitGlobalAtomicAddF32(EmitContext& ctx); -Id EmitGlobalAtomicAddF16x2(EmitContext& ctx); -Id EmitGlobalAtomicAddF32x2(EmitContext& ctx); -Id EmitGlobalAtomicMinF16x2(EmitContext& ctx); -Id EmitGlobalAtomicMinF32x2(EmitContext& ctx); -Id EmitGlobalAtomicMaxF16x2(EmitContext& ctx); -Id EmitGlobalAtomicMaxF32x2(EmitContext& ctx); +Id EmitGlobalAtomicAddF32(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicAddF16x2(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicAddF32x2(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicMinF16x2(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicMinF32x2(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicMaxF16x2(EmitContext& ctx, Id address, Id value); +Id EmitGlobalAtomicMaxF32x2(EmitContext& ctx, Id address, Id value); Id EmitLogicalOr(EmitContext& ctx, Id a, Id b); Id EmitLogicalAnd(EmitContext& ctx, Id a, Id b); Id EmitLogicalXor(EmitContext& ctx, Id a, Id b); diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp index bef4421fed..cc72f4baf1 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_memory.cpp @@ -69,93 +69,71 @@ void WriteStorage32(EmitContext& ctx, const IR::Value& binding, const IR::Value& &StorageDefinitions::U32, index_offset); } -void WriteStorageByCasLoop(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, - Id value, Id bit_offset, Id bit_count) { +void WriteStorageBits(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset, + Id value, Id bit_offset, Id bit_count) { const Id pointer{StoragePointer(ctx, binding, offset, ctx.storage_types.U32, sizeof(u32), &StorageDefinitions::U32)}; - ctx.OpFunctionCall(ctx.TypeVoid(), ctx.write_storage_cas_loop_func, pointer, value, bit_offset, - bit_count); + ctx.AtomicBitFieldInsert(pointer, value, bit_offset, bit_count); } } // Anonymous namespace -void EmitLoadGlobalU8(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitLoadGlobalU8(EmitContext& ctx, Id address) { + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobalU8, ctx.U32[1], address, ctx.u32_zero_value); } -void EmitLoadGlobalS8(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitLoadGlobalS8(EmitContext& ctx, Id address) { + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobalS8, ctx.U32[1], address, ctx.u32_zero_value); } -void EmitLoadGlobalU16(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitLoadGlobalU16(EmitContext& ctx, Id address) { + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobalU16, ctx.U32[1], address, + ctx.u32_zero_value); } -void EmitLoadGlobalS16(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +Id EmitLoadGlobalS16(EmitContext& ctx, Id address) { + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobalS16, ctx.U32[1], address, + ctx.u32_zero_value); } Id EmitLoadGlobal32(EmitContext& ctx, Id address) { - if (ctx.profile.support_int64) { - return ctx.OpFunctionCall(ctx.U32[1], ctx.load_global_func_u32, address); - } - LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation"); - return ctx.Const(0u); + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobal32, ctx.U32[1], address, ctx.u32_zero_value); } Id EmitLoadGlobal64(EmitContext& ctx, Id address) { - if (ctx.profile.support_int64) { - return ctx.OpFunctionCall(ctx.U32[2], ctx.load_global_func_u32x2, address); - } - LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation"); - return ctx.Const(0u, 0u); + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobal64, ctx.U32[2], address, ctx.u32_zero_value); } Id EmitLoadGlobal128(EmitContext& ctx, Id address) { - if (ctx.profile.support_int64) { - return ctx.OpFunctionCall(ctx.U32[4], ctx.load_global_func_u32x4, address); - } - LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation"); - return ctx.Const(0u, 0u, 0u, 0u); + return ctx.CallGlobalMemory(IR::Opcode::LoadGlobal128, ctx.U32[4], address, + ctx.u32_zero_value); } -void EmitWriteGlobalU8(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +void EmitWriteGlobalU8(EmitContext& ctx, Id address, Id value) { + ctx.CallGlobalMemory(IR::Opcode::WriteGlobalU8, ctx.void_id, address, value); } -void EmitWriteGlobalS8(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +void EmitWriteGlobalS8(EmitContext& ctx, Id address, Id value) { + ctx.CallGlobalMemory(IR::Opcode::WriteGlobalS8, ctx.void_id, address, value); } -void EmitWriteGlobalU16(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +void EmitWriteGlobalU16(EmitContext& ctx, Id address, Id value) { + ctx.CallGlobalMemory(IR::Opcode::WriteGlobalU16, ctx.void_id, address, value); } -void EmitWriteGlobalS16(EmitContext&) { - throw NotImplementedException("SPIR-V Instruction"); +void EmitWriteGlobalS16(EmitContext& ctx, Id address, Id value) { + ctx.CallGlobalMemory(IR::Opcode::WriteGlobalS16, ctx.void_id, address, value); } void EmitWriteGlobal32(EmitContext& ctx, Id address, Id value) { - if (ctx.profile.support_int64) { - ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32, address, value); - return; - } - LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation"); + ctx.CallGlobalMemory(IR::Opcode::WriteGlobal32, ctx.void_id, address, value); } void EmitWriteGlobal64(EmitContext& ctx, Id address, Id value) { - if (ctx.profile.support_int64) { - ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32x2, address, value); - return; - } - LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation"); + ctx.CallGlobalMemory(IR::Opcode::WriteGlobal64, ctx.void_id, address, value); } void EmitWriteGlobal128(EmitContext& ctx, Id address, Id value) { - if (ctx.profile.support_int64) { - ctx.OpFunctionCall(ctx.void_id, ctx.write_global_func_u32x4, address, value); - return; - } - LOG_WARNING(Shader_SPIRV, "Int64 not supported, ignoring memory operation"); + ctx.CallGlobalMemory(IR::Opcode::WriteGlobal128, ctx.void_id, address, value); } Id EmitLoadStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Value& offset) { @@ -239,7 +217,7 @@ void EmitWriteStorageU8(EmitContext& ctx, const IR::Value& binding, const IR::Va WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U8, value), ctx.storage_types.U8, sizeof(u8), &StorageDefinitions::U8); } else { - WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u)); + WriteStorageBits(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u)); } } @@ -250,7 +228,7 @@ void EmitWriteStorageS8(EmitContext& ctx, const IR::Value& binding, const IR::Va WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S8, value), ctx.storage_types.S8, sizeof(s8), &StorageDefinitions::S8); } else { - WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u)); + WriteStorageBits(ctx, binding, offset, value, ctx.BitOffset8(offset), ctx.Const(8u)); } } @@ -261,7 +239,7 @@ void EmitWriteStorageU16(EmitContext& ctx, const IR::Value& binding, const IR::V WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.U16, value), ctx.storage_types.U16, sizeof(u16), &StorageDefinitions::U16); } else { - WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u)); + WriteStorageBits(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u)); } } @@ -272,7 +250,7 @@ void EmitWriteStorageS16(EmitContext& ctx, const IR::Value& binding, const IR::V WriteStorage(ctx, binding, offset, ctx.OpSConvert(ctx.S16, value), ctx.storage_types.S16, sizeof(s16), &StorageDefinitions::S16); } else { - WriteStorageByCasLoop(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u)); + WriteStorageBits(ctx, binding, offset, value, ctx.BitOffset16(offset), ctx.Const(16u)); } } diff --git a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp index ffd5d9c29f..8d83509961 100644 --- a/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp +++ b/src/shader_recompiler/backend/spirv/emit_spirv_warp.cpp @@ -77,20 +77,57 @@ Id GetMaxThreadId(EmitContext& ctx, Id thread_id, Id clamp, Id segmentation_mask return ComputeMaxThreadId(ctx, min_thread_id, clamp, not_seg_mask); } -Id SelectValue(EmitContext& ctx, Id in_range, Id value, Id src_thread_id) { - if (!StageSupportsSubgroups(ctx)) { - return value; +Id HostThreadId(EmitContext& ctx, Id thread_id) { + if (!ctx.profile.warp_size_potentially_larger_than_guest) { + return thread_id; } - return ctx.OpSelect( - ctx.U32[1], in_range, - ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id), value); -} - -Id AddPartitionBase(EmitContext& ctx, Id thread_id) { const Id partition_idx{ctx.OpShiftRightLogical(ctx.U32[1], GetThreadId(ctx), ctx.Const(5u))}; const Id partition_base{ctx.OpShiftLeftLogical(ctx.U32[1], partition_idx, ctx.Const(5u))}; return ctx.OpIAdd(ctx.U32[1], thread_id, partition_base); } + +Id GuestLane(EmitContext& ctx, Id index) { + return ctx.OpBitwiseAnd(ctx.U32[1], index, ctx.Const(31U)); +} + +Id ShuffleAbsolute(EmitContext& ctx, Id value, Id src_thread_id) { + if (!ctx.profile.has_broken_spirv_subgroup_shuffle) { + return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id); + } + Id result{ctx.u32_zero_value}; + for (u32 lane = 0; lane < ctx.profile.max_subgroup_size; ++lane) { + const Id read{ + ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, ctx.Const(lane))}; + const Id matches{ctx.OpIEqual(ctx.U1, src_thread_id, ctx.Const(lane))}; + result = ctx.OpSelect(ctx.U32[1], matches, read, result); + } + return result; +} + +Id ShuffleRelative(EmitContext& ctx, Id value, Id delta, Id src_thread_id, spv::Op op) { + if (!ctx.profile.support_shuffle_relative) { + return ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id)); + } + const Id result{ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, delta)}; + ctx.opcode_rewrites.emplace_back(result.value, op); + return result; +} + +Id BroadcastLane(EmitContext& ctx, Id value, u32 lane) { + Id result{ + ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, ctx.Const(lane))}; + if (!ctx.profile.warp_size_potentially_larger_than_guest) { + return result; + } + const Id partition_idx{ctx.OpShiftRightLogical(ctx.U32[1], GetThreadId(ctx), ctx.Const(5u))}; + for (u32 base = 32; base < ctx.profile.max_subgroup_size; base += 32) { + const Id read{ctx.OpGroupNonUniformBroadcast(ctx.U32[1], SubgroupScope(ctx), value, + ctx.Const(base + lane))}; + const Id matches{ctx.OpIEqual(ctx.U1, partition_idx, ctx.Const(base >> 5))}; + result = ctx.OpSelect(ctx.U32[1], matches, read, result); + } + return result; +} } // Anonymous namespace Id EmitLaneId(EmitContext& ctx) { @@ -203,61 +240,75 @@ Id EmitShuffleIndex(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id cla const Id min_thread_id{ComputeMinThreadId(ctx, thread_id, segmentation_mask)}; const Id max_thread_id{ComputeMaxThreadId(ctx, min_thread_id, clamp, not_seg_mask)}; - const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], index, not_seg_mask)}; - Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], lhs, min_thread_id)}; + const Id lhs{ctx.OpBitwiseAnd(ctx.U32[1], GuestLane(ctx, index), not_seg_mask)}; + const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], lhs, min_thread_id)}; const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)}; - if (ctx.profile.warp_size_potentially_larger_than_guest) { - src_thread_id = AddPartitionBase(ctx, src_thread_id); - } - SetInBoundsFlag(inst, in_range); - return SelectValue(ctx, in_range, value, src_thread_id); + if (!StageSupportsSubgroups(ctx)) { + return value; + } + const IR::Value lane{inst->Arg(1).Resolve()}; + const IR::Value segment{inst->Arg(3).Resolve()}; + if (lane.IsImmediate() && segment.IsImmediate() && segment.U32() == 0) { + return ctx.OpSelect(ctx.U32[1], in_range, BroadcastLane(ctx, value, lane.U32() & 31), + value); + } + const Id shuffled{ShuffleAbsolute(ctx, value, HostThreadId(ctx, src_thread_id))}; + return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value); } Id EmitShuffleUp(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp, Id segmentation_mask) { + if (!StageSupportsSubgroups(ctx)) { + SetInBoundsFlag(inst, ctx.false_value); + return value; + } + const Id delta{GuestLane(ctx, index)}; const Id thread_id{EmitLaneId(ctx)}; const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)}; - Id src_thread_id{ctx.OpISub(ctx.U32[1], thread_id, index)}; + const Id src_thread_id{ctx.OpISub(ctx.U32[1], thread_id, delta)}; const Id in_range{ctx.OpSGreaterThanEqual(ctx.U1, src_thread_id, max_thread_id)}; - if (ctx.profile.warp_size_potentially_larger_than_guest) { - src_thread_id = AddPartitionBase(ctx, src_thread_id); - } - SetInBoundsFlag(inst, in_range); - return SelectValue(ctx, in_range, value, src_thread_id); + const Id shuffled{ShuffleRelative(ctx, value, delta, src_thread_id, + spv::Op::OpGroupNonUniformShuffleUp)}; + return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value); } Id EmitShuffleDown(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp, Id segmentation_mask) { + if (!StageSupportsSubgroups(ctx)) { + SetInBoundsFlag(inst, ctx.false_value); + return value; + } + const Id delta{GuestLane(ctx, index)}; const Id thread_id{EmitLaneId(ctx)}; const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)}; - Id src_thread_id{ctx.OpIAdd(ctx.U32[1], thread_id, index)}; + const Id src_thread_id{ctx.OpIAdd(ctx.U32[1], thread_id, delta)}; const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)}; - if (ctx.profile.warp_size_potentially_larger_than_guest) { - src_thread_id = AddPartitionBase(ctx, src_thread_id); - } - SetInBoundsFlag(inst, in_range); - return SelectValue(ctx, in_range, value, src_thread_id); + const Id shuffled{ShuffleRelative(ctx, value, delta, src_thread_id, + spv::Op::OpGroupNonUniformShuffleDown)}; + return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value); } Id EmitShuffleButterfly(EmitContext& ctx, IR::Inst* inst, Id value, Id index, Id clamp, Id segmentation_mask) { + if (!StageSupportsSubgroups(ctx)) { + SetInBoundsFlag(inst, ctx.false_value); + return value; + } + const Id mask{GuestLane(ctx, index)}; const Id thread_id{EmitLaneId(ctx)}; const Id max_thread_id{GetMaxThreadId(ctx, thread_id, clamp, segmentation_mask)}; - Id src_thread_id{ctx.OpBitwiseXor(ctx.U32[1], thread_id, index)}; + const Id src_thread_id{ctx.OpBitwiseXor(ctx.U32[1], thread_id, mask)}; const Id in_range{ctx.OpSLessThanEqual(ctx.U1, src_thread_id, max_thread_id)}; - if (ctx.profile.warp_size_potentially_larger_than_guest) { - src_thread_id = AddPartitionBase(ctx, src_thread_id); - } - SetInBoundsFlag(inst, in_range); - return SelectValue(ctx, in_range, value, src_thread_id); + const Id shuffled{ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, mask)}; + return ctx.OpSelect(ctx.U32[1], in_range, shuffled, value); } Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) { @@ -267,10 +318,16 @@ Id EmitQuadBroadcast(EmitContext& ctx, Id value, Id lane) { const Id base{ctx.OpBitwiseAnd(ctx.U32[1], GetThreadId(ctx), ctx.Const(~3u))}; const Id local_lane{ctx.OpBitwiseAnd(ctx.U32[1], lane, ctx.Const(3u))}; const Id src_thread_id{ctx.OpBitwiseOr(ctx.U32[1], base, local_lane)}; - return ctx.OpGroupNonUniformShuffle(ctx.U32[1], SubgroupScope(ctx), value, src_thread_id); + return ShuffleAbsolute(ctx, value, src_thread_id); } Id EmitQuadSwap(EmitContext& ctx, Id value, Id direction) { + if (ctx.profile.support_quad_shuffles) { + const Id result{ + ctx.OpGroupNonUniformQuadBroadcast(ctx.U32[1], SubgroupScope(ctx), value, direction)}; + ctx.opcode_rewrites.emplace_back(result.value, spv::Op::OpGroupNonUniformQuadSwap); + return result; + } const Id xor_mask{ctx.OpIAdd(ctx.U32[1], direction, ctx.Const(1u))}; return ctx.OpGroupNonUniformShuffleXor(ctx.U32[1], SubgroupScope(ctx), value, xor_mask); } diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp index b562e82983..f174251625 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.cpp @@ -486,8 +486,7 @@ EmitContext::EmitContext(const Profile& profile_, const RuntimeInfo& runtime_inf DefineTextures(program.info, texture_binding, bindings.texture_scaling_index); DefineImages(program.info, image_binding, bindings.image_scaling_index); DefineAttributeMemAccess(program.info); - DefineWriteStorageCasLoopFunction(program.info); - DefineGlobalMemoryFunctions(program.info); + DefineGlobalMemoryFunctions(program); DefineRescalingInput(program.info); DefineRenderArea(program.info); } @@ -532,6 +531,24 @@ Id EmitContext::BitOffset16(const IR::Value& offset) { return OpBitwiseAnd(U32[1], OpShiftLeftLogical(U32[1], Def(offset), Const(3u)), Const(16u)); } +Id EmitContext::CallGlobalMemory(IR::Opcode opcode, Id result_type, Id address, Id value) { + if (profile.support_int64) { + return OpFunctionCall(result_type, global_memory_funcs.at(opcode), address, value); + } + if (result_type.value == void_id.value) { + return Id{}; + } + return ConstantNull(result_type); +} + +void EmitContext::AtomicBitFieldInsert(Id pointer, Id value, Id offset, Id count) { + const Id scope{Const(static_cast(spv::Scope::Device))}; + const Id mask{OpBitFieldInsert(U32[1], u32_zero_value, Const(0xFFFFFFFFU), offset, count)}; + const Id bits{OpBitFieldInsert(U32[1], u32_zero_value, value, offset, count)}; + OpAtomicAnd(U32[1], pointer, scope, u32_zero_value, OpNot(U32[1], mask)); + OpAtomicOr(U32[1], pointer, scope, u32_zero_value, bits); +} + void EmitContext::DefineCommonTypes(const Info& info) { void_id = TypeVoid(); @@ -889,135 +906,256 @@ void EmitContext::DefineAttributeMemAccess(const Info& info) { } } -void EmitContext::DefineWriteStorageCasLoopFunction(const Info& info) { - if (profile.support_int8 && profile.support_int16) { - return; - } - if (!info.uses_int8 && !info.uses_int16) { - return; - } - - AddCapability(spv::Capability::VariablePointersStorageBuffer); - - const Id ptr_type{TypePointer(spv::StorageClass::StorageBuffer, U32[1])}; - const Id func_type{TypeFunction(void_id, ptr_type, U32[1], U32[1], U32[1])}; - const Id func{OpFunction(void_id, spv::FunctionControlMask::MaskNone, func_type)}; - const Id pointer{OpFunctionParameter(ptr_type)}; - const Id value{OpFunctionParameter(U32[1])}; - const Id bit_offset{OpFunctionParameter(U32[1])}; - const Id bit_count{OpFunctionParameter(U32[1])}; - - AddLabel(); - const Id scope_device{Const(1u)}; - const Id ordering_relaxed{u32_zero_value}; - const Id body_label{OpLabel()}; - const Id continue_label{OpLabel()}; - const Id endloop_label{OpLabel()}; - const Id beginloop_label{OpLabel()}; - OpBranch(beginloop_label); - - AddLabel(beginloop_label); - OpLoopMerge(endloop_label, continue_label, spv::LoopControlMask::MaskNone); - OpBranch(body_label); - - AddLabel(body_label); - const Id expected_value{OpLoad(U32[1], pointer)}; - const Id desired_value{OpBitFieldInsert(U32[1], expected_value, value, bit_offset, bit_count)}; - const Id actual_value{OpAtomicCompareExchange(U32[1], pointer, scope_device, ordering_relaxed, - ordering_relaxed, desired_value, expected_value)}; - const Id store_successful{OpIEqual(U1, expected_value, actual_value)}; - OpBranchConditional(store_successful, endloop_label, continue_label); - - AddLabel(endloop_label); - OpReturn(); - - AddLabel(continue_label); - OpBranch(beginloop_label); - - OpFunctionEnd(); - - write_storage_cas_loop_func = func; -} - -void EmitContext::DefineGlobalMemoryFunctions(const Info& info) { +void EmitContext::DefineGlobalMemoryFunctions(const IR::Program& program) { + const Info& info{program.info}; if (!info.uses_global_memory || !profile.support_int64) { return; } - using DefPtr = Id StorageDefinitions::*; const Id zero{u32_zero_value}; - const auto define_body{[&](DefPtr ssbo_member, Id addr, Id element_pointer, u32 shift, - auto&& callback) { + const Id scope{Const(static_cast(spv::Scope::Device))}; + const Id align_mask{Const(~(static_cast(profile.min_ssbo_alignment) - 1U))}; + const auto word_pointer{[&](Id ssbo, Id word, u32 element) { + return OpAccessChain(storage_types.U32.element, ssbo, zero, + OpIAdd(U32[1], word, Const(element))); + }}; + const auto cbuf_word{[&](u32 index, u32 offset) { + if (profile.support_descriptor_aliasing) { + return OpLoad(U32[1], OpAccessChain(uniform_types.U32, cbufs[index].U32, zero, + Const(offset / 4))); + } + const Id vector{OpLoad(U32[4], OpAccessChain(uniform_types.U32x4, cbufs[index].U32x4, + zero, Const(offset / 16)))}; + return OpCompositeExtract(U32[1], vector, (offset / 4) % 4); + }}; + const auto bits{[&](Id offset, u32 count) { + return OpBitwiseAnd(U32[1], OpShiftLeftLogical(U32[1], offset, Const(3U)), + Const(32U - count)); + }}; + const auto define{[&](IR::Opcode opcode, Id result_type, Id value_type, auto&& callback) { + const std::array params{U64, value_type}; + const Id func{OpFunction(result_type, spv::FunctionControlMask::MaskNone, + TypeFunction(result_type, params))}; + const Id addr{OpFunctionParameter(U64)}; + const Id value{OpFunctionParameter(value_type)}; + const bool returns_value{result_type.value != void_id.value}; AddLabel(); - const size_t num_buffers{info.storage_buffers_descriptors.size()}; - for (size_t index = 0; index < num_buffers; ++index) { - if (!info.nvn_buffer_used[index]) { + const Id addr_words{OpBitcast(U32[2], addr)}; + const Id addr_low{OpCompositeExtract(U32[1], addr_words, 0U)}; + const Id addr_high{OpCompositeExtract(U32[1], addr_words, 1U)}; + for (size_t index = 0; index < info.storage_buffers_descriptors.size(); ++index) { + const auto& desc{info.storage_buffers_descriptors[index]}; + if (!desc.is_global_fallback) { continue; } - const auto& ssbo{info.storage_buffers_descriptors[index]}; - const Id ssbo_addr_cbuf_offset{Const(ssbo.cbuf_offset / 8)}; - const Id ssbo_size_cbuf_offset{Const(ssbo.cbuf_offset / 4 + 2)}; - const Id ssbo_addr_pointer{OpAccessChain( - uniform_types.U32x2, cbufs[ssbo.cbuf_index].U32x2, zero, ssbo_addr_cbuf_offset)}; - const Id ssbo_size_pointer{OpAccessChain(uniform_types.U32, cbufs[ssbo.cbuf_index].U32, - zero, ssbo_size_cbuf_offset)}; - - const u64 ssbo_align_mask{~(profile.min_ssbo_alignment - 1U)}; - const Id unaligned_addr{OpBitcast(U64, OpLoad(U32[2], ssbo_addr_pointer))}; - const Id ssbo_addr{OpBitwiseAnd(U64, unaligned_addr, Constant(U64, ssbo_align_mask))}; - const Id ssbo_size{OpUConvert(U64, OpLoad(U32[1], ssbo_size_pointer))}; - const Id ssbo_end{OpIAdd(U64, ssbo_addr, ssbo_size)}; - const Id cond{OpLogicalAnd(U1, OpUGreaterThanEqual(U1, addr, ssbo_addr), - OpULessThan(U1, addr, ssbo_end))}; + const Id ssbo_low{ + OpBitwiseAnd(U32[1], cbuf_word(desc.cbuf_index, desc.cbuf_offset), align_mask)}; + const Id ssbo_high{cbuf_word(desc.cbuf_index, desc.cbuf_offset + 4)}; + const Id ssbo_size{cbuf_word(desc.cbuf_index, desc.cbuf_offset + 8)}; + const Id offset{OpISub(U32[1], addr_low, ssbo_low)}; + const Id borrow{ + OpSelect(U32[1], OpULessThan(U1, addr_low, ssbo_low), Const(1U), zero)}; + const Id cond{OpLogicalAnd(U1, OpULessThan(U1, offset, ssbo_size), + OpIEqual(U1, OpISub(U32[1], addr_high, borrow), ssbo_high))}; const Id then_label{OpLabel()}; const Id else_label{OpLabel()}; OpSelectionMerge(else_label, spv::SelectionControlMask::MaskNone); OpBranchConditional(cond, then_label, else_label); AddLabel(then_label); - const Id ssbo_id{ssbos[index].*ssbo_member}; - const Id ssbo_offset{OpUConvert(U32[1], OpISub(U64, addr, ssbo_addr))}; - const Id ssbo_index{OpShiftRightLogical(U32[1], ssbo_offset, Const(shift))}; - const Id ssbo_pointer{OpAccessChain(element_pointer, ssbo_id, zero, ssbo_index)}; - callback(ssbo_pointer); + const Id word{OpShiftRightLogical(U32[1], offset, Const(2U))}; + const Id result{callback(ssbos[index], word, offset, value)}; + if (returns_value) { + OpReturnValue(result); + } else { + OpReturn(); + } AddLabel(else_label); } - }}; - const auto define_load{[&](DefPtr ssbo_member, Id element_pointer, Id type, u32 shift) { - const Id function_type{TypeFunction(type, U64)}; - const Id func_id{OpFunction(type, spv::FunctionControlMask::MaskNone, function_type)}; - const Id addr{OpFunctionParameter(U64)}; - define_body(ssbo_member, addr, element_pointer, shift, - [&](Id ssbo_pointer) { OpReturnValue(OpLoad(type, ssbo_pointer)); }); - OpReturnValue(ConstantNull(type)); - OpFunctionEnd(); - return func_id; - }}; - const auto define_write{[&](DefPtr ssbo_member, Id element_pointer, Id type, u32 shift) { - const Id function_type{TypeFunction(void_id, U64, type)}; - const Id func_id{OpFunction(void_id, spv::FunctionControlMask::MaskNone, function_type)}; - const Id addr{OpFunctionParameter(U64)}; - const Id data{OpFunctionParameter(type)}; - define_body(ssbo_member, addr, element_pointer, shift, [&](Id ssbo_pointer) { - OpStore(ssbo_pointer, data); + if (returns_value) { + OpReturnValue(ConstantNull(result_type)); + } else { OpReturn(); - }); - OpReturn(); + } OpFunctionEnd(); - return func_id; + global_memory_funcs.emplace(opcode, func); }}; - const auto define{ - [&](DefPtr ssbo_member, const StorageTypeDefinition& type_def, Id type, size_t size) { - const Id element_type{type_def.element}; - const u32 shift{static_cast(std::countr_zero(size))}; - const Id load_func{define_load(ssbo_member, element_type, type, shift)}; - const Id write_func{define_write(ssbo_member, element_type, type, shift)}; - return std::make_pair(load_func, write_func); - }}; - std::tie(load_global_func_u32, write_global_func_u32) = - define(&StorageDefinitions::U32, storage_types.U32, U32[1], sizeof(u32)); - std::tie(load_global_func_u32x2, write_global_func_u32x2) = - define(&StorageDefinitions::U32x2, storage_types.U32x2, U32[2], sizeof(u32[2])); - std::tie(load_global_func_u32x4, write_global_func_u32x4) = - define(&StorageDefinitions::U32x4, storage_types.U32x4, U32[4], sizeof(u32[4])); + const auto vector_pointer{[&](const StorageDefinitions& ssbo, Id word, u32 count) { + if (count == 2) { + return OpAccessChain(storage_types.U32x2.element, ssbo.U32x2, zero, + OpShiftRightLogical(U32[1], word, Const(1U))); + } + return OpAccessChain(storage_types.U32x4.element, ssbo.U32x4, zero, + OpShiftRightLogical(U32[1], word, Const(2U))); + }}; + const auto load{[&](Id type, u32 count) { + return [&, type, count](const StorageDefinitions& ssbo, Id word, Id, Id) { + if (count > 1 && profile.support_descriptor_aliasing) { + return OpLoad(type, vector_pointer(ssbo, word, count)); + } + std::array words{}; + for (u32 element = 0; element < count; ++element) { + words[element] = OpLoad(U32[1], word_pointer(ssbo.U32, word, element)); + } + if (count == 1) { + return words[0]; + } + return OpCompositeConstruct(type, std::span(words.data(), count)); + }; + }}; + const auto store{[&](u32 count) { + return [&, count](const StorageDefinitions& ssbo, Id word, Id, Id value) { + if (count > 1 && profile.support_descriptor_aliasing) { + OpStore(vector_pointer(ssbo, word, count), value); + return Id{}; + } + if (count == 1) { + OpStore(word_pointer(ssbo.U32, word, 0), value); + return Id{}; + } + for (u32 element = 0; element < count; ++element) { + OpStore(word_pointer(ssbo.U32, word, element), + OpCompositeExtract(U32[1], value, element)); + } + return Id{}; + }; + }}; + const auto extract{[&](bool is_signed, u32 count) { + return [&, is_signed, count](const StorageDefinitions& ssbo, Id word, Id offset, Id) { + const Id loaded{OpLoad(U32[1], word_pointer(ssbo.U32, word, 0))}; + if (is_signed) { + return OpBitFieldSExtract(U32[1], loaded, bits(offset, count), Const(count)); + } + return OpBitFieldUExtract(U32[1], loaded, bits(offset, count), Const(count)); + }; + }}; + const auto insert{[&](u32 count) { + return [&, count](const StorageDefinitions& ssbo, Id word, Id offset, Id value) { + AtomicBitFieldInsert(word_pointer(ssbo.U32, word, 0), value, bits(offset, count), + Const(count)); + return Id{}; + }; + }}; + const auto atomic{[&](Id (Sirit::Module::*func)(Id, Id, Id, Id, Id)) { + return [&, func](const StorageDefinitions& ssbo, Id word, Id, Id value) { + return (this->*func)(U32[1], word_pointer(ssbo.U32, word, 0), scope, zero, value); + }; + }}; + const auto cas{[&](Id type, Id helper) { + return [&, type, helper](const StorageDefinitions& ssbo, Id word, Id, Id value) { + return OpFunctionCall(type, helper, word, value, ssbo.U32); + }; + }}; + const auto packed{[&](bool is_half, Id helper) { + return [&, is_half, helper](const StorageDefinitions& ssbo, Id word, Id, Id value) { + if (is_half) { + return OpBitcast(U32[1], OpFunctionCall(F16[2], helper, word, value, ssbo.U32)); + } + return OpPackHalf2x16(U32[1], OpFunctionCall(F32[2], helper, word, value, ssbo.U32)); + }; + }}; + for (const IR::Block* const block : program.post_order_blocks) { + for (const IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if (global_memory_funcs.contains(opcode)) { + continue; + } + switch (opcode) { + case IR::Opcode::LoadGlobalU8: + define(opcode, U32[1], U32[1], extract(false, 8)); + break; + case IR::Opcode::LoadGlobalS8: + define(opcode, U32[1], U32[1], extract(true, 8)); + break; + case IR::Opcode::LoadGlobalU16: + define(opcode, U32[1], U32[1], extract(false, 16)); + break; + case IR::Opcode::LoadGlobalS16: + define(opcode, U32[1], U32[1], extract(true, 16)); + break; + case IR::Opcode::LoadGlobal32: + define(opcode, U32[1], U32[1], load(U32[1], 1)); + break; + case IR::Opcode::LoadGlobal64: + define(opcode, U32[2], U32[1], load(U32[2], 2)); + break; + case IR::Opcode::LoadGlobal128: + define(opcode, U32[4], U32[1], load(U32[4], 4)); + break; + case IR::Opcode::WriteGlobalU8: + case IR::Opcode::WriteGlobalS8: + define(opcode, void_id, U32[1], insert(8)); + break; + case IR::Opcode::WriteGlobalU16: + case IR::Opcode::WriteGlobalS16: + define(opcode, void_id, U32[1], insert(16)); + break; + case IR::Opcode::WriteGlobal32: + define(opcode, void_id, U32[1], store(1)); + break; + case IR::Opcode::WriteGlobal64: + define(opcode, void_id, U32[2], store(2)); + break; + case IR::Opcode::WriteGlobal128: + define(opcode, void_id, U32[4], store(4)); + break; + case IR::Opcode::GlobalAtomicIAdd32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicIAdd)); + break; + case IR::Opcode::GlobalAtomicSMin32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicSMin)); + break; + case IR::Opcode::GlobalAtomicUMin32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicUMin)); + break; + case IR::Opcode::GlobalAtomicSMax32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicSMax)); + break; + case IR::Opcode::GlobalAtomicUMax32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicUMax)); + break; + case IR::Opcode::GlobalAtomicAnd32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicAnd)); + break; + case IR::Opcode::GlobalAtomicOr32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicOr)); + break; + case IR::Opcode::GlobalAtomicXor32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicXor)); + break; + case IR::Opcode::GlobalAtomicExchange32: + define(opcode, U32[1], U32[1], atomic(&Sirit::Module::OpAtomicExchange)); + break; + case IR::Opcode::GlobalAtomicInc32: + define(opcode, U32[1], U32[1], cas(U32[1], increment_cas_ssbo)); + break; + case IR::Opcode::GlobalAtomicDec32: + define(opcode, U32[1], U32[1], cas(U32[1], decrement_cas_ssbo)); + break; + case IR::Opcode::GlobalAtomicAddF32: + define(opcode, F32[1], F32[1], cas(F32[1], f32_add_cas)); + break; + case IR::Opcode::GlobalAtomicAddF16x2: + define(opcode, U32[1], F16[2], packed(true, f16x2_add_cas)); + break; + case IR::Opcode::GlobalAtomicMinF16x2: + define(opcode, U32[1], F16[2], packed(true, f16x2_min_cas)); + break; + case IR::Opcode::GlobalAtomicMaxF16x2: + define(opcode, U32[1], F16[2], packed(true, f16x2_max_cas)); + break; + case IR::Opcode::GlobalAtomicAddF32x2: + define(opcode, U32[1], F32[2], packed(false, f32x2_add_cas)); + break; + case IR::Opcode::GlobalAtomicMinF32x2: + define(opcode, U32[1], F32[2], packed(false, f32x2_min_cas)); + break; + case IR::Opcode::GlobalAtomicMaxF32x2: + define(opcode, U32[1], F32[2], packed(false, f32x2_max_cas)); + break; + default: + break; + } + } + } } void EmitContext::DefineRescalingInput(const Info& info) { diff --git a/src/shader_recompiler/backend/spirv/spirv_emit_context.h b/src/shader_recompiler/backend/spirv/spirv_emit_context.h index 7edc104708..68761e1f1a 100644 --- a/src/shader_recompiler/backend/spirv/spirv_emit_context.h +++ b/src/shader_recompiler/backend/spirv/spirv_emit_context.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include "common/container/unordered_set.h" @@ -173,6 +174,9 @@ public: [[nodiscard]] Id BitOffset8(const IR::Value& offset); [[nodiscard]] Id BitOffset16(const IR::Value& offset); + Id CallGlobalMemory(IR::Opcode opcode, Id result_type, Id address, Id value); + void AtomicBitFieldInsert(Id pointer, Id value, Id offset, Id count); + Id Const(u32 value) { return Constant(U32[1], value); } @@ -335,14 +339,7 @@ public: Id f32x2_min_cas{}; Id f32x2_max_cas{}; - Id write_storage_cas_loop_func{}; - - Id load_global_func_u32{}; - Id load_global_func_u32x2{}; - Id load_global_func_u32x4{}; - Id write_global_func_u32{}; - Id write_global_func_u32x2{}; - Id write_global_func_u32x4{}; + std::unordered_map global_memory_funcs; bool need_input_position_indirect{}; Id input_position{}; @@ -361,6 +358,7 @@ public: Id frag_depth{}; std::vector interfaces; + std::vector> opcode_rewrites; Id load_const_func_u8{}; Id load_const_func_u16{}; @@ -392,8 +390,7 @@ private: void DefineTextures(const Info& info, u32& binding, u32& scaling_index); void DefineImages(const Info& info, u32& binding, u32& scaling_index); void DefineAttributeMemAccess(const Info& info); - void DefineWriteStorageCasLoopFunction(const Info& info); - void DefineGlobalMemoryFunctions(const Info& info); + void DefineGlobalMemoryFunctions(const IR::Program& program); void DefineRescalingInput(const Info& info); void DefineRescalingInputPushConstant(); void DefineRescalingInputUniformConstant(); diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/atomic_operations_global_memory.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/atomic_operations_global_memory.cpp index 75087777dd..9497c33e30 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/atomic_operations_global_memory.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/atomic_operations_global_memory.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -93,7 +93,7 @@ IR::U64 AtomOffset(TranslatorVisitor& v, u64 insn) { } const mem{insn}; const IR::U64 address{[&]() -> IR::U64 { - if (mem.e == 0) + if (mem.e == 0 || mem.addr_reg == IR::Reg::RZ) return v.ir.UConvert(64, v.X(mem.addr_reg)); return v.L(mem.addr_reg); }()}; @@ -108,14 +108,9 @@ IR::U64 AtomOffset(TranslatorVisitor& v, u64 insn) { return v.ir.IAdd(address, v.ir.Imm64(addr_offset)); } -// INC, DEC for U32/S32/U64 does nothing -// ADD, INC, DEC for S64 does nothing -// Only ADD does something for F32 -// Only ADD, MIN and MAX does something for F16x2 bool AtomOpNotApplicable(AtomSize size, AtomOp op) { // TODO: SAFEADD switch (size) { - case AtomSize::U32: case AtomSize::S32: case AtomSize::U64: return (op == AtomOp::INC || op == AtomOp::DEC); diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/barrier_operations.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/barrier_operations.cpp index aa494e24cc..c210f634b0 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/barrier_operations.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/barrier_operations.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -36,6 +36,12 @@ void TranslatorVisitor::DEPBAR(u64) { // DEPBAR is a no-op } +void TranslatorVisitor::CCTL(u64) {} + +void TranslatorVisitor::CCTLL(u64) {} + +void TranslatorVisitor::CCTLT(u64) {} + void TranslatorVisitor::BAR(u64 insn) { enum class Mode { RedPopc, diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/load_store_memory.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/load_store_memory.cpp index 8f2cf897d4..d700a05d59 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/load_store_memory.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/load_store_memory.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -55,7 +58,7 @@ IR::U64 Address(TranslatorVisitor& v, u64 insn) { } const mem{insn}; const IR::U64 address{[&]() -> IR::U64 { - if (mem.e == 0) { + if (mem.e == 0 || mem.addr_reg == IR::Reg::RZ) { // LDG/STG without .E uses a 32-bit pointer, zero-extend it return v.ir.UConvert(64, v.X(mem.addr_reg)); } diff --git a/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp b/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp index 369af8e03a..5e12c37430 100644 --- a/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate/impl/not_implemented.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -43,18 +43,6 @@ void TranslatorVisitor::CAL(u64) { // CAL is a no-op } -void TranslatorVisitor::CCTL(u64) { - ThrowNotImplemented(Opcode::CCTL); -} - -void TranslatorVisitor::CCTLL(u64) { - ThrowNotImplemented(Opcode::CCTLL); -} - -void TranslatorVisitor::CCTLT(u64) { - ThrowNotImplemented(Opcode::CCTLT); -} - void TranslatorVisitor::CONT(u64) { ThrowNotImplemented(Opcode::CONT); } diff --git a/src/shader_recompiler/frontend/maxwell/translate_program.cpp b/src/shader_recompiler/frontend/maxwell/translate_program.cpp index ebc5a825dd..d1d45d6b33 100644 --- a/src/shader_recompiler/frontend/maxwell/translate_program.cpp +++ b/src/shader_recompiler/frontend/maxwell/translate_program.cpp @@ -5,7 +5,10 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include +#include #include +#include #include #include @@ -118,9 +121,12 @@ void AddNVNStorageBuffers(IR::Program& program) { continue; } const u32 offset{base + index * descriptor_size}; - const auto it{std::ranges::find(descs, offset, &StorageBufferDescriptor::cbuf_offset)}; + const auto it{std::ranges::find_if(descs, [&](const StorageBufferDescriptor& desc) { + return desc.cbuf_index == driver_cbuf && desc.cbuf_offset == offset; + })}; if (it != descs.end()) { it->is_written |= program.info.stores_global_memory; + it->is_global_fallback = true; continue; } descs.push_back({ @@ -128,6 +134,7 @@ void AddNVNStorageBuffers(IR::Program& program) { .cbuf_offset = offset, .count = 1, .is_written = program.info.stores_global_memory, + .is_global_fallback = true, }); } } @@ -172,7 +179,17 @@ std::map GenerateLegacyToGenericMappings( void EmitGeometryPassthrough(IR::IREmitter& ir, const IR::Program& program, const Shader::VaryingState& passthrough_mask, bool passthrough_position, - std::optional passthrough_layer_attr) { + std::optional passthrough_layer_attr, + std::optional viewport_mask_reg) { + constexpr std::array CULLED_POSITION{2.0f, 2.0f, 2.0f, 1.0f}; + IR::U1 culled{ir.Imm1(false)}; + IR::U32 viewport{ir.Imm32(0)}; + if (viewport_mask_reg) { + const IR::U32 mask{ir.GetReg(*viewport_mask_reg)}; + const IR::U32 lowest_bit{ir.BitwiseAnd(mask, IR::U32{ir.INeg(mask)})}; + culled = ir.IEqual(mask, ir.Imm32(0)); + viewport = IR::U32{ir.Select(culled, ir.Imm32(0), ir.FindUMsb(lowest_bit))}; + } for (u32 i = 0; i < program.output_vertices; i++) { // Assign generics from input for (u32 j = 0; j < 32; j++) { @@ -190,10 +207,19 @@ void EmitGeometryPassthrough(IR::IREmitter& ir, const IR::Program& program, if (passthrough_position) { // Assign position from input const IR::Attribute attr = IR::Attribute::PositionX; - ir.SetAttribute(attr + 0, ir.GetAttribute(attr + 0, ir.Imm32(i)), ir.Imm32(0)); - ir.SetAttribute(attr + 1, ir.GetAttribute(attr + 1, ir.Imm32(i)), ir.Imm32(0)); - ir.SetAttribute(attr + 2, ir.GetAttribute(attr + 2, ir.Imm32(i)), ir.Imm32(0)); - ir.SetAttribute(attr + 3, ir.GetAttribute(attr + 3, ir.Imm32(i)), ir.Imm32(0)); + for (u32 component = 0; component < 4; ++component) { + IR::F32 value{ir.GetAttribute(attr + component, ir.Imm32(i))}; + if (viewport_mask_reg) { + value = IR::F32{ + ir.Select(culled, ir.Imm32(CULLED_POSITION[component]), value)}; + } + ir.SetAttribute(attr + component, value, ir.Imm32(0)); + } + } + + if (viewport_mask_reg) { + ir.SetAttribute(IR::Attribute::ViewportIndex, ir.BitCast(viewport), + ir.Imm32(0)); } if (passthrough_layer_attr) { @@ -219,19 +245,91 @@ u32 GetOutputTopologyVertices(OutputTopology output_topology) { } } +std::optional FindFreeRegister(const IR::Program& program) { + std::bitset used; + for (IR::Block* const block : program.blocks) { + for (const IR::Inst& inst : block->Instructions()) { + const IR::Opcode opcode{inst.GetOpcode()}; + if (opcode == IR::Opcode::GetRegister || opcode == IR::Opcode::SetRegister) { + used.set(IR::RegIndex(inst.Arg(0).Reg())); + } + } + } + for (size_t index = IR::NUM_USER_REGS; index-- > 0;) { + if (!used.test(index)) { + return static_cast(index); + } + } + return std::nullopt; +} + +std::optional LowerViewportMask(const IR::Program& program) { + const std::optional reg{FindFreeRegister(program)}; + if (!reg || program.blocks.empty()) { + return std::nullopt; + } + bool stores_mask{}; + for (IR::Block* const block : program.blocks) { + for (IR::Inst& inst : block->Instructions()) { + if (inst.GetOpcode() != IR::Opcode::SetAttribute || + inst.Arg(0).Attribute() != IR::Attribute::ViewportMask) { + continue; + } + IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)}; + ir.SetReg(*reg, ir.BitCast(IR::F32{inst.Arg(1)})); + inst.Invalidate(); + stores_mask = true; + } + } + if (!stores_mask) { + return std::nullopt; + } + IR::Block& entry{*program.blocks.front()}; + IR::IREmitter ir{entry, entry.begin()}; + ir.SetReg(*reg, ir.Imm32(1)); + return reg; +} + void LowerGeometryPassthrough(const IR::Program& program, const HostTranslateInfo& host_info) { + std::optional viewport_mask_reg; + if (!host_info.support_viewport_mask) { + viewport_mask_reg = LowerViewportMask(program); + } for (IR::Block* const block : program.blocks) { for (IR::Inst& inst : block->Instructions()) { if (inst.GetOpcode() == IR::Opcode::Epilogue) { IR::IREmitter ir{*block, IR::Block::InstructionList::s_iterator_to(inst)}; EmitGeometryPassthrough( ir, program, program.info.passthrough, - program.info.passthrough.AnyComponent(IR::Attribute::PositionX), {}); + program.info.passthrough.AnyComponent(IR::Attribute::PositionX), {}, + viewport_mask_reg); } } } } +void TightenOutputVertices(IR::Program& program) { + if (program.stage != Stage::Geometry || program.is_geometry_passthrough) { + return; + } + const bool has_loops = std::ranges::any_of(program.syntax_list, [](const auto& node) { + return node.type == IR::AbstractSyntaxNode::Type::Loop; + }); + if (has_loops) { + return; + } + u32 num_emits = 0; + for (IR::Block* const block : program.blocks) { + for (const IR::Inst& inst : block->Instructions()) { + if (inst.GetOpcode() == IR::Opcode::EmitVertex) { + ++num_emits; + } + } + } + if (num_emits != 0) { + program.output_vertices = (std::min)(program.output_vertices, num_emits); + } +} } // Anonymous namespace IR::Program TranslateProgram(ObjectPool& inst_pool, ObjectPool& block_pool, @@ -299,12 +397,14 @@ IR::Program TranslateProgram(ObjectPool& inst_pool, ObjectPool& inst_pool, IR::IREmitter ir{*current_block}; EmitGeometryPassthrough(ir, program, program.info.stores, true, - source_program.info.emulated_layer); + source_program.info.emulated_layer, std::nullopt); IR::Block* return_block{block_pool.Create(inst_pool)}; IR::IREmitter{*return_block}.Epilogue(); diff --git a/src/shader_recompiler/host_translate_info.h b/src/shader_recompiler/host_translate_info.h index 93ce881c2d..44df85ccb2 100644 --- a/src/shader_recompiler/host_translate_info.h +++ b/src/shader_recompiler/host_translate_info.h @@ -34,10 +34,12 @@ struct HostTranslateInfo { bool needs_demote_reorder{}; ///< True when the device needs DemoteToHelperInvocation reordered bool support_snorm_render_buffer{}; ///< True when the device supports SNORM render buffers bool support_viewport_index_layer{}; ///< True when the device supports gl_Layer in VS + bool support_viewport_mask{}; bool support_geometry_shader_passthrough{}; ///< True when the device supports geometry ///< passthrough shaders bool support_conditional_barrier{}; ///< True when the device supports barriers in conditional ///< control flow + bool single_lane_geometry_subgroups{}; void ApplyDescriptorLimitPolicy() noexcept { if (min_ssbo_alignment == 0) { diff --git a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp index 9010c2bf68..441058c64d 100644 --- a/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp +++ b/src/shader_recompiler/ir_opt/collect_shader_info_pass.cpp @@ -424,8 +424,8 @@ void VisitUsages(Info& info, IR::Inst& inst) { case IR::Opcode::LoadGlobal128: info.uses_int64 = true; info.uses_global_memory = true; - info.used_constant_buffer_types |= IR::Type::U32 | IR::Type::U32x2; - info.used_storage_buffer_types |= IR::Type::U32 | IR::Type::U32x2 | IR::Type::U32x4; + info.used_constant_buffer_types |= IR::Type::U32; + info.used_storage_buffer_types |= IR::Type::U32; break; case IR::Opcode::LoadLocal: case IR::Opcode::WriteLocal: @@ -632,6 +632,8 @@ void VisitUsages(Info& info, IR::Inst& inst) { case IR::Opcode::StorageAtomicExchange32: info.used_storage_buffer_types |= IR::Type::U32; break; + case IR::Opcode::LoadGlobal64: + case IR::Opcode::WriteGlobal64: case IR::Opcode::LoadStorage64: case IR::Opcode::WriteStorage64: case IR::Opcode::StorageAtomicIAdd32x2: @@ -645,6 +647,8 @@ void VisitUsages(Info& info, IR::Inst& inst) { case IR::Opcode::StorageAtomicExchange32x2: info.used_storage_buffer_types |= IR::Type::U32x2; break; + case IR::Opcode::LoadGlobal128: + case IR::Opcode::WriteGlobal128: case IR::Opcode::LoadStorage128: case IR::Opcode::WriteStorage128: info.used_storage_buffer_types |= IR::Type::U32x4; diff --git a/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp b/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp index 57c08588fb..279d32d9e4 100644 --- a/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp +++ b/src/shader_recompiler/ir_opt/constant_propagation_pass.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project @@ -414,6 +414,14 @@ void FoldSelect(IR::Inst& inst) { } } +void FoldAtomicWrap(IR::Inst& inst, IR::Opcode opcode, u32 addend) { + const IR::Value limit{inst.Arg(1)}; + if (limit.IsImmediate() && limit.U32() == 0xFFFFFFFFU) { + inst.ReplaceOpcode(opcode); + inst.SetArg(1, IR::Value{addend}); + } +} + void FoldFPAdd32(IR::Inst& inst) { if (FoldWhenAllImmediates(inst, [](f32 a, f32 b) { return a + b; })) { return; @@ -651,6 +659,29 @@ IR::Value GetThroughCast(IR::Value value, IR::Opcode expected_cast) { return value; } +u32 QuadButterflyMask(const IR::Inst& inst) { + if (inst.GetOpcode() == IR::Opcode::QuadSwap) { + const IR::Value direction{inst.Arg(1)}; + if (!direction.IsImmediate()) { + return 0; + } + return direction.U32() + 1; + } + if (inst.GetOpcode() != IR::Opcode::ShuffleButterfly) { + return 0; + } + const IR::Value index{inst.Arg(1)}; + const IR::Value clamp{inst.Arg(2)}; + const IR::Value segmentation_mask{inst.Arg(3)}; + if (!index.IsImmediate() || !clamp.IsImmediate() || !segmentation_mask.IsImmediate()) { + return 0; + } + if (clamp.U32() != 3 || segmentation_mask.U32() != 28) { + return 0; + } + return index.U32(); +} + void FoldFSwizzleAdd(IR::Block& block, IR::Inst& inst) { const IR::Value swizzle{inst.Arg(2)}; if (!swizzle.IsImmediate()) { @@ -666,7 +697,8 @@ void FoldFSwizzleAdd(IR::Block& block, IR::Inst& inst) { return; } IR::Inst* const inst2{value_1.InstRecursive()}; - if (inst2->GetOpcode() != IR::Opcode::ShuffleButterfly) { + const u32 lane_mask{QuadButterflyMask(*inst2)}; + if (lane_mask == 0) { return; } const IR::Value value_3{GetThroughCast(inst2->Arg(0).Resolve(), IR::Opcode::BitCastU32F32)}; @@ -678,24 +710,15 @@ void FoldFSwizzleAdd(IR::Block& block, IR::Inst& inst) { return; } } - const IR::Value index{inst2->Arg(1)}; - const IR::Value clamp{inst2->Arg(2)}; - const IR::Value segmentation_mask{inst2->Arg(3)}; - if (!index.IsImmediate() || !clamp.IsImmediate() || !segmentation_mask.IsImmediate()) { - return; - } - if (clamp.U32() != 3 || segmentation_mask.U32() != 28) { - return; - } if (swizzle_value == 0x99) { // DPdxFine - if (index.U32() == 1) { + if (lane_mask == 1) { IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)}; inst.ReplaceUsesWith(ir.DPdxFine(IR::F32{inst.Arg(1)})); } } else if (swizzle_value == 0xA5) { // DPdyFine - if (index.U32() == 2) { + if (lane_mask == 2) { IR::IREmitter ir{block, IR::Block::InstructionList::s_iterator_to(inst)}; inst.ReplaceUsesWith(ir.DPdyFine(IR::F32{inst.Arg(1)})); } @@ -709,6 +732,13 @@ bool FindGradient3DDerivatives(std::array& results, IR::Value coor const auto check_through_shuffle = [](IR::Value input, IR::Value& result) { const IR::Value value_1{GetThroughCast(input.Resolve(), IR::Opcode::BitCastF32U32)}; IR::Inst* const inst2{value_1.InstRecursive()}; + if (inst2->GetOpcode() == IR::Opcode::QuadBroadcast) { + if (!inst2->Arg(1).Resolve().IsImmediate()) { + return false; + } + result = GetThroughCast(inst2->Arg(0).Resolve(), IR::Opcode::BitCastU32F32); + return true; + } if (inst2->GetOpcode() != IR::Opcode::ShuffleIndex) { return false; } @@ -1082,6 +1112,14 @@ void ConstantPropagation(Environment& env, IR::Block& block, IR::Inst& inst) { IR::Opcode::CompositeInsertF16x4); case IR::Opcode::FSwizzleAdd: return FoldFSwizzleAdd(block, inst); + case IR::Opcode::GlobalAtomicInc32: + return FoldAtomicWrap(inst, IR::Opcode::GlobalAtomicIAdd32, 1U); + case IR::Opcode::GlobalAtomicDec32: + return FoldAtomicWrap(inst, IR::Opcode::GlobalAtomicIAdd32, 0xFFFFFFFFU); + case IR::Opcode::SharedAtomicInc32: + return FoldAtomicWrap(inst, IR::Opcode::SharedAtomicIAdd32, 1U); + case IR::Opcode::SharedAtomicDec32: + return FoldAtomicWrap(inst, IR::Opcode::SharedAtomicIAdd32, 0xFFFFFFFFU); case IR::Opcode::GetCbufF32: case IR::Opcode::GetCbufU32: if (env.HasHLEMacroState()) { diff --git a/src/shader_recompiler/ir_opt/geometry_compaction_pass.cpp b/src/shader_recompiler/ir_opt/geometry_compaction_pass.cpp new file mode 100644 index 0000000000..98cfa47401 --- /dev/null +++ b/src/shader_recompiler/ir_opt/geometry_compaction_pass.cpp @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include + +#include "shader_recompiler/frontend/ir/ir_emitter.h" +#include "shader_recompiler/host_translate_info.h" +#include "shader_recompiler/ir_opt/passes.h" + +namespace Shader::Optimization { +namespace { +constexpr int MAX_BALLOT_DEPTH = 8; + +struct SlotAtomic { + IR::Block* block; + IR::Inst* atomic; + IR::Inst* ballot; +}; + +IR::Inst* FindBallot(const IR::Value& value, int depth) { + if (depth > MAX_BALLOT_DEPTH || value.IsImmediate()) { + return nullptr; + } + IR::Inst* const inst{value.InstRecursive()}; + if (inst->GetOpcode() == IR::Opcode::SubgroupBallot) { + return inst; + } + if (inst->GetOpcode() == IR::Opcode::Phi) { + return nullptr; + } + for (size_t index = 0; index < inst->NumArgs(); ++index) { + if (IR::Inst* const ballot{FindBallot(inst->Arg(index), depth + 1)}) { + return ballot; + } + } + return nullptr; +} + +bool IsShuffle(IR::Opcode opcode) { + switch (opcode) { + case IR::Opcode::ShuffleIndex: + case IR::Opcode::ShuffleUp: + case IR::Opcode::ShuffleDown: + case IR::Opcode::ShuffleButterfly: + return true; + default: + return false; + } +} + +IR::U32 PrimitiveSlot(IR::IREmitter& ir, u32 invocations, const IR::U32& amount) { + IR::U32 key{ir.GetAttributeU32(IR::Attribute::PrimitiveId)}; + if (invocations > 1) { + key = ir.IAdd(ir.IMul(key, ir.Imm32(invocations)), ir.InvocationId()); + } + return ir.IMul(key, amount); +} + +bool MergesAtomic(const IR::Inst& phi, const IR::Block& block, const IR::Inst& atomic) { + for (size_t index = 0; index < phi.NumArgs(); ++index) { + const IR::Value arg{phi.Arg(index)}; + if (phi.PhiBlock(index) == &block && !arg.IsImmediate() && + arg.InstRecursive() == &atomic) { + return true; + } + } + return false; +} + +void RewriteMergedSlots(IR::Block& block, IR::Inst& atomic, u32 invocations) { + for (IR::Block* const successor : block.ImmSuccessors()) { + for (IR::Inst& phi : successor->Instructions()) { + if (phi.GetOpcode() != IR::Opcode::Phi || !MergesAtomic(phi, block, atomic)) { + continue; + } + for (size_t index = 0; index < phi.NumArgs(); ++index) { + const IR::Value arg{phi.Arg(index)}; + if (arg.IsImmediate() || !IsShuffle(arg.InstRecursive()->GetOpcode())) { + continue; + } + IR::IREmitter ir{*phi.PhiBlock(index)}; + const IR::U32 amount{arg.InstRecursive()->Arg(0)}; + phi.SetArg(index, PrimitiveSlot(ir, invocations, amount)); + } + } + } +} + +void RewriteAtomic(IR::Block& block, IR::Inst& atomic, u32 invocations) { + const auto insert_point{IR::Block::InstructionList::s_iterator_to(atomic)}; + IR::IREmitter ir{block, insert_point}; + const IR::U32 amount{atomic.Arg(2)}; + const IR::U32 slot{PrimitiveSlot(ir, invocations, amount)}; + block.PrependNewInst(insert_point, IR::Opcode::StorageAtomicUMax32, + {atomic.Arg(0), atomic.Arg(1), ir.IAdd(slot, amount)}); + atomic.ReplaceUsesWith(slot); +} + +void NeutralizePredicate(IR::Inst& ballot) { + const IR::Value pred{ballot.Arg(0)}; + if (!pred.IsImmediate()) { + pred.InstRecursive()->ReplaceUsesWith(IR::Value{true}); + } +} +} + +void GeometryCompactionPass(IR::Program& program, const HostTranslateInfo& host_info) { + if (program.stage != Stage::Geometry || !host_info.single_lane_geometry_subgroups) { + return; + } + boost::container::small_vector slot_atomics; + for (IR::Block* const block : program.post_order_blocks) { + for (IR::Inst& inst : block->Instructions()) { + if (inst.GetOpcode() != IR::Opcode::StorageAtomicIAdd32) { + continue; + } + if (IR::Inst* const ballot{FindBallot(inst.Arg(2), 0)}) { + slot_atomics.push_back({block, &inst, ballot}); + } + } + } + for (const SlotAtomic& slot_atomic : slot_atomics) { + RewriteMergedSlots(*slot_atomic.block, *slot_atomic.atomic, program.invocations); + RewriteAtomic(*slot_atomic.block, *slot_atomic.atomic, program.invocations); + NeutralizePredicate(*slot_atomic.ballot); + } +} + +} diff --git a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp index f49d54e7ab..bf40b0695b 100644 --- a/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp +++ b/src/shader_recompiler/ir_opt/global_memory_to_storage_buffer_pass.cpp @@ -4,14 +4,15 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include +#include #include #include #include "common/alignment.h" #include "shader_recompiler/frontend/ir/basic_block.h" -#include "shader_recompiler/frontend/ir/breadth_first_search.h" #include "shader_recompiler/frontend/ir/ir_emitter.h" #include "shader_recompiler/frontend/ir/value.h" #include "shader_recompiler/host_translate_info.h" @@ -49,6 +50,7 @@ using StorageBufferSet = using StorageInstVector = small_vector; using StorageWritesSet = flat_set, small_vector>; +using LocalStores = std::multimap; struct StorageInfo { StorageBufferSet set; @@ -333,7 +335,7 @@ std::optional TrackLowAddress(IR::Inst* inst) { } /// Tries to track the storage buffer address used by a global memory instruction -std::optional Track(const IR::Value& value, const Bias* bias) { +StorageBufferSet Track(const IR::Value& value, const Bias* bias, const LocalStores& local_stores) { const auto pred{[bias](const IR::Inst* inst) -> std::optional { if (inst->GetOpcode() != IR::Opcode::GetCbufU32 && inst->GetOpcode() != IR::Opcode::GetCbufU32x2) { @@ -366,11 +368,83 @@ std::optional Track(const IR::Value& value, const Bias* bias) } return storage_buffer; }}; - return BreadthFirstSearch(value, pred); + StorageBufferSet result; + std::unordered_set visited; + small_vector pending; + const auto push{[&](const IR::Value& arg) { + if (!arg.IsImmediate() && visited.insert(arg.InstRecursive()).second) { + pending.push_back(arg.InstRecursive()); + } + }}; + push(value); + while (!pending.empty()) { + const IR::Inst* const inst{pending.back()}; + pending.pop_back(); + if (const std::optional storage_buffer{pred(inst)}) { + result.insert(*storage_buffer); + continue; + } + switch (inst->GetOpcode()) { + case IR::Opcode::LoadLocal: + if (inst->Arg(0).IsImmediate()) { + const auto [begin, end]{local_stores.equal_range(inst->Arg(0).U32())}; + for (auto it = begin; it != end; ++it) { + push(it->second); + } + } + continue; + case IR::Opcode::SelectU32: + case IR::Opcode::SelectU64: + push(inst->Arg(1)); + push(inst->Arg(2)); + continue; + case IR::Opcode::GetCbufU8: + case IR::Opcode::GetCbufS8: + case IR::Opcode::GetCbufU16: + case IR::Opcode::GetCbufS16: + case IR::Opcode::GetCbufU32: + case IR::Opcode::GetCbufF32: + case IR::Opcode::GetCbufU32x2: + case IR::Opcode::LoadSharedU8: + case IR::Opcode::LoadSharedS8: + case IR::Opcode::LoadSharedU16: + case IR::Opcode::LoadSharedS16: + case IR::Opcode::LoadSharedU32: + case IR::Opcode::LoadSharedU64: + case IR::Opcode::LoadSharedU128: + continue; + default: + break; + } + if (IsGlobalMemory(*inst) || inst->MayHaveSideEffects()) { + continue; + } + for (size_t arg = 0; arg < inst->NumArgs(); ++arg) { + push(inst->Arg(arg)); + } + } + return result; +} + +LocalStores GatherLocalStores(const IR::Program& program) { + LocalStores stores; + for (IR::Block* const block : program.post_order_blocks) { + for (const IR::Inst& inst : block->Instructions()) { + if (inst.GetOpcode() != IR::Opcode::WriteLocal) { + continue; + } + if (!inst.Arg(0).IsImmediate()) { + return {}; + } + stores.emplace(inst.Arg(0).U32(), inst.Arg(1)); + } + } + return stores; } /// Collects the storage buffer used by a global memory instruction and the instruction itself -void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info) { +void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info, + const LocalStores& local_stores) { // NVN puts storage buffers in a specific range, we have to bias towards these addresses to // avoid getting false positives static constexpr Bias nvn_bias{ @@ -387,25 +461,24 @@ void CollectStorageBuffers(IR::Block& block, IR::Inst& inst, StorageInfo& info) } // First try to find storage buffers in the NVN address const IR::U32 low_addr{low_addr_info->value}; - std::optional storage_buffer{Track(low_addr, &nvn_bias)}; - if (!storage_buffer) { + StorageBufferSet candidates{Track(low_addr, &nvn_bias, local_stores)}; + if (candidates.empty()) { // If it fails, track without a bias - storage_buffer = Track(low_addr, nullptr); - if (!storage_buffer) { - // If that also fails, use NVN fallbacks - LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks"); - return; - } - LOG_WARNING(Shader, "Storage buffer tracked without bias, index {} offset {}", - storage_buffer->index, storage_buffer->offset); + candidates = Track(low_addr, nullptr, local_stores); } + if (candidates.size() != 1) { + // If that also fails, use NVN fallbacks + LOG_WARNING(Shader, "Storage buffer failed to track, using global memory fallbacks"); + return; + } + const StorageBufferAddr storage_buffer{*candidates.begin()}; // Collect storage buffer and the instruction if (IsGlobalMemoryWrite(inst)) { - info.writes.insert(*storage_buffer); + info.writes.insert(storage_buffer); } - info.set.insert(*storage_buffer); + info.set.insert(storage_buffer); info.to_replace.push_back(StorageInst{ - .storage_buffer{*storage_buffer}, + .storage_buffer{storage_buffer}, .inst = &inst, .block = &block, }); @@ -525,12 +598,13 @@ void Replace(IR::Block& block, IR::Inst& inst, const IR::U32& storage_index, void GlobalMemoryToStorageBufferPass(IR::Program& program, const HostTranslateInfo& host_info) { StorageInfo info; + const LocalStores local_stores{GatherLocalStores(program)}; for (IR::Block* const block : program.post_order_blocks) { for (IR::Inst& inst : block->Instructions()) { if (!IsGlobalMemory(inst)) { continue; } - CollectStorageBuffers(*block, inst, info); + CollectStorageBuffers(*block, inst, info, local_stores); } } for (const StorageBufferAddr& storage_buffer : info.set) { @@ -562,6 +636,7 @@ void JoinStorageInfo(Info& base, Info& source) { })}; if (it != descriptors.end()) { it->is_written |= desc.is_written; + it->is_global_fallback |= desc.is_global_fallback; continue; } descriptors.push_back(desc); diff --git a/src/shader_recompiler/ir_opt/passes.h b/src/shader_recompiler/ir_opt/passes.h index 1e637cb23c..1ddbc58f7b 100644 --- a/src/shader_recompiler/ir_opt/passes.h +++ b/src/shader_recompiler/ir_opt/passes.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -16,6 +19,7 @@ void CollectShaderInfoPass(Environment& env, IR::Program& program); void ConditionalBarrierPass(IR::Program& program); void ConstantPropagationPass(Environment& env, IR::Program& program); void DeadCodeEliminationPass(IR::Program& program); +void GeometryCompactionPass(IR::Program& program, const HostTranslateInfo& host_info); void GlobalMemoryToStorageBufferPass(IR::Program& program, const HostTranslateInfo& host_info); void IdentityRemovalPass(IR::Program& program); void LowerFp64ToFp32(IR::Program& program); diff --git a/src/shader_recompiler/ir_opt/texture_pass.cpp b/src/shader_recompiler/ir_opt/texture_pass.cpp index fe34a67555..0228ce117d 100644 --- a/src/shader_recompiler/ir_opt/texture_pass.cpp +++ b/src/shader_recompiler/ir_opt/texture_pass.cpp @@ -33,7 +33,8 @@ using TextureInstVector = boost::container::small_vector; constexpr u32 DESCRIPTOR_SIZE = 8; constexpr u32 DESCRIPTOR_SIZE_SHIFT = u32(std::countr_zero(DESCRIPTOR_SIZE)); -constexpr u32 DESCRIPTOR_MAX_COUNT = 1024; +constexpr u32 DESCRIPTOR_MAX_COUNT = 128; +constexpr u32 DESCRIPTOR_CBUF_BYTES = 16 * 1024; u32 DynamicDescriptorSizeShift(const IR::U32& dynamic_offset) { const IR::Inst* const inst = dynamic_offset.InstRecursive(); @@ -48,11 +49,10 @@ u32 DynamicDescriptorSizeShift(const IR::U32& dynamic_offset) { u32 DynamicDescriptorCount(u32 base_offset, u32 size_shift, u32 max_descriptors) { auto const descriptor_limit = (std::max)(1U, max_descriptors); - auto const max_cbuf_bytes = 16 * descriptor_limit; - if (size_shift >= 31 || base_offset >= max_cbuf_bytes) + if (size_shift >= 31 || base_offset >= DESCRIPTOR_CBUF_BYTES) return 1; auto const stride = 1U << size_shift; - auto const available = max_cbuf_bytes - base_offset; + auto const available = DESCRIPTOR_CBUF_BYTES - base_offset; if (available < DESCRIPTOR_SIZE) return 1; auto const available_count = 1U + (available - DESCRIPTOR_SIZE) / stride; diff --git a/src/shader_recompiler/profile.h b/src/shader_recompiler/profile.h index d2ac480ab8..8e9c83f888 100644 --- a/src/shader_recompiler/profile.h +++ b/src/shader_recompiler/profile.h @@ -40,6 +40,7 @@ struct Profile { bool support_shader_quad_control{}; bool support_quad_shuffles{}; bool support_vote{}; + bool support_shuffle_relative{}; u32 supported_subgroup_stages{0x7F}; bool support_viewport_index_layer_non_geometry{}; bool support_viewport_mask{}; @@ -102,6 +103,8 @@ struct Profile { bool ignore_nan_fp_comparisons{}; /// Some drivers have broken support for OpVectorExtractDynamic on subgroup mask inputs bool has_broken_spirv_subgroup_mask_vector_extract_dynamic{}; + bool has_broken_spirv_subgroup_shuffle{}; + u32 max_subgroup_size{}; u32 gl_max_compute_smem_size{}; diff --git a/src/shader_recompiler/shader_info.h b/src/shader_recompiler/shader_info.h index 71309e040e..b8d760a30e 100644 --- a/src/shader_recompiler/shader_info.h +++ b/src/shader_recompiler/shader_info.h @@ -172,6 +172,7 @@ struct StorageBufferDescriptor { u32 cbuf_offset; u32 count; bool is_written; + bool is_global_fallback{}; auto operator<=>(const StorageBufferDescriptor&) const = default; }; diff --git a/src/video_core/CMakeLists.txt b/src/video_core/CMakeLists.txt index 64d14ab2a6..96facedf79 100644 --- a/src/video_core/CMakeLists.txt +++ b/src/video_core/CMakeLists.txt @@ -22,6 +22,7 @@ add_library(video_core STATIC buffer_cache/usage_tracker.h buffer_cache/virtual_range_cache.h buffer_cache/word_manager.h + cache_reclaim.h cache_types.h capture.h cdma_pusher.cpp @@ -167,6 +168,8 @@ add_library(video_core STATIC renderer_vulkan/vk_fence_manager.h renderer_vulkan/vk_graphics_pipeline.cpp renderer_vulkan/vk_graphics_pipeline.h + renderer_vulkan/vk_guest_memory.cpp + renderer_vulkan/vk_guest_memory.h renderer_vulkan/vk_multi_range_buffer.cpp renderer_vulkan/vk_multi_range_buffer.h renderer_vulkan/vk_master_semaphore.cpp @@ -256,8 +259,6 @@ add_library(video_core STATIC texture_cache/util.h textures/astc.h textures/astc.cpp - textures/bcn.cpp - textures/bcn.h textures/decoders.cpp textures/decoders.h textures/texture.cpp @@ -406,7 +407,7 @@ if (ENABLE_OPENGL) endif() target_link_libraries(video_core PUBLIC common core) -target_link_libraries(video_core PUBLIC shader_recompiler stb bc_decoder gpu_logging) +target_link_libraries(video_core PUBLIC shader_recompiler bc_decoder gpu_logging) if (ENABLE_OPENGL) target_link_libraries(video_core PUBLIC glad) endif() diff --git a/src/video_core/buffer_cache/buffer_base.h b/src/video_core/buffer_cache/buffer_base.h index d7eb21612f..cdfb235ef0 100644 --- a/src/video_core/buffer_cache/buffer_base.h +++ b/src/video_core/buffer_cache/buffer_base.h @@ -129,13 +129,49 @@ public: write_tick = write_tick_; } + u64 ContentSerial() const noexcept { + return content_serial; + } + + void MarkContentModified() noexcept { + content_serial = ++next_content_serial; + } + + [[nodiscard]] bool HasDrawHazard(u64 pass, u64 wfi, DAddr addr, u64 size, + bool check_feedback) const noexcept { + return draw_write_pass == pass && + (draw_write_wfi < wfi || (check_feedback && draw_write_feedback)) && + addr < draw_write_end && addr + size > draw_write_begin; + } + + void MarkDrawWrite(u64 pass, u64 wfi, DAddr addr, u64 size, bool feedback) noexcept { + if (draw_write_pass != pass) { + draw_write_pass = pass; + draw_write_begin = addr; + draw_write_end = addr + size; + draw_write_feedback = false; + } + draw_write_wfi = wfi; + draw_write_feedback |= feedback; + draw_write_begin = (std::min)(draw_write_begin, addr); + draw_write_end = (std::max)(draw_write_end, addr + size); + } + private: + static inline u64 next_content_serial = 0; + VAddr cpu_addr = 0; BufferFlagBits flags{}; int stream_score = 0; size_t lru_id = SIZE_MAX; size_t size_bytes = 0; u64 write_tick = 0; + u64 content_serial = ++next_content_serial; + u64 draw_write_pass = 0; + u64 draw_write_wfi = 0; + DAddr draw_write_begin = 0; + DAddr draw_write_end = 0; + bool draw_write_feedback = false; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/buffer_cache.h b/src/video_core/buffer_cache/buffer_cache.h index 4c345dd213..8f7ca3fe3c 100644 --- a/src/video_core/buffer_cache/buffer_cache.h +++ b/src/video_core/buffer_cache/buffer_cache.h @@ -30,34 +30,54 @@ BufferCache

::BufferCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, R #ifdef YUZU_LEGACY immediately_free = (Settings::values.vram_usage_mode.GetValue() == Settings::VramUsageMode::Aggressive); #endif - if (!runtime.CanReportMemoryUsage()) { - minimum_memory = DEFAULT_EXPECTED_MEMORY; - critical_memory = DEFAULT_CRITICAL_MEMORY; - return; - } - - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - minimum_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); + device_local_memory = runtime.GetDeviceLocalMemory(); + const auto thresholds = VideoCommon::MakeReclaimThresholds( + device_local_memory, static_cast(TARGET_THRESHOLD), + static_cast(DEFAULT_EXPECTED_MEMORY), static_cast(DEFAULT_CRITICAL_MEMORY), + HEAP_PRESSURE_HEADROOM); + minimum_memory = thresholds.minimum; + expected_memory = thresholds.expected; + critical_memory = thresholds.critical; + heap_headroom = thresholds.headroom; } template BufferCache

::~BufferCache() = default; +template +void BufferCache

::ReclaimInline() { + if (total_used_memory < minimum_memory) { + return; + } + int num_iterations = 8; + const auto clean_up = [this, &num_iterations](BufferId buffer_id) { + if (num_iterations == 0) { + return true; + } + --num_iterations; + Buffer& buffer = slot_buffers[buffer_id]; + if (memory_tracker.IsRegionGpuModified(buffer.CpuAddr(), buffer.SizeBytes())) { + return false; + } + DeleteBuffer(buffer_id); + return false; + }; + lru_cache.ForEachItemBelow(frame_tick - INLINE_TICKS_TO_DESTROY, clean_up); +} + template void BufferCache

::RunGarbageCollector() { - const bool aggressive_gc = total_used_memory >= critical_memory; - const u64 ticks_to_destroy = aggressive_gc ? 60 : 120; - int num_iterations = aggressive_gc ? 64 : 32; + const bool aggressive_gc = heap_pressure || total_used_memory >= critical_memory; + const bool priority_gc = aggressive_gc || total_used_memory >= expected_memory; + u64 ticks_to_destroy = 120; + int num_iterations = 32; + if (aggressive_gc) { + ticks_to_destroy = 30; + num_iterations = 64; + } else if (priority_gc) { + ticks_to_destroy = 60; + num_iterations = 48; + } const auto clean_up = [this, &num_iterations](BufferId buffer_id) { if (num_iterations == 0) { return true; @@ -96,11 +116,11 @@ void BufferCache

::TickFrame() { const bool skip_preferred = hits * 256 < shots * 251; channel_state->uniform_buffer_skip_cache_size = skip_preferred ? DEFAULT_SKIP_CACHE_SIZE : 0; - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); + heap_pressure = false; + if (device_local_memory != 0 && runtime.CanReportMemoryUsage()) { + heap_pressure = runtime.GetDeviceMemoryUsage() + heap_headroom >= device_local_memory; } - if (total_used_memory >= minimum_memory) { + if (total_used_memory >= minimum_memory || heap_pressure) { RunGarbageCollector(); } ++frame_tick; @@ -246,6 +266,7 @@ bool BufferCache

::DMACopy(GPUVAddr src_address, GPUVAddr dest_address, u64 am const auto& copy = copies[0]; src_buffer.MarkUsage(copy.src_offset, copy.size); dest_buffer.MarkUsage(copy.dst_offset, copy.size); + dest_buffer.MarkContentModified(); runtime.CopyBuffer(dest_buffer, src_buffer, copies, true); if (has_new_downloads) { memory_tracker.MarkRegionAsGpuModified(*cpu_dest_address, amount); @@ -277,6 +298,7 @@ bool BufferCache

::DMAClear(GPUVAddr dst_address, u64 amount, u32 value) { const u32 offset = dest_buffer.Offset(*cpu_dst_address); runtime.ClearBuffer(dest_buffer, offset, size, value); dest_buffer.MarkUsage(offset, size); + dest_buffer.MarkContentModified(); return true; } @@ -305,6 +327,7 @@ std::pair BufferCache

::ObtainCPUBuffer( default: break; } + buffer.MarkUsage(buffer.Offset(device_addr), size); switch (post_op) { case ObtainBufferOperation::MarkAsWritten: @@ -344,14 +367,45 @@ void BufferCache

::DisableGraphicsUniformBuffer(size_t stage, u32 index) { template void BufferCache

::UpdateGraphicsBuffers(bool is_indexed) { + if constexpr (!IS_OPENGL) { + draw_writes.clear(); + draw_pass = runtime.RenderPassSerial(); + draw_wfi = runtime.WaitForIdleSerial(); + draw_hazard = false; + recording_draw = true; + } + ReclaimInline(); do { channel_state->has_deleted_buffers = false; DoUpdateGraphicsBuffers(is_indexed); } while (channel_state->has_deleted_buffers); } +template +bool BufferCache

::TakeDrawHazard() noexcept { + return std::exchange(draw_hazard, false); +} + +template +void BufferCache

::CommitDrawWrites() { + if constexpr (!IS_OPENGL) { + recording_draw = false; + if (draw_writes.empty()) { + return; + } + const u64 pass = runtime.RenderPassSerial(); + for (const DrawWrite& write : draw_writes) { + Buffer& buffer = slot_buffers[write.buffer_id]; + buffer.setWriteTick(runtime.CurrentTick()); + buffer.MarkDrawWrite(pass, draw_wfi, write.device_addr, write.size, write.feedback); + } + runtime.MarkRenderPassWrites(); + } +} + template void BufferCache

::UpdateComputeBuffers() { + ReclaimInline(); do { channel_state->has_deleted_buffers = false; DoUpdateComputeBuffers(); @@ -748,17 +802,35 @@ void BufferCache

::BindHostIndexBuffer() { const u32 offset = buffer.Offset(channel_state->index_buffer.device_addr); const u32 size = channel_state->index_buffer.size; const auto& draw_state = maxwell3d->draw_manager.draw_state; + if constexpr (!HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT) { + if (draw_state.topology == Maxwell::PrimitiveTopology::Quads || + draw_state.topology == Maxwell::PrimitiveTopology::QuadStrip) { + const DAddr device_addr = channel_state->index_buffer.device_addr; + std::span indices = draw_state.inline_index_draw_indexes; + if (indices.empty() && !IsRegionGpuModified(device_addr, size)) { + indices = ImmediateBufferWithData(device_addr, size); + } + if (runtime.BindQuadIndices(draw_state.topology, draw_state.index_buffer.format, + draw_state.index_buffer.first, + draw_state.index_buffer.count, indices)) { + return; + } + } + } if (draw_state.inline_index_draw_indexes.empty()) { SynchronizeBuffer(buffer, channel_state->index_buffer.device_addr, size); + } else if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { + const auto upload_staging = runtime.UploadStagingBuffer(size); + std::memcpy(upload_staging.mapped_span.data(), + draw_state.inline_index_draw_indexes.data(), size); + runtime.BindIndexBuffer(draw_state.topology, draw_state.index_buffer.format, + draw_state.index_buffer.first, draw_state.index_buffer.count, + upload_staging.buffer, static_cast(upload_staging.offset), + size); + return; } else { - if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { - auto upload_staging = runtime.UploadStagingBuffer(size); - std::array copies{{BufferCopy{.src_offset = upload_staging.offset, .dst_offset = 0, .size = size}}}; - std::memcpy(upload_staging.mapped_span.data(), draw_state.inline_index_draw_indexes.data(), size); - runtime.CopyBuffer(buffer, upload_staging.buffer, copies, true); - } else { - buffer.ImmediateUpload(0, draw_state.inline_index_draw_indexes); - } + buffer.MarkContentModified(); + buffer.ImmediateUpload(0, draw_state.inline_index_draw_indexes); } if constexpr (HAS_FULL_INDEX_AND_PRIMITIVE_SUPPORT) { const u32 new_offset = offset + draw_state.index_buffer.first * u32(draw_state.index_buffer.FormatSizeInBytes()); @@ -903,6 +975,7 @@ void BufferCache

::BindHostDrawIndirectBuffers() { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); SynchronizeBuffer(buffer, binding.device_addr, binding.size); + buffer.MarkUsage(buffer.Offset(binding.device_addr), binding.size); }; if (current_draw_indirect->include_count) { bind_buffer(channel_state->count_buffer_binding); @@ -1006,15 +1079,14 @@ void BufferCache

::BindHostGraphicsUniformBuffer(size_t stage, u32 index, u32 } template -void BufferCache

::ResolveMultiRangeStorage(Binding& binding, bool is_written, - std::vector& pool) { +void BufferCache

::ResolveMultiRangeStorage(Binding& binding, + std::vector& pool, + bool is_written) { binding.segment_first = 0; binding.segment_count = 0; if constexpr (requires { runtime.BindMultiRangeStorageBuffer(u64{}, bool{}); }) { - if (binding.gpu_addr == 0 || binding.size == 0) { - return; - } - if (is_written && !runtime.PrefersSparseSources()) { + if (binding.gpu_addr == 0 || binding.size == 0 || + (is_written && !runtime.PrefersSparseSources())) { return; } const VirtualSegments* found = @@ -1022,7 +1094,7 @@ void BufferCache

::ResolveMultiRangeStorage(Binding& binding, bool is_written, if (!found || found->size() < 2) { return; } - const VirtualSegments segments = *found; + const VirtualSegments& segments = *found; const u32 first = static_cast(pool.size()); const bool prefer_sparse = runtime.PrefersSparseSources(); for (const VirtualSegment& segment : segments) { @@ -1059,9 +1131,7 @@ bool BufferCache

::BindMultiRangeStorage(const Binding& binding, bool is_writt const MultiRangeSegment& segment = pool[binding.segment_first + index]; Buffer& buffer = slot_buffers[segment.buffer_id]; TouchBuffer(buffer, segment.buffer_id); - if (SynchronizeBuffer(buffer, segment.device_addr, segment.size)) { - runtime.InvalidateMultiRange(key); - } + SynchronizeBuffer(buffer, segment.device_addr, segment.size); const u32 offset = buffer.Offset(segment.device_addr); buffer.MarkUsage(offset, segment.size); if (is_written) { @@ -1152,8 +1222,8 @@ void BufferCache

::BindHostTransformFeedbackBuffers() { Buffer& buffer = slot_buffers[binding.buffer_id]; TouchBuffer(buffer, binding.buffer_id); size = binding.size; - SynchronizeBuffer(buffer, binding.device_addr, size); - MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size); + SynchronizeBuffer(buffer, binding.device_addr, size, false); + MarkWrittenBuffer(binding.buffer_id, binding.device_addr, size, true); offset = buffer.Offset(binding.device_addr); buffer.MarkUsage(offset, size); host_buffer = &buffer; @@ -1317,13 +1387,15 @@ void BufferCache

::UpdateIndexBuffer() { flags[Dirty::IndexBuffer] = false; if (!draw_state.inline_index_draw_indexes.empty()) [[unlikely]] { auto inline_index_size = static_cast(draw_state.inline_index_draw_indexes.size()); - u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE); - if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] { - inline_buffer_id = CreateBuffer(0, buffer_size, false); - } - if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] { - slot_buffers.erase(inline_buffer_id); - inline_buffer_id = CreateBuffer(0, buffer_size, false); + if constexpr (!USE_MEMORY_MAPS_FOR_UPLOADS) { + const u32 buffer_size = Common::AlignUp(inline_index_size, CACHING_PAGESIZE); + if (inline_buffer_id == NULL_BUFFER_ID) [[unlikely]] { + inline_buffer_id = CreateBuffer(0, buffer_size, false); + } + if (slot_buffers[inline_buffer_id].SizeBytes() < buffer_size) [[unlikely]] { + DeleteBuffer(inline_buffer_id, true); + inline_buffer_id = CreateBuffer(0, buffer_size, false); + } } channel_state->index_buffer = Binding{ .device_addr = 0, @@ -1437,10 +1509,12 @@ void BufferCache

::UpdateStorageBuffers(size_t stage) { ForEachEnabledBit(channel_state->enabled_storage_buffers[stage], [&](u32 index) { // Resolve buffer Binding& binding = channel_state->storage_buffers[stage][index]; - const BufferId buffer_id = FindBuffer(binding.device_addr, binding.size, false); - binding.buffer_id = buffer_id; const bool is_written = ((channel_state->written_storage_buffers[stage] >> index) & 1) != 0; - ResolveMultiRangeStorage(binding, is_written, graphics_segments); + ResolveMultiRangeStorage(binding, graphics_segments, is_written); + binding.buffer_id = NULL_BUFFER_ID; + if (binding.segment_count == 0 || is_written) { + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); + } }); } @@ -1503,10 +1577,12 @@ void BufferCache

::UpdateComputeStorageBuffers() { ForEachEnabledBit(channel_state->enabled_compute_storage_buffers, [&](u32 index) { // Resolve buffer Binding& binding = channel_state->compute_storage_buffers[index]; - binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); - const bool is_written = - ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; - ResolveMultiRangeStorage(binding, is_written, compute_segments); + const bool is_written = ((channel_state->written_compute_storage_buffers >> index) & 1) != 0; + ResolveMultiRangeStorage(binding, compute_segments, is_written); + binding.buffer_id = NULL_BUFFER_ID; + if (binding.segment_count == 0 || is_written) { + binding.buffer_id = FindBuffer(binding.device_addr, binding.size, false); + } }); } @@ -1519,10 +1595,15 @@ void BufferCache

::UpdateComputeTextureBuffers() { } template -void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size) { +void BufferCache

::MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, + bool feedback) { if constexpr (!IS_OPENGL) { Buffer& buffer = slot_buffers[buffer_id]; buffer.setWriteTick(runtime.CurrentTick()); + buffer.MarkContentModified(); + if (recording_draw) { + draw_writes.push_back({buffer_id, device_addr, size, feedback}); + } } memory_tracker.MarkRegionAsGpuModified(device_addr, size); gpu_modified_ranges.Add(device_addr, size); @@ -1680,6 +1761,11 @@ BufferId BufferCache

::CreateBuffer(DAddr device_addr, u32 wanted_size, wanted_size = static_cast(device_addr_end - device_addr); const OverlapResult overlap = ResolveOverlaps(device_addr, wanted_size); const u32 size = static_cast(overlap.end - overlap.begin); + if constexpr (requires(Buffer& buffer) { buffer.IsSparseCompatible(); }) { + for (const BufferId overlap_id : overlap.ids) { + sparse_compatible |= slot_buffers[overlap_id].IsSparseCompatible(); + } + } const BufferId new_buffer_id = slot_buffers.insert(runtime, overlap.begin, size, sparse_compatible); auto& new_buffer = slot_buffers[new_buffer_id]; @@ -1737,7 +1823,12 @@ void BufferCache

::TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept { } template -bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size) { +bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size, + bool check_feedback) { + if constexpr (!IS_OPENGL) { + draw_hazard |= + buffer.HasDrawHazard(draw_pass, draw_wfi, device_addr, size, check_feedback); + } upload_copies.clear(); u64 total_size_bytes = 0; u64 largest_copy = 0; @@ -1756,13 +1847,16 @@ bool BufferCache

::SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 si } const std::span copies_span(upload_copies.data(), upload_copies.size()); UploadMemory(buffer, total_size_bytes, largest_copy, copies_span); - any_buffer_uploaded = true; + if constexpr (IS_OPENGL) { + any_buffer_uploaded = true; + } return false; } template void BufferCache

::UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy, std::span copies) { + buffer.MarkContentModified(); if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { MappedUploadMemory(buffer, total_size_bytes, copies); } else { @@ -1804,6 +1898,20 @@ void BufferCache

::MappedUploadMemory([[maybe_unused]] Buffer& buffer, [[maybe_unused]] u64 total_size_bytes, [[maybe_unused]] std::span copies) { if constexpr (USE_MEMORY_MAPS) { + if constexpr (requires { runtime.DirectUploadSpan(buffer, copies); }) { + const std::span direct = runtime.DirectUploadSpan(buffer, copies); + if (!direct.empty()) { + for (const BufferCopy& copy : copies) { + const DAddr device_addr = buffer.CpuAddr() + copy.dst_offset; + if (Settings::values.enable_gpu_buffer_readback.GetValue()) { + DownloadBufferMemory(buffer, device_addr, copy.size); + } + device_memory.ReadBlockUnsafe(device_addr, direct.data() + copy.dst_offset, + copy.size); + } + return; + } + } auto upload_staging = runtime.UploadStagingBuffer(total_size_bytes); const std::span staging_pointer = upload_staging.mapped_span; for (BufferCopy& copy : copies) { @@ -1848,6 +1956,7 @@ void BufferCache

::InlineMemoryImplementation(DAddr dest_address, size_t copy_ BufferId buffer_id = FindBuffer(dest_address, static_cast(copy_size), false); auto& buffer = slot_buffers[buffer_id]; SynchronizeBuffer(buffer, dest_address, static_cast(copy_size)); + buffer.MarkContentModified(); if constexpr (USE_MEMORY_MAPS_FOR_UPLOADS) { auto upload_staging = runtime.UploadStagingBuffer(copy_size); @@ -1902,6 +2011,16 @@ void BufferCache

::DownloadBufferMemory(Buffer& buffer, DAddr device_addr, u64 } if constexpr (USE_MEMORY_MAPS) { + if constexpr (requires { runtime.DirectDownloadSpan(buffer); }) { + const std::span direct = runtime.DirectDownloadSpan(buffer); + if (!direct.empty()) { + for (const BufferCopy& copy : copies) { + device_memory.WriteBlockUnsafe(buffer.CpuAddr() + copy.src_offset, + direct.data() + copy.src_offset, copy.size); + } + return; + } + } auto download_staging = runtime.DownloadStagingBuffer(total_size_bytes); const u8* const mapped_memory = download_staging.mapped_span.data(); const std::span copies_span(copies.data(), copies.data() + copies.size()); @@ -1970,6 +2089,10 @@ void BufferCache

::DeleteBuffer(BufferId buffer_id, bool do_not_mark) { memory_tracker.MarkRegionAsCpuModified(buffer.CpuAddr(), buffer.SizeBytes()); } + if (inline_buffer_id == buffer_id) { + inline_buffer_id = NULL_BUFFER_ID; + } + Unregister(buffer_id); #ifdef YUZU_LEGACY diff --git a/src/video_core/buffer_cache/buffer_cache_base.h b/src/video_core/buffer_cache/buffer_cache_base.h index 0dbb368c47..d1f9ff5f97 100644 --- a/src/video_core/buffer_cache/buffer_cache_base.h +++ b/src/video_core/buffer_cache/buffer_cache_base.h @@ -31,6 +31,7 @@ #include "video_core/buffer_cache/buffer_base.h" #include "video_core/buffer_cache/virtual_range_cache.h" #include "video_core/control/channel_state_cache.h" +#include "video_core/cache_reclaim.h" #include "video_core/delayed_destruction_ring.h" #include "video_core/dirty_flags.h" #include "video_core/engines/maxwell_3d.h" @@ -199,6 +200,8 @@ class BufferCache : public VideoCommon::ChannelSetupCaches pool); - void ResolveMultiRangeStorage(Binding& binding, bool is_written, - std::vector& pool); + + + + + void ResolveMultiRangeStorage(Binding& binding, std::vector& pool, + bool is_written); void UnmapGPUMemory(size_t as_id, GPUVAddr gpu_addr, size_t size); @@ -253,6 +260,10 @@ public: void UpdateComputeBuffers(); + [[nodiscard]] bool TakeDrawHazard() noexcept; + + void CommitDrawWrites(); + void BindHostGeometryBuffers(bool is_indexed); void BindHostStageBuffers(size_t stage); @@ -378,6 +389,8 @@ private: void RunGarbageCollector(); + void ReclaimInline(); + void BindHostIndexBuffer(); void BindHostVertexBuffers(); @@ -430,7 +443,8 @@ private: void UpdateComputeTextureBuffers(); - void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size); + void MarkWrittenBuffer(BufferId buffer_id, DAddr device_addr, u32 size, + bool feedback = false); [[nodiscard]] BufferId FindBuffer(DAddr device_addr, u32 size, bool sparse_compatible); @@ -452,7 +466,8 @@ private: void TouchBuffer(Buffer& buffer, BufferId buffer_id) noexcept; - bool SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size); + bool SynchronizeBuffer(Buffer& buffer, DAddr device_addr, u32 size, + bool check_feedback = true); void UploadMemory(Buffer& buffer, u64 total_size_bytes, u64 largest_copy, std::span copies); @@ -511,6 +526,18 @@ private: boost::container::small_vector upload_copies; + struct DrawWrite { + BufferId buffer_id; + DAddr device_addr; + u32 size; + bool feedback; + }; + boost::container::small_vector draw_writes; + u64 draw_pass = 0; + u64 draw_wfi = 0; + bool draw_hazard = false; + bool recording_draw = false; + MemoryTracker memory_tracker; Common::RangeSet uncommitted_gpu_modified_ranges; Common::RangeSet gpu_modified_ranges; @@ -537,8 +564,12 @@ private: std::vector compute_segments; u64 frame_tick = 0; u64 total_used_memory = 0; + u64 device_local_memory = 0; u64 minimum_memory = 0; + u64 expected_memory = 0; u64 critical_memory = 0; + u64 heap_headroom = 0; + bool heap_pressure = false; BufferId inline_buffer_id; #ifdef YUZU_LEGACY bool immediately_free = false; diff --git a/src/video_core/buffer_cache/usage_tracker.h b/src/video_core/buffer_cache/usage_tracker.h index 6f84e463e3..1b3ab2f998 100644 --- a/src/video_core/buffer_cache/usage_tracker.h +++ b/src/video_core/buffer_cache/usage_tracker.h @@ -6,7 +6,9 @@ #pragma once -#include "common/alignment.h" +#include +#include + #include "common/common_types.h" namespace VideoCommon { @@ -15,74 +17,77 @@ class UsageTracker { // PAGE_SHIFT is a macro on FreeBSD static constexpr size_t BUFFER_BYTES_PER_BITSHIFT = 6; static constexpr size_t BUFFER_PAGE_SHIFT = 6 + BUFFER_BYTES_PER_BITSHIFT; - static constexpr size_t BUFFER_PAGE_BYTES = 1 << BUFFER_PAGE_SHIFT; + static constexpr u64 BUFFER_PAGE_BYTES = u64{1} << BUFFER_PAGE_SHIFT; + + struct Page { + u64 bits; + u64 tick; + }; + public: - explicit UsageTracker(size_t size) { - const size_t num_pages = (size >> BUFFER_PAGE_SHIFT) + 1; - pages.resize(num_pages, 0ULL); - } + explicit UsageTracker(size_t size) + : pages((size >> BUFFER_PAGE_SHIFT) + 1), full_ticks(pages.size()) {} - void Reset() noexcept { - std::ranges::fill(pages, 0ULL); - } - - void Track(u64 offset, u64 size) noexcept { - const size_t page = offset >> BUFFER_PAGE_SHIFT; - const size_t page_end = (offset + size) >> BUFFER_PAGE_SHIFT; - if (page_end < page || page_end >= pages.size()) { + void Track(u64 offset, u64 size, u64 tick, u64 gpu_tick) noexcept { + const u64 end = offset + size; + if (size == 0 || ((end - 1) >> BUFFER_PAGE_SHIFT) >= pages.size()) { return; } - TrackPage(page, offset, size); - if (page == page_end) { + const u64 first_page = offset >> BUFFER_PAGE_SHIFT; + const u64 last_page = (end - 1) >> BUFFER_PAGE_SHIFT; + TrackPage(first_page, PageMask(first_page, offset, end), tick, gpu_tick); + if (last_page == first_page) { return; } - for (size_t i = page + 1; i < page_end; i++) { - pages[i] = ~u64{0}; - } - const size_t offset_end = offset + size; - const size_t offset_end_page_aligned = Common::AlignDown(offset_end, BUFFER_PAGE_BYTES); - TrackPage(page_end, offset_end_page_aligned, offset_end - offset_end_page_aligned); + std::fill(full_ticks.begin() + first_page + 1, full_ticks.begin() + last_page, tick); + TrackPage(last_page, PageMask(last_page, offset, end), tick, gpu_tick); } - [[nodiscard]] bool IsUsed(u64 offset, u64 size) const noexcept { - const size_t page = offset >> BUFFER_PAGE_SHIFT; - const size_t page_end = (offset + size) >> BUFFER_PAGE_SHIFT; - if (page_end < page || page_end >= pages.size()) { + [[nodiscard]] bool IsUsed(u64 offset, u64 size, u64 gpu_tick) const noexcept { + const u64 end = offset + size; + if (size == 0 || ((end - 1) >> BUFFER_PAGE_SHIFT) >= pages.size()) { return false; } - if (IsPageUsed(page, offset, size)) { + const u64 first_page = offset >> BUFFER_PAGE_SHIFT; + const u64 last_page = (end - 1) >> BUFFER_PAGE_SHIFT; + if (IsPageUsed(first_page, PageMask(first_page, offset, end), gpu_tick) || + IsPageUsed(last_page, PageMask(last_page, offset, end), gpu_tick)) { return true; } - for (size_t i = page + 1; i < page_end; i++) { - if (pages[i] != 0) { + for (u64 page = first_page + 1; page < last_page; ++page) { + if (IsPageUsed(page, ~u64{0}, gpu_tick)) { return true; } } - const size_t offset_end = offset + size; - const size_t offset_end_page_aligned = Common::AlignDown(offset_end, BUFFER_PAGE_BYTES); - return IsPageUsed(page_end, offset_end_page_aligned, offset_end - offset_end_page_aligned); + return false; } private: - void TrackPage(u64 page, u64 offset, u64 size) noexcept { - const size_t offset_in_page = offset % BUFFER_PAGE_BYTES; - const size_t first_bit = offset_in_page >> BUFFER_BYTES_PER_BITSHIFT; - const size_t num_bits = std::min(size, BUFFER_PAGE_BYTES) >> BUFFER_BYTES_PER_BITSHIFT; - const size_t mask = ~u64{0} >> (64 - num_bits); - pages[page] |= (~u64{0} & mask) << first_bit; + void TrackPage(u64 page, u64 mask, u64 tick, u64 gpu_tick) noexcept { + Page& entry = pages[page]; + if (entry.tick <= gpu_tick) { + entry.bits = 0; + } + entry.bits |= mask; + entry.tick = (std::max)(entry.tick, tick); } - bool IsPageUsed(u64 page, u64 offset, u64 size) const noexcept { - const size_t offset_in_page = offset % BUFFER_PAGE_BYTES; - const size_t first_bit = offset_in_page >> BUFFER_BYTES_PER_BITSHIFT; - const size_t num_bits = std::min(size, BUFFER_PAGE_BYTES) >> BUFFER_BYTES_PER_BITSHIFT; - const size_t mask = ~u64{0} >> (64 - num_bits); - const size_t mask2 = (~u64{0} & mask) << first_bit; - return (pages[page] & mask2) != 0; + [[nodiscard]] bool IsPageUsed(u64 page, u64 mask, u64 gpu_tick) const noexcept { + const Page& entry = pages[page]; + return full_ticks[page] > gpu_tick || (entry.tick > gpu_tick && (entry.bits & mask) != 0); } -private: - std::vector pages; + [[nodiscard]] static u64 PageMask(u64 page, u64 offset, u64 end) noexcept { + const u64 page_begin = page << BUFFER_PAGE_SHIFT; + const u64 first = + ((std::max)(offset, page_begin) - page_begin) >> BUFFER_BYTES_PER_BITSHIFT; + const u64 last = ((std::min)(end, page_begin + BUFFER_PAGE_BYTES) - 1 - page_begin) >> + BUFFER_BYTES_PER_BITSHIFT; + return (~u64{0} >> (63 - last)) & (~u64{0} << first); + } + + std::vector pages; + std::vector full_ticks; }; } // namespace VideoCommon diff --git a/src/video_core/buffer_cache/virtual_range_cache.h b/src/video_core/buffer_cache/virtual_range_cache.h index f4284d6abb..542fb52b23 100644 --- a/src/video_core/buffer_cache/virtual_range_cache.h +++ b/src/video_core/buffer_cache/virtual_range_cache.h @@ -126,11 +126,11 @@ private: } void ApplyDeferred() { - std::vector pending; bool overflow = false; { std::scoped_lock lock{deferred_mutex}; has_deferred.store(false, std::memory_order_release); + pending.clear(); pending.swap(deferred); overflow = deferred_overflow; deferred_overflow = false; @@ -165,6 +165,7 @@ private: ::Common::unordered_map entries; std::vector deferred; + std::vector pending; std::mutex deferred_mutex; std::atomic has_deferred{false}; bool deferred_overflow{}; diff --git a/src/video_core/cache_reclaim.h b/src/video_core/cache_reclaim.h new file mode 100644 index 0000000000..a8870ea96b --- /dev/null +++ b/src/video_core/cache_reclaim.h @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include "common/common_types.h" + +namespace VideoCommon { + +struct ReclaimThresholds { + u64 minimum{}; + u64 expected{}; + u64 critical{}; + u64 headroom{}; +}; + +[[nodiscard]] constexpr ReclaimThresholds MakeReclaimThresholds(u64 device_local_memory, + u64 target_threshold, + u64 default_expected, + u64 default_critical, + u64 default_headroom) { + u64 critical = default_critical; + if (device_local_memory != 0) { + const u64 budget = (std::min)(device_local_memory, target_threshold); + critical = (std::min)(critical, budget / 2); + } + const u64 expected = (std::min)(default_expected, (critical * 3) / 4); + return ReclaimThresholds{ + .minimum = (expected * 3) / 4, + .expected = expected, + .critical = critical, + .headroom = (std::clamp)(device_local_memory / 4, default_headroom, default_headroom * 2), + }; +} + +} // namespace VideoCommon diff --git a/src/video_core/dma_pusher.cpp b/src/video_core/dma_pusher.cpp index 2417659f09..1eb665ad79 100644 --- a/src/video_core/dma_pusher.cpp +++ b/src/video_core/dma_pusher.cpp @@ -56,7 +56,7 @@ bool DmaPusher::Step() { return true; } - if (prefetch_size > 0) { + if (command_list_size == 0) { ProcessCommands(command_list.prefetch_command_list); dma_pushbuffer.pop(); return true; @@ -89,6 +89,7 @@ bool DmaPusher::Step() { } if (++dma_pushbuffer_subindex >= command_list_size) { + ProcessCommands(command_list.prefetch_command_list); dma_pushbuffer.pop(); dma_pushbuffer_subindex = 0; } else { diff --git a/src/video_core/engines/draw_manager.cpp b/src/video_core/engines/draw_manager.cpp index 5891c25622..0d3558a8f3 100644 --- a/src/video_core/engines/draw_manager.cpp +++ b/src/video_core/engines/draw_manager.cpp @@ -14,6 +14,10 @@ namespace Tegra::Engines { void Maxwell3D::DrawManager::ProcessMethodCall(Maxwell3D& maxwell3d, u32 method, u32 argument) { + if (draw_state.draw_mode == DrawMode::InstanceArray && + method != MAXWELL3D_REG_INDEX(vertex_array_instance_subsequent)) { + DrawDeferred(maxwell3d); + } switch (method) { case MAXWELL3D_REG_INDEX(clear_surface): return Clear(maxwell3d, 1); @@ -72,7 +76,20 @@ void Maxwell3D::DrawManager::Clear(Maxwell3D& maxwell3d, u32 layer_count) { } } +void Maxwell3D::DrawManager::FlushInstanceArray(Maxwell3D& maxwell3d) { + if (draw_state.draw_mode != DrawMode::InstanceArray) { + return; + } + const u32 instance_count = draw_state.instance_count + 1; + draw_state.draw_mode = DrawMode::General; + draw_state.instance_count = 0; + if (maxwell3d.ShouldExecute()) { + maxwell3d.rasterizer->Draw(false, instance_count); + } +} + void Maxwell3D::DrawManager::DrawDeferred(Maxwell3D& maxwell3d) { + FlushInstanceArray(maxwell3d); if (draw_state.draw_mode != DrawMode::Instance || draw_state.instance_count == 0) { return; } @@ -81,6 +98,7 @@ void Maxwell3D::DrawManager::DrawDeferred(Maxwell3D& maxwell3d) { } void Maxwell3D::DrawManager::DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, u32 base_instance, u32 num_instances) { + FlushInstanceArray(maxwell3d); draw_state.topology = topology; draw_state.vertex_buffer.first = vertex_first; draw_state.vertex_buffer.count = vertex_count; @@ -89,19 +107,29 @@ void Maxwell3D::DrawManager::DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::Pr } void Maxwell3D::DrawManager::DrawArrayInstanced(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, bool subsequent) { + if (subsequent && draw_state.draw_mode == DrawMode::InstanceArray && + instance_topology == topology && draw_state.vertex_buffer.first == vertex_first && + draw_state.vertex_buffer.count == vertex_count) { + ++draw_state.instance_count; + return; + } + u32 base_instance = 0; + if (subsequent) { + base_instance = draw_state.base_instance + draw_state.instance_count + 1; + } + DrawDeferred(maxwell3d); + instance_topology = topology; draw_state.topology = topology; draw_state.vertex_buffer.first = vertex_first; draw_state.vertex_buffer.count = vertex_count; - if (!subsequent) { - draw_state.instance_count = 1; - } - draw_state.base_instance = draw_state.instance_count - 1; - draw_state.draw_mode = DrawMode::Instance; - draw_state.instance_count++; - ProcessDraw(maxwell3d, false, 1); + draw_state.base_instance = base_instance; + draw_state.instance_count = 0; + draw_state.draw_mode = DrawMode::InstanceArray; + UpdateTopology(maxwell3d); } void Maxwell3D::DrawManager::DrawIndex(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 index_first, u32 index_count, u32 base_index, u32 base_instance, u32 num_instances) { + FlushInstanceArray(maxwell3d); draw_state.topology = topology; draw_state.index_buffer = maxwell3d.regs.index_buffer; draw_state.index_buffer.first = index_first; @@ -112,11 +140,13 @@ void Maxwell3D::DrawManager::DrawIndex(Maxwell3D& maxwell3d, Maxwell3D::Regs::Pr } void Maxwell3D::DrawManager::DrawArrayIndirect(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology) { + FlushInstanceArray(maxwell3d); draw_state.topology = topology; ProcessDrawIndirect(maxwell3d); } void Maxwell3D::DrawManager::DrawIndexedIndirect(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 index_first, u32 index_count) { + FlushInstanceArray(maxwell3d); draw_state.topology = topology; draw_state.index_buffer = maxwell3d.regs.index_buffer; draw_state.index_buffer.first = index_first; @@ -186,6 +216,9 @@ void Maxwell3D::DrawManager::DrawBegin(Maxwell3D& maxwell3d) { void Maxwell3D::DrawManager::DrawEnd(Maxwell3D& maxwell3d, u32 instance_count, bool force_draw) { switch (draw_state.draw_mode) { + case DrawMode::InstanceArray: + FlushInstanceArray(maxwell3d); + break; case DrawMode::Instance: if (!force_draw) { break; @@ -249,28 +282,68 @@ void Maxwell3D::DrawManager::DrawTexture(Maxwell3D& maxwell3d) { } void Maxwell3D::DrawManager::UpdateTopology(Maxwell3D& maxwell3d) { + using Topology = Maxwell3D::Regs::PrimitiveTopology; + using Override = Maxwell3D::Regs::PrimitiveTopologyOverride; switch (maxwell3d.regs.primitive_topology_control) { case Maxwell3D::Regs::PrimitiveTopologyControl::UseInBeginMethods: break; case Maxwell3D::Regs::PrimitiveTopologyControl::UseSeparateState: switch (maxwell3d.regs.topology_override) { - case Maxwell3D::Regs::PrimitiveTopologyOverride::None: + case Override::None: break; - case Maxwell3D::Regs::PrimitiveTopologyOverride::Points: - draw_state.topology = Maxwell3D::Regs::PrimitiveTopology::Points; + case Override::Points: + case Override::LegacyPoints: + draw_state.topology = Topology::Points; break; - case Maxwell3D::Regs::PrimitiveTopologyOverride::Lines: - draw_state.topology = Maxwell3D::Regs::PrimitiveTopology::Lines; + case Override::Lines: + case Override::LegacyLines: + case Override::LegacyLinesImm: + case Override::LegacyIndexedLines: + case Override::LegacyIndexedLines2: + draw_state.topology = Topology::Lines; break; - case Maxwell3D::Regs::PrimitiveTopologyOverride::LineStrip: - draw_state.topology = Maxwell3D::Regs::PrimitiveTopology::LineStrip; + case Override::LineStrip: + case Override::LegacyLineStrip: + case Override::LegacyIndexedLineStrip: + draw_state.topology = Topology::LineStrip; break; - default: - draw_state.topology = Maxwell3D::Regs::PrimitiveTopology(maxwell3d.regs.topology_override); + case Override::Triangles: + case Override::LegacyTriangles: + case Override::LegacyIndexedTriangles: + case Override::LegacyIndexedTriangles2: + draw_state.topology = Topology::Triangles; + break; + case Override::TriangleStrip: + case Override::LegacyTriangleStrip: + case Override::LegacyIndexedTriangleStrip: + draw_state.topology = Topology::TriangleStrip; + break; + case Override::LegacyTriangleFan: + case Override::LegacyTriangleFanImm: + case Override::LegacyIndexedTriangleFan: + draw_state.topology = Topology::TriangleFan; + break; + case Override::LinesAdjacency: + draw_state.topology = Topology::LinesAdjacency; + break; + case Override::LineStripAdjacency: + draw_state.topology = Topology::LineStripAdjacency; + break; + case Override::TrianglesAdjacency: + draw_state.topology = Topology::TrianglesAdjacency; + break; + case Override::TriangleStripAdjacency: + draw_state.topology = Topology::TriangleStripAdjacency; + break; + case Override::Patches: + draw_state.topology = Topology::Patches; break; } break; } + if (u32(draw_state.topology) > u32(Topology::Patches)) { + draw_state.topology = Topology::Triangles; + } } void Maxwell3D::DrawManager::ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed, u32 instance_count) { diff --git a/src/video_core/engines/maxwell_3d.cpp b/src/video_core/engines/maxwell_3d.cpp index e7da9d9138..4adab42e67 100644 --- a/src/video_core/engines/maxwell_3d.cpp +++ b/src/video_core/engines/maxwell_3d.cpp @@ -588,7 +588,7 @@ void Maxwell3D::ProcessCounterReset() { void Maxwell3D::ProcessSyncPoint() { const u32 sync_point = regs.sync_info.sync_point.Value(); [[maybe_unused]] const u32 cache_flush = regs.sync_info.clean_l2.Value(); - rasterizer->SignalSyncPoint(sync_point); + rasterizer->SignalSyncPoint(sync_point, 1); } void Maxwell3D::ProcessCBBind(size_t stage_index) { diff --git a/src/video_core/engines/maxwell_3d.h b/src/video_core/engines/maxwell_3d.h index 0cab2ae986..ae4b761ced 100644 --- a/src/video_core/engines/maxwell_3d.h +++ b/src/video_core/engines/maxwell_3d.h @@ -3034,7 +3034,7 @@ public: // clang-format on struct DrawManager { - enum class DrawMode : u32 { General = 0, Instance, InlineIndex }; + enum class DrawMode : u32 { General = 0, Instance, InlineIndex, InstanceArray }; struct State { Maxwell3D::Regs::PrimitiveTopology topology{}; DrawMode draw_mode{}; @@ -3071,6 +3071,7 @@ public: void ProcessMethodCall(Maxwell3D& maxwell3d, u32 method, u32 argument); void Clear(Maxwell3D& maxwell3d, u32 layer_count); void DrawDeferred(Maxwell3D& maxwell3d); + void FlushInstanceArray(Maxwell3D& maxwell3d); void DrawArray(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, u32 base_instance, u32 num_instances); void DrawArrayInstanced(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 vertex_first, u32 vertex_count, bool subsequent); void DrawIndex(Maxwell3D& maxwell3d, Maxwell3D::Regs::PrimitiveTopology topology, u32 index_first, u32 index_count, u32 base_index, u32 base_instance, u32 num_instances); @@ -3086,6 +3087,7 @@ public: void ProcessDraw(Maxwell3D& maxwell3d, bool draw_indexed, u32 instance_count); void ProcessDrawIndirect(Maxwell3D& maxwell3d); State draw_state{}; + Maxwell3D::Regs::PrimitiveTopology instance_topology{}; DrawTextureState draw_texture_state{}; IndirectParams indirect_state{}; }; diff --git a/src/video_core/engines/puller.cpp b/src/video_core/engines/puller.cpp index 0b05d31718..38b5a3e785 100644 --- a/src/video_core/engines/puller.cpp +++ b/src/video_core/engines/puller.cpp @@ -4,6 +4,8 @@ // SPDX-FileCopyrightText: 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later +#include + #include "common/assert.h" #include "common/logging.h" #include "common/settings.h" @@ -71,7 +73,7 @@ void Puller::ProcessFenceActionMethod(DmaPusher& dma_pusher) { dma_pusher.rasterizer->ReleaseFences(); break; case Puller::FenceOperation::Increment: - dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id); + dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id, 1); break; default: UNIMPLEMENTED_MSG("Unimplemented operation {}", regs.fence_action.op.Value()); @@ -278,6 +280,9 @@ void Puller::CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, ASSERT(subchannel < bound_engines.size()); if (ExecuteMethodOnEngine(dma_pusher, method)) { CallEngineMultiMethod(dma_pusher, method, subchannel, base_start, amount, methods_pending); + } else if (IsIncrementRun(method, base_start, amount)) { + regs.reg_array[method] = base_start[0]; + dma_pusher.rasterizer->SignalSyncPoint(regs.fence_action.syncpoint_id, amount); } else { for (u32 i = 0; i < amount; i++) { CallPullerMethod(dma_pusher, MethodCall{ @@ -290,6 +295,16 @@ void Puller::CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, } } +bool Puller::IsIncrementRun(u32 method, const u32* base_start, u32 amount) const { + if (BufferMethods(method) != BufferMethods::SyncpointOperation || amount < 2) { + return false; + } + const FenceAction action{.raw = base_start[0]}; + return action.op == FenceOperation::Increment && + std::all_of(base_start, base_start + amount, + [first = base_start[0]](u32 argument) { return argument == first; }); +} + /// Determines where the method should be executed. [[nodiscard]] bool Puller::ExecuteMethodOnEngine(DmaPusher& dma_pusher, u32 method) { const auto buffer_method = BufferMethods(method); diff --git a/src/video_core/engines/puller.h b/src/video_core/engines/puller.h index e8994f5640..c1a1d86e2a 100644 --- a/src/video_core/engines/puller.h +++ b/src/video_core/engines/puller.h @@ -72,6 +72,7 @@ public: void CallMethod(DmaPusher& dma_pusher, const MethodCall& method_call); void CallMultiMethod(DmaPusher& dma_pusher, u32 method, u32 subchannel, const u32* base_start, u32 amount, u32 methods_pending); + [[nodiscard]] bool IsIncrementRun(u32 method, const u32* base_start, u32 amount) const; void BindRasterizer(DmaPusher& dma_pusher, VideoCore::RasterizerInterface* rasterizer); void CallPullerMethod(DmaPusher& dma_pusher, const MethodCall& method_call); void CallEngineMethod(DmaPusher& dma_pusher, const MethodCall& method_call); diff --git a/src/video_core/engines/sw_blitter/blitter.cpp b/src/video_core/engines/sw_blitter/blitter.cpp index 8bcc2f7a76..a360ffb125 100644 --- a/src/video_core/engines/sw_blitter/blitter.cpp +++ b/src/video_core/engines/sw_blitter/blitter.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later @@ -92,7 +95,8 @@ void Bilinear(std::span input, std::span output, size_t src_widt const auto read_src = [&](f32 in_x, f32 in_y) { const size_t read_from = - ((static_cast(in_x) * src_width + static_cast(in_y)) >> 32) * + ((std::min)(static_cast(in_y), src_height - 1) * src_width + + (std::min)(static_cast(in_x), src_width - 1)) * ir_components; return std::span(&input[read_from], ir_components); }; diff --git a/src/video_core/fence_manager.h b/src/video_core/fence_manager.h index 885c714025..b7d53ad583 100644 --- a/src/video_core/fence_manager.h +++ b/src/video_core/fence_manager.h @@ -101,9 +101,15 @@ public: rasterizer.InvalidateGPUCache(); } - void SignalSyncPoint(u32 value) { - syncpoint_manager.IncrementGuest(value); - std::function func([this, value] { syncpoint_manager.IncrementHost(value); }); + void SignalSyncPoint(u32 value, u32 count) { + for (u32 i = 0; i < count; ++i) { + syncpoint_manager.IncrementGuest(value); + } + std::function func([this, value, count] { + for (u32 i = 0; i < count; ++i) { + syncpoint_manager.IncrementHost(value); + } + }); SignalFence(std::move(func)); } diff --git a/src/video_core/frame_gen/lossless_dll.cpp b/src/video_core/frame_gen/lossless_dll.cpp index 6be1e5530f..32c36f195e 100644 --- a/src/video_core/frame_gen/lossless_dll.cpp +++ b/src/video_core/frame_gen/lossless_dll.cpp @@ -157,7 +157,7 @@ private: [[nodiscard]] std::optional RvaToFileOffset(std::span sections, u32 rva) { for (const Section& section : sections) { - const u32 span = std::max(section.virtual_size, section.raw_size); + const u32 span = (std::max)(section.virtual_size, section.raw_size); if (span == 0 || rva < section.virtual_address) { continue; } diff --git a/src/video_core/gpu.cpp b/src/video_core/gpu.cpp index 9901cf4262..eb834308e0 100644 --- a/src/video_core/gpu.cpp +++ b/src/video_core/gpu.cpp @@ -152,6 +152,10 @@ struct GPU::Impl { /// Tick pending requests within the GPU. void TickWork() { + if (last_sync_fence.load(std::memory_order_relaxed) == + current_sync_fence.load(std::memory_order_relaxed)) { + return; + } std::unique_lock lck{sync_request_mutex}; while (!sync_requests.empty()) { auto request = std::move(sync_requests.front()); @@ -328,7 +332,7 @@ struct GPU::Impl { std::list> sync_requests; std::atomic current_sync_fence{}; - u64 last_sync_fence{}; + std::atomic last_sync_fence{}; std::mutex sync_request_mutex; std::condition_variable sync_request_cv; diff --git a/src/video_core/host_shaders/CMakeLists.txt b/src/video_core/host_shaders/CMakeLists.txt index c4e7ac3f6f..468bdc19ec 100644 --- a/src/video_core/host_shaders/CMakeLists.txt +++ b/src/video_core/host_shaders/CMakeLists.txt @@ -22,7 +22,6 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/blit_depth_stencil_msaa.frag ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d.comp - ${CMAKE_CURRENT_SOURCE_DIR}/block_linear_unswizzle_3d_bcn.comp ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d24s8.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_abgr8_to_d32f.frag ${CMAKE_CURRENT_SOURCE_DIR}/convert_d32f_to_abgr8.frag @@ -79,6 +78,7 @@ set(SHADER_FILES ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_fidelityfx_fsr_easu_fp32.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_fidelityfx_fsr_rcas_fp16.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_fidelityfx_fsr_rcas_fp32.frag + ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_indirect_quads.comp ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_present.frag ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_present.vert ${CMAKE_CURRENT_SOURCE_DIR}/vulkan_present_scaleforce_fp16.frag diff --git a/src/video_core/host_shaders/astc_decoder.comp b/src/video_core/host_shaders/astc_decoder.comp index e059c683ff..af6fac0e84 100644 --- a/src/video_core/host_shaders/astc_decoder.comp +++ b/src/video_core/host_shaders/astc_decoder.comp @@ -55,6 +55,7 @@ const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT; const uint BYTES_PER_BLOCK_LOG2 = 4; +// DO NOT CHANGE - code depends on the value of these! const uint JUST_BITS = 0u; const uint QUINT = 1u; const uint TRIT = 2u; @@ -62,27 +63,45 @@ const uint TRIT = 2u; // ASTC Encodings data, sorted in ascending order based on their BitLength value // (see GetBitLength() function) const uint encoding_values[22] = uint[]( - (JUST_BITS), (JUST_BITS | (1u << 8u)), (TRIT), (JUST_BITS | (2u << 8u)), - (QUINT), (TRIT | (1u << 8u)), (JUST_BITS | (3u << 8u)), (QUINT | (1u << 8u)), - (TRIT | (2u << 8u)), (JUST_BITS | (4u << 8u)), (QUINT | (2u << 8u)), (TRIT | (3u << 8u)), - (JUST_BITS | (5u << 8u)), (QUINT | (3u << 8u)), (TRIT | (4u << 8u)), (JUST_BITS | (6u << 8u)), - (QUINT | (4u << 8u)), (TRIT | (5u << 8u)), (JUST_BITS | (7u << 8u)), (QUINT | (5u << 8u)), - (TRIT | (6u << 8u)), (JUST_BITS | (8u << 8u))); + (JUST_BITS), + (JUST_BITS | (1u << 8u)), + (TRIT), + (JUST_BITS | (2u << 8u)), + (QUINT), + (TRIT | (1u << 8u)), + (JUST_BITS | (3u << 8u)), + (QUINT | (1u << 8u)), + (TRIT | (2u << 8u)), + (JUST_BITS | (4u << 8u)), + (QUINT | (2u << 8u)), + (TRIT | (3u << 8u)), + (JUST_BITS | (5u << 8u)), + (QUINT | (3u << 8u)), + (TRIT | (4u << 8u)), + (JUST_BITS | (6u << 8u)), + (QUINT | (4u << 8u)), + (TRIT | (5u << 8u)), + (JUST_BITS | (7u << 8u)), + (QUINT | (5u << 8u)), + (TRIT | (6u << 8u)), + (JUST_BITS | (8u << 8u)) +); // Input ASTC texture globals -int total_bitsread = 0; -uvec4 local_buff; - +uint total_bitsread = 0; // Color data globals uvec4 color_endpoint_data; -int color_bitsread = 0; +uint color_bitsread = 0; +// Global "vector" to be pushed into when decoding +// At most will require BLOCK_WIDTH x BLOCK_HEIGHT in single plane mode +// At most will require BLOCK_WIDTH x BLOCK_HEIGHT x 2 in dual plane mode +// So the maximum would be 144 (12 x 12) elements, x 2 for two planes #define MAX_WEIGHT_VALUES 64 -uint result_vector[MAX_WEIGHT_VALUES]; +uint result_vector[MAX_WEIGHT_VALUES + 1]; int result_index = 0; uint result_vector_max_index; -bool result_limit_reached = false; // EncodingData helpers uint Encoding(EncodingData val) { @@ -118,12 +137,10 @@ EncodingData CreateEncodingData(uint encoding, uint num_bits, uint bit_val, uint void ResultEmplaceBack(EncodingData val) { - if (result_index >= result_vector_max_index) { - // Alert callers to avoid decoding more than needed by this phase - result_limit_reached = true; - return; - } - result_vector[result_index] = val.data; + // A = result_index, B = result_vector_max_index + // A >= B -> A - B >= 0 + // A < B -> A - B < 0 + result_vector[min(uint(MAX_WEIGHT_VALUES), result_index)] = val.data; ++result_index; } @@ -139,49 +156,31 @@ uint ReplicateBitTo9(uint value) { return value * 511; } -uint ReplicateBits(uint value, uint num_bits, uint to_bit) { - if (value == 0 || num_bits == 0) { - return 0; - } - if (num_bits >= to_bit) { - return value; - } - const uint v = value & uint((1 << num_bits) - 1); - uint res = v; - uint reslen = num_bits; - while (reslen < to_bit) { - const uint num_dst_bits_to_shift_up = min(num_bits, to_bit - reslen); - const uint num_src_bits_to_shift_down = num_bits - num_dst_bits_to_shift_up; - - res <<= num_dst_bits_to_shift_up; - res |= (v >> num_src_bits_to_shift_down); - reslen += num_bits; - } - return res; +const uint mod6_table = 0 + | ((6 % 1) << (2 * 1)) | ((6 % 2) << (2 * 2)) + | ((6 % 3) << (2 * 3)) | ((6 % 4) << (2 * 4)) + | ((6 % 5) << (2 * 5)) | ((6 % 6) << (2 * 6)); +const uint mod8_table = 0 + | ((8 % 1) << (2 * 1)) | ((8 % 2) << (2 * 2)) + | ((8 % 3) << (2 * 3)) | ((8 % 4) << (2 * 4)) + | ((8 % 5) << (2 * 5)) | ((8 % 6) << (2 * 6)) + | ((8 % 7) << (2 * 7)) | ((8 % 8) << (2 * 8)); +// Assumes num_bits < to_bit, num_bits and to_bit != 0 +uint ReplicateBits(uint value, uint num_bits, uint to_bit, uint table) { + const uint repl = value & ((1 << num_bits) - 1); + const uint shift = (table >> (num_bits << 1)) & 3; + uint v = repl; + v |= v << (num_bits << 0); // [ xxxx xxrr ] + v |= v << (num_bits << 1); // [ xxxx rrrr ] + v |= v << (num_bits << 2); // [ rrrr rrrr ] + v = (v << shift) | repl >> (num_bits - shift); + return v & ((1 << to_bit) - 1); } - uint FastReplicateTo8(uint value, uint num_bits) { - return ReplicateBits(value, num_bits, 8); + return ReplicateBits(value, num_bits, 8, mod8_table); } - uint FastReplicateTo6(uint value, uint num_bits) { - return ReplicateBits(value, num_bits, 6); -} - -uint Div3Floor(uint v) { - return (v * 0x5556) >> 16; -} - -uint Div3Ceil(uint v) { - return Div3Floor(v + 2); -} - -uint Div5Floor(uint v) { - return (v * 0x3334) >> 16; -} - -uint Div5Ceil(uint v) { - return Div5Floor(v + 4); + return ReplicateBits(value, num_bits, 6, mod6_table); } uint Hash52(uint p) { @@ -198,139 +197,85 @@ uint Hash52(uint p) { return p; } -uint Select2DPartition(uint seed, uint x, uint y, uint partition_count) { - if ((block_dims.y * block_dims.x) < 32) { - x <<= 1; - y <<= 1; - } - +uint Select2DPartition(uint seed, uvec2 pos, uint partition_count) { + pos <<= uint(block_dims.y * block_dims.x < 32); seed += (partition_count - 1) * 1024; - const uint rnum = Hash52(uint(seed)); - uint seed1 = uint(rnum & 0xF); - uint seed2 = uint((rnum >> 4) & 0xF); - uint seed3 = uint((rnum >> 8) & 0xF); - uint seed4 = uint((rnum >> 12) & 0xF); - uint seed5 = uint((rnum >> 16) & 0xF); - uint seed6 = uint((rnum >> 20) & 0xF); - uint seed7 = uint((rnum >> 24) & 0xF); - uint seed8 = uint((rnum >> 28) & 0xF); - - seed1 = (seed1 * seed1); - seed2 = (seed2 * seed2); - seed3 = (seed3 * seed3); - seed4 = (seed4 * seed4); - seed5 = (seed5 * seed5); - seed6 = (seed6 * seed6); - seed7 = (seed7 * seed7); - seed8 = (seed8 * seed8); - - uint sh1, sh2; - if ((seed & 1) > 0) { - sh1 = (seed & 2) > 0 ? 4 : 5; - sh2 = (partition_count == 3) ? 6 : 5; - } else { - sh1 = (partition_count == 3) ? 6 : 5; - sh2 = (seed & 2) > 0 ? 4 : 5; - } - seed1 >>= sh1; - seed2 >>= sh2; - seed3 >>= sh1; - seed4 >>= sh2; - seed5 >>= sh1; - seed6 >>= sh2; - seed7 >>= sh1; - seed8 >>= sh2; - - uint a = seed1 * x + seed2 * y + (rnum >> 14); - uint b = seed3 * x + seed4 * y + (rnum >> 10); - uint c = seed5 * x + seed6 * y + (rnum >> 6); - uint d = seed7 * x + seed8 * y + (rnum >> 2); - - a &= 0x3F; - b &= 0x3F; - c &= 0x3F; - d &= 0x3F; - - if (partition_count < 4) { - d = 0; - } - if (partition_count < 3) { - c = 0; - } - - if (a >= b && a >= c && a >= d) { + uvec2 shift = uvec2( + (seed & 2) > 0 ? 4 : 5, + (partition_count == 3) ? 6 : 5 + ); + shift.xy = (seed & 1) > 0 ? shift.xy : shift.yx; + uvec4 rseed[2] = uvec4[]( + (uvec4(rnum) >> uvec4(0, 4, 8, 12)) & 0xf, + (uvec4(rnum) >> uvec4(16, 20, 24, 28)) & 0xf + ); + rseed[0] = (rseed[0] * rseed[0]) >> shift.xyxy; + rseed[1] = (rseed[1] * rseed[1]) >> shift.xyxy; + const uvec4 rnum_vec = uvec4(rnum) >> uvec4(14, 10, 6, 2); + const uvec4 result_mask = ((uvec4(0, 1, 2, 3) - partition_count) >> 8) & 0x3f; + uvec4 result = uvec4( + rseed[0].xz * pos.xx + rseed[0].yw * pos.yy + rnum_vec.xy, + rseed[1].xz * pos.xx + rseed[1].yw * pos.yy + rnum_vec.zw + ) & result_mask; + if (result.x >= result.y && result.x >= result.z && result.x >= result.w) { return 0; - } else if (b >= c && b >= d) { + } else if (result.y >= result.z && result.y >= result.w) { return 1; - } else if (c >= d) { + } else if (result.z >= result.w) { return 2; } else { return 3; } } -uint ExtractBits(uvec4 payload, int offset, int bits) { - if (bits <= 0) { +uint ExtractBits(uvec4 payload, uint offset, uint bits) { + if (bits == 0 || bits > 32 || offset >= 128) return 0; - } - if (bits > 32) { - return 0; - } - const int last_offset = offset + bits - 1; - const int shifted_offset = offset >> 5; - if ((last_offset >> 5) == shifted_offset) { - return bitfieldExtract(payload[shifted_offset], offset & 31, bits); - } - const int first_bits = 32 - (offset & 31); - const int result_first = int(bitfieldExtract(payload[shifted_offset], offset & 31, first_bits)); - const int result_second = int(bitfieldExtract(payload[shifted_offset + 1], 0, bits - first_bits)); - return result_first | (result_second << first_bits); + const uint avail_bits = min(bits, 128 - offset); + const uint last_offset = offset + avail_bits - 1; + const uint shifted_offset = offset >> 5; + if ((last_offset >> 5) == shifted_offset) + return bitfieldExtract(payload[shifted_offset], int(offset & 31), int(avail_bits)); + const uint first_bits = 32 - (offset & 31); + const uvec4 next = uvec4(payload.yzw, 0); + return bitfieldExtract(payload[shifted_offset], int(offset & 31), int(first_bits)) + | (bitfieldExtract(next[shifted_offset], 0, int(avail_bits - first_bits)) << first_bits); } -uint StreamBits(uint num_bits) { - const int int_bits = int(num_bits); - const uint ret = ExtractBits(local_buff, total_bitsread, int_bits); - total_bitsread += int_bits; +uint StreamBits(uvec4 local_buff, uint num_bits) { + const uint ret = ExtractBits(local_buff, total_bitsread, num_bits); + total_bitsread += num_bits; return ret; } void SkipBits(uint num_bits) { - const int int_bits = int(num_bits); - total_bitsread += int_bits; + total_bitsread += num_bits; } uint StreamColorBits(uint num_bits) { - const int int_bits = int(num_bits); - const uint ret = ExtractBits(color_endpoint_data, color_bitsread, int_bits); - color_bitsread += int_bits; + const uint ret = ExtractBits(color_endpoint_data, color_bitsread, num_bits); + color_bitsread += num_bits; return ret; } -EncodingData GetEncodingFromVector(uint index) { - const uint data = result_vector[index]; - return EncodingData(data); -} - // Returns the number of bits required to encode n_vals values. uint GetBitLength(uint n_vals, uint encoding_index) { + // uint Div3Floor(uint v) { return (v * 0x5556) >> 16; } + // uint Div3Ceil(uint v) { return Div3Floor(v + 2); } + // uint Div5Floor(uint v) { return (v * 0x3334) >> 16; } + // uint Div5Ceil(uint v) { return Div5Floor(v + 4); } const EncodingData encoding_value = EncodingData(encoding_values[encoding_index]); const uint encoding = Encoding(encoding_value); - uint total_bits = NumBits(encoding_value) * n_vals; - if (encoding == TRIT) { - total_bits += Div5Ceil(n_vals * 8); - } else if (encoding == QUINT) { - total_bits += Div3Ceil(n_vals * 7); - } - return total_bits; + const uint num_bits = NumBits(encoding_value); + const uvec3 div_constant = uvec3(0, 0x5556, 0x3334); + return num_bits * n_vals + + ((((n_vals * ((0x870 >> (encoding << 2)) & 0xf)) + ((0x420 >> (encoding << 2)) & 0xf)) + * div_constant[encoding]) >> 16); } uint GetNumWeightValues(uvec2 size, bool dual_plane) { - uint n_vals = size.x * size.y; - if (dual_plane) { - n_vals *= 2; - } - return n_vals; + return (size.x * size.y) << uint(dual_plane); } uint GetPackedBitSize(uvec2 size, bool dual_plane, uint max_weight) { @@ -359,13 +304,16 @@ void DecodeQuintBlock(uint num_bits) { if (BitsOp(qQ.w, 1, 2) == 3 && BitsOp(qQ.w, 5, 6) == 0) { qQ.x = 4; qQ.y = 4; - qQ.z = (BitsBracket(qQ.w, 0) << 2) | ((BitsBracket(qQ.w, 4) & ~BitsBracket(qQ.w, 0)) << 1) | - (BitsBracket(qQ.w, 3) & ~BitsBracket(qQ.w, 0)); + qQ.z = (BitsBracket(qQ.w, 0) << 2) + | ((BitsBracket(qQ.w, 4) & ~BitsBracket(qQ.w, 0)) << 1) + | (BitsBracket(qQ.w, 3) & ~BitsBracket(qQ.w, 0)); } else { uint C = 0; if (BitsOp(qQ.w, 1, 2) == 3) { qQ.z = 4; - C = (BitsOp(qQ.w, 3, 4) << 3) | ((~BitsOp(qQ.w, 5, 6) & 3) << 1) | BitsBracket(qQ.w, 0); + C = (BitsOp(qQ.w, 3, 4) << 3) + | ((~BitsOp(qQ.w, 5, 6) & 3) << 1) + | BitsBracket(qQ.w, 0); } else { qQ.z = BitsOp(qQ.w, 5, 6); C = BitsOp(qQ.w, 0, 4); @@ -378,39 +326,37 @@ void DecodeQuintBlock(uint num_bits) { qQ.x = BitsOp(C, 0, 2); } } - for (uint i = 0; i < 3; i++) { - const EncodingData val = CreateEncodingData(QUINT, num_bits, m[i], qQ[i]); - ResultEmplaceBack(val); - } + for (uint i = 0; i < 3; i++) + ResultEmplaceBack(CreateEncodingData(QUINT, num_bits, m[i], qQ[i])); } void DecodeTritBlock(uint num_bits) { uvec4 m; uvec4 t; - uvec3 Tm5t5; + uvec3 tm5t5; m[0] = StreamColorBits(num_bits); - Tm5t5.x = StreamColorBits(2); + tm5t5.x = StreamColorBits(2); m[1] = StreamColorBits(num_bits); - Tm5t5.x |= StreamColorBits(2) << 2; + tm5t5.x |= StreamColorBits(2) << 2; m[2] = StreamColorBits(num_bits); - Tm5t5.x |= StreamColorBits(1) << 4; + tm5t5.x |= StreamColorBits(1) << 4; m[3] = StreamColorBits(num_bits); - Tm5t5.x |= StreamColorBits(2) << 5; - Tm5t5.y = StreamColorBits(num_bits); - Tm5t5.x |= StreamColorBits(1) << 7; + tm5t5.x |= StreamColorBits(2) << 5; + tm5t5.y = StreamColorBits(num_bits); + tm5t5.x |= StreamColorBits(1) << 7; uint C = 0; - if (BitsOp(Tm5t5.x, 2, 4) == 7) { - C = (BitsOp(Tm5t5.x, 5, 7) << 2) | BitsOp(Tm5t5.x, 0, 1); - Tm5t5.z = 2; + if (BitsOp(tm5t5.x, 2, 4) == 7) { + C = (BitsOp(tm5t5.x, 5, 7) << 2) | BitsOp(tm5t5.x, 0, 1); + tm5t5.z = 2; t[3] = 2; } else { - C = BitsOp(Tm5t5.x, 0, 4); - if (BitsOp(Tm5t5.x, 5, 6) == 3) { - Tm5t5.z = 2; - t[3] = BitsBracket(Tm5t5.x, 7); + C = BitsOp(tm5t5.x, 0, 4); + if (BitsOp(tm5t5.x, 5, 6) == 3) { + tm5t5.z = 2; + t[3] = BitsBracket(tm5t5.x, 7); } else { - Tm5t5.z = BitsBracket(Tm5t5.x, 7); - t[3] = BitsOp(Tm5t5.x, 5, 6); + tm5t5.z = BitsBracket(tm5t5.x, 7); + t[3] = BitsOp(tm5t5.x, 5, 6); } } if (BitsOp(C, 0, 1) == 3) { @@ -426,156 +372,147 @@ void DecodeTritBlock(uint num_bits) { t[1] = BitsOp(C, 2, 3); t[0] = (BitsBracket(C, 1) << 1) | (BitsBracket(C, 0) & ~BitsBracket(C, 1)); } - for (uint i = 0; i < 4; i++) { - const EncodingData val = CreateEncodingData(TRIT, num_bits, m[i], t[i]); - ResultEmplaceBack(val); - } - const EncodingData val = CreateEncodingData(TRIT, num_bits, Tm5t5.y, Tm5t5.z); - ResultEmplaceBack(val); + for (uint i = 0; i < 4; i++) + ResultEmplaceBack(CreateEncodingData(TRIT, num_bits, m[i], t[i])); + ResultEmplaceBack(CreateEncodingData(TRIT, num_bits, tm5t5.y, tm5t5.z)); } void DecodeIntegerSequence(uint max_range, uint num_values) { EncodingData val = EncodingData(encoding_values[max_range]); const uint encoding = Encoding(val); const uint num_bits = NumBits(val); - uint vals_decoded = 0; - while (vals_decoded < num_values && !result_limit_reached) { + for (uint i = 0; i < num_values && result_index < result_vector_max_index; ) { switch (encoding) { case QUINT: DecodeQuintBlock(num_bits); - vals_decoded += 3; + i += 3; break; case TRIT: DecodeTritBlock(num_bits); - vals_decoded += 5; + i += 5; break; case JUST_BITS: BitValue(val, StreamColorBits(num_bits)); ResultEmplaceBack(val); - vals_decoded++; + i++; break; } } } -void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, out uint color_values[32]) { - uint num_values = 0; - for (uint i = 0; i < num_partitions; i++) { - num_values += ((modes[i] >> 2) + 1) << 1; +uint DecodeSingleColorValue(EncodingData val) { + const uint encoding = Encoding(val); + const uint bitlen = NumBits(val); + const uint bitval = BitValue(val); + uint B = 0, C = 0, D = 0; + uint A = ReplicateBitTo9((bitval & 1)); + switch (encoding) { + case JUST_BITS: + break; + case TRIT: { + D = QuintTritValue(val); + switch (bitlen) { + case 1: + C = 204; + break; + case 2: { + C = 93; + const uint b = (bitval >> 1) & 1; + B = (b << 8) | (b << 4) | (b << 2) | (b << 1); + break; + } + case 3: { + C = 44; + const uint cb = (bitval >> 1) & 3; + B = (cb << 7) | (cb << 2) | cb; + break; + } + case 4: { + C = 22; + const uint dcb = (bitval >> 1) & 7; + B = (dcb << 6) | dcb; + break; + } + case 5: { + C = 11; + const uint edcb = (bitval >> 1) & 0xF; + B = (edcb << 5) | (edcb >> 2); + break; + } + case 6: { + C = 5; + const uint fedcb = (bitval >> 1) & 0x1F; + B = (fedcb << 4) | (fedcb >> 4); + break; + } + } + break; } + case QUINT: { + D = QuintTritValue(val); + switch (bitlen) { + case 1: + C = 113; + break; + case 2: { + C = 54; + const uint b = (bitval >> 1) & 1; + B = (b << 8) | (b << 3) | (b << 2); + break; + } + case 3: { + C = 26; + const uint cb = (bitval >> 1) & 3; + B = (cb << 7) | (cb << 1) | (cb >> 1); + break; + } + case 4: { + C = 13; + const uint dcb = (bitval >> 1) & 7; + B = (dcb << 6) | (dcb >> 1); + break; + } + case 5: { + C = 6; + const uint edcb = (bitval >> 1) & 0xF; + B = (edcb << 5) | (edcb >> 3); + break; + } + } + break; + } + } + uint unq = D * C + B; + unq = unq ^ A; + unq = (A & 0x80) | (unq >> 2); + return encoding == JUST_BITS ? FastReplicateTo8(bitval, bitlen) : unq; +} + +void DecodeColorValues(uvec4 modes, uint num_partitions, uint color_data_bits, out uint color_values[32]) { + // TODO: modes[] zero on invalid, so less ops + const uvec4 num_values_tmp = (((modes >> 2) + 1) << 1) & ((uvec4(0, 1, 2, 3) - num_partitions) >> 8); + uint num_values = num_values_tmp.x + num_values_tmp.y + num_values_tmp.z + num_values_tmp.w; // Find the largest encoding that's within color_data_bits // TODO(ameerj): profile with binary search - int range = 0; + uint range = 0; while (++range < encoding_values.length()) { - const uint bit_length = GetBitLength(num_values, range); + const uint bit_length = GetBitLength(num_values, int(range)); if (bit_length > color_data_bits) { break; } } + const uint upper_bound = num_values; DecodeIntegerSequence(range - 1, num_values); - uint out_index = 0; - for (int itr = 0; itr < result_index; ++itr) { - if (out_index >= num_values) { - break; - } - const EncodingData val = GetEncodingFromVector(itr); - const uint encoding = Encoding(val); - const uint bitlen = NumBits(val); - const uint bitval = BitValue(val); - uint A = 0, B = 0, C = 0, D = 0; - A = ReplicateBitTo9((bitval & 1)); - switch (encoding) { - case JUST_BITS: - color_values[out_index++] = FastReplicateTo8(bitval, bitlen); - break; - case TRIT: { - D = QuintTritValue(val); - switch (bitlen) { - case 1: - C = 204; - break; - case 2: { - C = 93; - const uint b = (bitval >> 1) & 1; - B = (b << 8) | (b << 4) | (b << 2) | (b << 1); - break; - } - case 3: { - C = 44; - const uint cb = (bitval >> 1) & 3; - B = (cb << 7) | (cb << 2) | cb; - break; - } - case 4: { - C = 22; - const uint dcb = (bitval >> 1) & 7; - B = (dcb << 6) | dcb; - break; - } - case 5: { - C = 11; - const uint edcb = (bitval >> 1) & 0xF; - B = (edcb << 5) | (edcb >> 2); - break; - } - case 6: { - C = 5; - const uint fedcb = (bitval >> 1) & 0x1F; - B = (fedcb << 4) | (fedcb >> 4); - break; - } - } - break; - } - case QUINT: { - D = QuintTritValue(val); - switch (bitlen) { - case 1: - C = 113; - break; - case 2: { - C = 54; - const uint b = (bitval >> 1) & 1; - B = (b << 8) | (b << 3) | (b << 2); - break; - } - case 3: { - C = 26; - const uint cb = (bitval >> 1) & 3; - B = (cb << 7) | (cb << 1) | (cb >> 1); - break; - } - case 4: { - C = 13; - const uint dcb = (bitval >> 1) & 7; - B = (dcb << 6) | (dcb >> 1); - break; - } - case 5: { - C = 6; - const uint edcb = (bitval >> 1) & 0xF; - B = (edcb << 5) | (edcb >> 3); - break; - } - } - break; - } - } - if (encoding != JUST_BITS) { - uint T = (D * C) + B; - T ^= A; - T = (A & 0x80) | (T >> 2); - color_values[out_index++] = T; - } + for (int i = 0; i < upper_bound; ++i) { + color_values[i + 1] = DecodeSingleColorValue(EncodingData(result_vector[i])); } } ivec2 BitTransferSigned(int a, int b) { - ivec2 transferred; - transferred.y = b >> 1; - transferred.y |= a & 0x80; - transferred.x = a >> 1; - transferred.x &= 0x3F; + ivec2 transferred = ivec2( + (a >> 1) & 0x3F, + (b >> 1) | (a & 0x80) + ); if ((transferred.x & 0x20) > 0) { transferred.x -= 0x40; } @@ -590,169 +527,17 @@ ivec4 BlueContract(int a, int r, int g, int b) { return ivec4(a, (r + b) >> 1, (g + b) >> 1, b); } -bool IsHDRColorEndpointMode(uint cem) { - return cem == 2u || cem == 3u || cem == 7u || cem == 11u || cem == 14u || cem == 15u; -} - -int SignExtend(int value, uint nbits) { - int sign_bit = 1 << (nbits - 1u); - return (value ^ sign_bit) - sign_bit; -} - -void DecodeHDREndpointMode7(uint v0, uint v1, uint v2, uint v3, out ivec3 e0, out ivec3 e1) { - uint modeval = ((v0 & 0xC0u) >> 6u) | ((v1 & 0x80u) >> 5u) | ((v2 & 0x80u) >> 4u); - - uint majcomp; - uint mode; - if ((modeval & 0xCu) != 0xCu) { - majcomp = modeval >> 2u; - mode = modeval & 3u; - } else if (modeval != 0xFu) { - majcomp = modeval & 3u; - mode = 4u; - } else { - majcomp = 0u; - mode = 5u; - } - - int red = int(v0 & 0x3Fu); - int green = int(v1 & 0x1Fu); - int blue = int(v2 & 0x1Fu); - int scale = int(v3 & 0x1Fu); - - uint x0 = (v1 >> 6u) & 1u; - uint x1 = (v1 >> 5u) & 1u; - uint x2 = (v2 >> 6u) & 1u; - uint x3 = (v2 >> 5u) & 1u; - uint x4 = (v3 >> 7u) & 1u; - uint x5 = (v3 >> 6u) & 1u; - uint x6 = (v3 >> 5u) & 1u; - - uint ohm = 1u << mode; - if ((ohm & 0x30u) != 0u) green |= int(x0 << 6u); - if ((ohm & 0x3Au) != 0u) green |= int(x1 << 5u); - if ((ohm & 0x30u) != 0u) blue |= int(x2 << 6u); - if ((ohm & 0x3Au) != 0u) blue |= int(x3 << 5u); - if ((ohm & 0x3Du) != 0u) scale |= int(x6 << 5u); - if ((ohm & 0x2Du) != 0u) scale |= int(x5 << 6u); - if ((ohm & 0x04u) != 0u) scale |= int(x4 << 7u); - if ((ohm & 0x3Bu) != 0u) red |= int(x4 << 6u); - if ((ohm & 0x04u) != 0u) red |= int(x3 << 6u); - if ((ohm & 0x10u) != 0u) red |= int(x5 << 7u); - if ((ohm & 0x0Fu) != 0u) red |= int(x2 << 7u); - if ((ohm & 0x05u) != 0u) red |= int(x1 << 8u); - if ((ohm & 0x0Au) != 0u) red |= int(x0 << 8u); - if ((ohm & 0x05u) != 0u) red |= int(x0 << 9u); - if ((ohm & 0x02u) != 0u) red |= int(x6 << 9u); - if ((ohm & 0x01u) != 0u) red |= int(x3 << 10u); - if ((ohm & 0x02u) != 0u) red |= int(x5 << 10u); - - int shamts[6] = int[](1, 1, 2, 3, 4, 5); - int shamt = shamts[mode]; - red <<= shamt; - green <<= shamt; - blue <<= shamt; - scale <<= shamt; - - if (mode != 5u) { - green = red - green; - blue = red - blue; - } - - if (majcomp == 1u) { - int t = red; red = green; green = t; - } - if (majcomp == 2u) { - int t = red; red = blue; blue = t; - } - - e1 = ivec3(clamp(red, 0, 0xFFF), clamp(green, 0, 0xFFF), clamp(blue, 0, 0xFFF)); - e0 = ivec3(clamp(red - scale, 0, 0xFFF), clamp(green - scale, 0, 0xFFF), - clamp(blue - scale, 0, 0xFFF)); -} - -void DecodeHDREndpointMode11(uint v0, uint v1, uint v2, uint v3, uint v4, uint v5, out ivec3 e0, - out ivec3 e1) { - uint majcomp = ((v4 & 0x80u) >> 7u) | ((v5 & 0x80u) >> 6u); - if (majcomp == 3u) { - e0 = ivec3(int(v0 << 4u), int(v2 << 4u), int((v4 & 0x7Fu) << 5u)); - e1 = ivec3(int(v1 << 4u), int(v3 << 4u), int((v5 & 0x7Fu) << 5u)); - return; - } - - uint mode = ((v1 & 0x80u) >> 7u) | ((v2 & 0x80u) >> 6u) | ((v3 & 0x80u) >> 5u); - int va = int(v0 | ((v1 & 0x40u) << 2u)); - int vb0 = int(v2 & 0x3Fu); - int vb1 = int(v3 & 0x3Fu); - int vc = int(v1 & 0x3Fu); - int vd0 = int(v4 & 0x7Fu); - int vd1 = int(v5 & 0x7Fu); - - int dbitstab[8] = int[](7, 6, 7, 6, 5, 6, 5, 6); - vd0 = SignExtend(vd0, uint(dbitstab[mode])); - vd1 = SignExtend(vd1, uint(dbitstab[mode])); - - uint x0 = (v2 >> 6u) & 1u; - uint x1 = (v3 >> 6u) & 1u; - uint x2 = (v4 >> 6u) & 1u; - uint x3 = (v5 >> 6u) & 1u; - uint x4 = (v4 >> 5u) & 1u; - uint x5 = (v5 >> 5u) & 1u; - - uint ohm = 1u << mode; - if ((ohm & 0xA4u) != 0u) va |= int(x0 << 9u); - if ((ohm & 0x08u) != 0u) va |= int(x2 << 9u); - if ((ohm & 0x50u) != 0u) va |= int(x4 << 9u); - if ((ohm & 0x50u) != 0u) va |= int(x5 << 10u); - if ((ohm & 0xA0u) != 0u) va |= int(x1 << 10u); - if ((ohm & 0xC0u) != 0u) va |= int(x2 << 11u); - if ((ohm & 0x04u) != 0u) vc |= int(x1 << 6u); - if ((ohm & 0xE8u) != 0u) vc |= int(x3 << 6u); - if ((ohm & 0x20u) != 0u) vc |= int(x2 << 7u); - if ((ohm & 0x5Bu) != 0u) vb0 |= int(x0 << 6u); - if ((ohm & 0x5Bu) != 0u) vb1 |= int(x1 << 6u); - if ((ohm & 0x12u) != 0u) vb0 |= int(x2 << 7u); - if ((ohm & 0x12u) != 0u) vb1 |= int(x3 << 7u); - - int shamt = (int(mode) >> 1) ^ 3; - va <<= shamt; - vb0 <<= shamt; - vb1 <<= shamt; - vc <<= shamt; - vd0 <<= shamt; - vd1 <<= shamt; - - int r1 = clamp(va, 0, 0xFFF); - int g1 = clamp(va - vb0, 0, 0xFFF); - int b1 = clamp(va - vb1, 0, 0xFFF); - int r0 = clamp(va - vc, 0, 0xFFF); - int g0 = clamp(va - vb0 - vc - vd0, 0, 0xFFF); - int b0 = clamp(va - vb1 - vc - vd1, 0, 0xFFF); - - if (majcomp == 1u) { - int t; - t = r0; r0 = g0; g0 = t; - t = r1; r1 = g1; g1 = t; - } else if (majcomp == 2u) { - int t; - t = r0; r0 = b0; b0 = t; - t = r1; r1 = b1; b1 = t; - } - e0 = ivec3(r0, g0, b0); - e1 = ivec3(r1, g1, b1); -} - void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, uint color_values[32], inout uint colvals_index) { #define READ_UINT_VALUES(N) \ uvec4 V[2]; \ for (uint i = 0; i < N; i++) { \ - V[i / 4][i % 4] = color_values[colvals_index++]; \ + V[i >> 2][i & 3] = color_values[++colvals_index]; \ } #define READ_INT_VALUES(N) \ ivec4 V[2]; \ for (uint i = 0; i < N; i++) { \ - V[i / 4][i % 4] = int(color_values[colvals_index++]); \ + V[i >> 2][i & 3] = int(color_values[++colvals_index]); \ } switch (color_endpoint_mode) { @@ -778,12 +563,8 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui } case 5: { READ_INT_VALUES(4) - ivec2 transferred = BitTransferSigned(V[0].y, V[0].x); - V[0].y = transferred.x; - V[0].x = transferred.y; - transferred = BitTransferSigned(V[0].w, V[0].z); - V[0].w = transferred.x; - V[0].z = transferred.y; + V[0].yx = BitTransferSigned(V[0].y, V[0].x); + V[0].wz = BitTransferSigned(V[0].w, V[0].z); ep1 = ClampByte(ivec4(V[0].z, V[0].x, V[0].x, V[0].x)); ep2 = ClampByte(ivec4(V[0].z + V[0].w, V[0].x + V[0].y, V[0].x + V[0].y, V[0].x + V[0].y)); break; @@ -807,15 +588,9 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui } case 9: { READ_INT_VALUES(6) - ivec2 transferred = BitTransferSigned(V[0].y, V[0].x); - V[0].y = transferred.x; - V[0].x = transferred.y; - transferred = BitTransferSigned(V[0].w, V[0].z); - V[0].w = transferred.x; - V[0].z = transferred.y; - transferred = BitTransferSigned(V[1].y, V[1].x); - V[1].y = transferred.x; - V[1].x = transferred.y; + V[0].yx = BitTransferSigned(V[0].y, V[0].x); + V[0].wz = BitTransferSigned(V[0].w, V[0].z); + V[1].yx = BitTransferSigned(V[1].y, V[1].x); if ((V[0].y + V[0].w + V[1].y) >= 0) { ep1 = ClampByte(ivec4(0xFF, V[0].x, V[0].z, V[1].x)); ep2 = ClampByte(ivec4(0xFF, V[0].x + V[0].y, V[0].z + V[0].w, V[1].x + V[1].y)); @@ -844,21 +619,10 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui } case 13: { READ_INT_VALUES(8) - ivec2 transferred = BitTransferSigned(V[0].y, V[0].x); - V[0].y = transferred.x; - V[0].x = transferred.y; - transferred = BitTransferSigned(V[0].w, V[0].z); - V[0].w = transferred.x; - V[0].z = transferred.y; - - transferred = BitTransferSigned(V[1].y, V[1].x); - V[1].y = transferred.x; - V[1].x = transferred.y; - - transferred = BitTransferSigned(V[1].w, V[1].z); - V[1].w = transferred.x; - V[1].z = transferred.y; - + V[0].yx = BitTransferSigned(V[0].y, V[0].x); + V[0].wz = BitTransferSigned(V[0].w, V[0].z); + V[1].yx = BitTransferSigned(V[1].y, V[1].x); + V[1].wz = BitTransferSigned(V[1].w, V[1].z); if ((V[0].y + V[0].w + V[1].y) >= 0) { ep1 = ClampByte(ivec4(V[1].z, V[0].x, V[0].z, V[1].x)); ep2 = ClampByte(ivec4(V[1].w + V[1].z, V[0].x + V[0].y, V[0].z + V[0].w, V[1].x + V[1].y)); @@ -868,86 +632,8 @@ void ComputeEndpoints(out uvec4 ep1, out uvec4 ep2, uint color_endpoint_mode, ui } break; } - case 2: { - READ_UINT_VALUES(2) - uint y0, y1; - if (V[0].y >= V[0].x) { - y0 = V[0].x << 4u; - y1 = V[0].y << 4u; - } else { - y0 = (V[0].y << 4u) + 8u; - y1 = (V[0].x << 4u) - 8u; - } - ep1 = uvec4(0x780u, y0, y0, y0); - ep2 = uvec4(0x780u, y1, y1, y1); - break; - } - case 3: { - READ_UINT_VALUES(2) - uint y0, d; - if ((V[0].x & 0x80u) != 0u) { - y0 = ((V[0].y & 0xE0u) << 4u) | ((V[0].x & 0x7Fu) << 2u); - d = (V[0].y & 0x1Fu) << 2u; - } else { - y0 = ((V[0].y & 0xF0u) << 4u) | ((V[0].x & 0x7Fu) << 1u); - d = (V[0].y & 0x0Fu) << 1u; - } - const uint y1 = min(y0 + d, 0xFFFu); - ep1 = uvec4(0x780u, y0, y0, y0); - ep2 = uvec4(0x780u, y1, y1, y1); - break; - } - case 7: { - READ_UINT_VALUES(4) - ivec3 e0, e1; - DecodeHDREndpointMode7(V[0].x, V[0].y, V[0].z, V[0].w, e0, e1); - ep1 = uvec4(0x780u, uint(e0.x), uint(e0.y), uint(e0.z)); - ep2 = uvec4(0x780u, uint(e1.x), uint(e1.y), uint(e1.z)); - break; - } - case 11: { - READ_UINT_VALUES(6) - ivec3 e0, e1; - DecodeHDREndpointMode11(V[0].x, V[0].y, V[0].z, V[0].w, V[1].x, V[1].y, e0, e1); - ep1 = uvec4(0x780u, uint(e0.x), uint(e0.y), uint(e0.z)); - ep2 = uvec4(0x780u, uint(e1.x), uint(e1.y), uint(e1.z)); - break; - } - case 14: { - READ_UINT_VALUES(8) - ivec3 e0, e1; - DecodeHDREndpointMode11(V[0].x, V[0].y, V[0].z, V[0].w, V[1].x, V[1].y, e0, e1); - ep1 = uvec4(V[1].z, uint(e0.x), uint(e0.y), uint(e0.z)); - ep2 = uvec4(V[1].w, uint(e1.x), uint(e1.y), uint(e1.z)); - break; - } - case 15: { - READ_UINT_VALUES(8) - ivec3 e0, e1; - DecodeHDREndpointMode11(V[0].x, V[0].y, V[0].z, V[0].w, V[1].x, V[1].y, e0, e1); - const uint mode = ((V[1].z >> 7u) & 1u) | ((V[1].w >> 6u) & 2u); - int a6 = int(V[1].z & 0x7Fu); - int a7 = int(V[1].w & 0x7Fu); - int alpha0, alpha1; - if (mode == 3u) { - alpha0 = a6 << 5; - alpha1 = a7 << 5; - } else { - a6 |= (a7 << int(mode + 1u)) & 0x780; - a7 &= int(0x3Fu >> mode); - a7 ^= int(0x20u >> mode); - a7 -= int(0x20u >> mode); - a6 <<= int(4u - mode); - a7 <<= int(4u - mode); - a7 += a6; - alpha0 = a6; - alpha1 = clamp(a7, 0, 0xFFF); - } - ep1 = uvec4(uint(alpha0), uint(e0.x), uint(e0.y), uint(e0.z)); - ep2 = uvec4(uint(alpha1), uint(e1.x), uint(e1.y), uint(e1.z)); - break; - } default: { + // HDR mode, or more likely a bug computing the color_endpoint_mode ep1 = uvec4(0xFF, 0xFF, 0, 0); ep2 = uvec4(0xFF, 0xFF, 0, 0); break; @@ -965,19 +651,19 @@ uint UnquantizeTexelWeight(EncodingData val) { : FastReplicateTo6(bitval, bitlen); } else if (encoding == TRIT || encoding == QUINT) { uint B = 0, C = 0, D = 0; - uint b_mask = (0x3100 >> (bitlen * 4)) & 0xf; + uint b_mask = (0x3100 >> (bitlen << 2)) & 0xf; uint b = (bitval >> 1) & b_mask; D = QuintTritValue(val); if (encoding == TRIT) { switch (bitlen) { - case 0: return D * 32; //0,32,64 + case 0: return D << 5; //0,32,64 case 1: C = 50; break; case 2: C = 23; B = (b << 6) | (b << 2) | b; break; case 3: C = 11; B = (b << 5) | b; break; } } else if (encoding == QUINT) { switch (bitlen) { - case 0: return D * 16; //0, 16, 32, 48, 64 + case 0: return D << 4; //0, 16, 32, 48, 64 case 1: C = 28; break; case 2: C = 13; B = (b << 6) | (b << 1); break; } @@ -993,107 +679,49 @@ void UnquantizeTexelWeights(uvec2 size, bool is_dual_plane) { const uint num_planes = is_dual_plane ? 2 : 1; const uint area = size.x * size.y; const uint loop_count = min(result_index, area * num_planes); - for (uint itr = 0; itr < loop_count; ++itr) { - result_vector[itr] = - UnquantizeTexelWeight(GetEncodingFromVector(itr)); + for (uint i = 0; i < loop_count; ++i) { + result_vector[i] = UnquantizeTexelWeight(EncodingData(result_vector[i])); } } uint GetUnquantizedTexelWeight(uint offset_base, uint plane, bool is_dual_plane) { - const uint offset = is_dual_plane ? 2 * offset_base + plane : offset_base; + const uint offset = is_dual_plane ? (offset_base << 1) + plane : offset_base; return result_vector[offset]; } -uvec4 GetUnquantizedWeightVector(uint t, uint s, uvec2 size, uint plane_index, bool is_dual_plane) { - const uint Ds = uint((block_dims.x * 0.5f + 1024) / (block_dims.x - 1)); - const uint Dt = uint((block_dims.y * 0.5f + 1024) / (block_dims.y - 1)); +uvec4 GetUnquantizedWeightVector(uvec2 pos, uvec2 size, uint plane_index, bool is_dual_plane) { const uint area = size.x * size.y; - - const uint cs = Ds * s; - const uint ct = Dt * t; - const uint gs = (cs * (size.x - 1) + 32) >> 6; - const uint gt = (ct * (size.y - 1) + 32) >> 6; - const uint js = gs >> 4; - const uint fs = gs & 0xF; - const uint jt = gt >> 4; - const uint ft = gt & 0x0F; - const uint w11 = (fs * ft + 8) >> 4; - const uint w10 = ft - w11; - const uint w01 = fs - w11; - const uint w00 = 16 - fs - ft + w11; + const uvec2 D = uvec2((block_dims * 0.5f + 1024.0f) / (block_dims - 1.0f)); + const uvec2 c = D * pos; + const uvec2 g = (c * (size - 1) + 32) >> 6; + const uvec2 j = g >> 4; + const uvec2 f = g & 0xF; + const uint w11 = (f.x * f.y + 8) >> 4; + const uint w10 = f.y - w11; + const uint w01 = f.x - w11; + const uint w00 = 16 - f.x - f.y + w11; const uvec4 w = uvec4(w00, w01, w10, w11); - const uint v0 = jt * size.x + js; - - uvec4 p0 = uvec4(0); - uvec4 p1 = uvec4(0); - - if (v0 < area) { - const uint offset_base = v0; - p0.x = GetUnquantizedTexelWeight(offset_base, 0, is_dual_plane); - p1.x = GetUnquantizedTexelWeight(offset_base, 1, is_dual_plane); - } - if ((v0 + 1) < (area)) { - const uint offset_base = v0 + 1; - p0.y = GetUnquantizedTexelWeight(offset_base, 0, is_dual_plane); - p1.y = GetUnquantizedTexelWeight(offset_base, 1, is_dual_plane); - } - if ((v0 + size.x) < (area)) { - const uint offset_base = v0 + size.x; - p0.z = GetUnquantizedTexelWeight(offset_base, 0, is_dual_plane); - p1.z = GetUnquantizedTexelWeight(offset_base, 1, is_dual_plane); - } - if ((v0 + size.x + 1) < (area)) { - const uint offset_base = v0 + size.x + 1; - p0.w = GetUnquantizedTexelWeight(offset_base, 0, is_dual_plane); - p1.w = GetUnquantizedTexelWeight(offset_base, 1, is_dual_plane); - } - - const uint primary_weight = (uint(dot(p0, w)) + 8) >> 4; - - uvec4 weight_vec = uvec4(primary_weight); - - if (is_dual_plane) { - const uint secondary_weight = (uint(dot(p1, w)) + 8) >> 4; - for (uint c = 0; c < 4; c++) { - const bool is_secondary = ((plane_index + 1u) & 3u) == c; - weight_vec[c] = is_secondary ? secondary_weight : primary_weight; - } - } - return weight_vec; + const uint v0 = j.y * size.x + j.x; + // + const uvec4 offset_base = uvec4(v0, v0 + 1, v0 + size.x, v0 + size.x + 1); + const uvec4 cmp_mask = uvec4(0) - uvec4(lessThan(offset_base, uvec4(area))); + uvec4 p0 = uvec4( + GetUnquantizedTexelWeight(offset_base.x, 0, is_dual_plane) & cmp_mask.x, + GetUnquantizedTexelWeight(offset_base.y, 0, is_dual_plane) & cmp_mask.y, + GetUnquantizedTexelWeight(offset_base.z, 0, is_dual_plane) & cmp_mask.z, + GetUnquantizedTexelWeight(offset_base.w, 0, is_dual_plane) & cmp_mask.w + ); + uvec4 p1 = uvec4( + GetUnquantizedTexelWeight(offset_base.x, 1, is_dual_plane) & cmp_mask.x, + GetUnquantizedTexelWeight(offset_base.y, 1, is_dual_plane) & cmp_mask.y, + GetUnquantizedTexelWeight(offset_base.z, 1, is_dual_plane) & cmp_mask.z, + GetUnquantizedTexelWeight(offset_base.w, 1, is_dual_plane) & cmp_mask.w + ); + const uvec2 final_weight = (uvec2(uint(dot(p0, w)), uint(dot(p1, w))) + 8) >> 4; + const uvec4 plane_mask = uvec4(0) - (uvec4(equal(uvec4(((plane_index + 1u) & 3u)), uvec4(0, 1, 2, 3))) & uvec4(0 - uint(is_dual_plane))); + return (final_weight.yyyy & plane_mask) | (final_weight.xxxx & ~plane_mask); } -int FindLayout(uint mode) { - if ((mode & 3) != 0) { - if ((mode & 8) != 0) { - if ((mode & 4) != 0) { - if ((mode & 0x100) != 0) { - return 4; - } - return 3; - } - return 2; - } - if ((mode & 4) != 0) { - return 1; - } - return 0; - } - if ((mode & 0x100) != 0) { - if ((mode & 0x80) != 0) { - if ((mode & 0x20) != 0) { - return 8; - } - return 7; - } - return 9; - } - if ((mode & 0x80) != 0) { - return 6; - } - return 5; -} - - void FillError(ivec3 coord) { for (uint j = 0; j < block_dims.y; j++) { for (uint i = 0; i < block_dims.x; i++) { @@ -1102,12 +730,13 @@ void FillError(ivec3 coord) { } } -void FillVoidExtentLDR(ivec3 coord) { +void FillVoidExtentLDR(uvec4 local_buff, ivec3 coord) { + // TODO: If you do extract bits, remember that it may be 11, or OTHER SkipBits(52); - const uint r_u = StreamBits(16); - const uint g_u = StreamBits(16); - const uint b_u = StreamBits(16); - const uint a_u = StreamBits(16); + const uint r_u = StreamBits(local_buff, 16); + const uint g_u = StreamBits(local_buff, 16); + const uint b_u = StreamBits(local_buff, 16); + const uint a_u = StreamBits(local_buff, 16); const float a = float(a_u) / 65535.0f; const float r = float(r_u) / 65535.0f; const float g = float(g_u) / 65535.0f; @@ -1119,13 +748,13 @@ void FillVoidExtentLDR(ivec3 coord) { } } -bool IsError(uint mode) { +bool IsError(uvec4 local_buff, uint mode) { if ((mode & 0x1ff) == 0x1fc) { if ((mode & 0x200) != 0) { // params.void_extent_hdr = true; return true; } - if ((mode & 0x400) == 0 || StreamBits(1) == 0) { + if ((mode & 0x400) == 0 || StreamBits(local_buff, 1) == 0) { return true; } return false; @@ -1139,9 +768,36 @@ bool IsError(uint mode) { return false; } -uvec2 DecodeBlockSize(uint mode) { +int FindLayout(uint mode) { + // Possible (Relevant), x = dont care, (in hex) + // Before shift: 02x, 08x, 10x, 00x, 0Ax, 12x, 18x, 1Ax + // After shift: 01, 04, 08, 00, 05, 09, 0c, 0d + // Reassemble and remove middle 0x040 (moronic fuzzy table) + // 0000 -> 0 0020 -> 1 0040 -> 0 0060 -> 1 + // 0080 -> 2 00a0 -> 3 00c0 -> 2 00e0 -> 3 + // 0100 -> 4 0120 -> 5 0140 -> 4 0160 -> 5 + // 0180 -> 6 01a0 -> 7 01c0 -> 6 01e0 -> 7 + // Key ranges: 01a0 -> 7, 0180 -> 6, 0100 -> 4, 0080 -> 2 + // 8>>2 = 2, 4>>2 = 1 :: () = 0, (4) = 1, (8) = 2, (8,4) = 3 + const uint mask = 0 - uint((mode & 3) != 0); + const uint sh3_mode = (mode >> 2) & 3; + const uint sh0_mode = ((mode >> 6) & 6) | ((mode >> 5) & 1); + const uint fl_const_table = 0 + | ((1) << (2 * 4)) //0080 -> 2, 1 + 5 = 6 + | ((1) << (3 * 4)) + | ((4) << (4 * 4)) //0100 -> 4, 4 + 5 = 9 + | ((4) << (5 * 4)) + | ((2) << (6 * 4)) //0180 -> 6, 2 + 5 = 7 + | ((3) << (7 * 4)) //01a0 -> 7, 3 + 5 = 8 + ; + const uint if_mode3_t = sh3_mode + uint((mode & 0x10c) == 0x10c); + const uint if_mode3_f = 5 + ((fl_const_table >> (sh0_mode << 2)) & 7); + return int((if_mode3_t & mask) | (if_mode3_f & ~mask)); +} + +uvec2 DecodeBlockSize(uint mode_layout, uint mode) { uint A, B; - switch (FindLayout(mode)) { + switch (mode_layout) { case 0: A = (mode >> 5) & 0x3; B = (mode >> 7) & 0x3; @@ -1181,130 +837,81 @@ uvec2 DecodeBlockSize(uint mode) { } } -uint DecodeMaxWeight(uint mode) { - const uint mode_layout = FindLayout(mode); - uint weight_index = (mode & 0x10) != 0 ? 1 : 0; - if (mode_layout < 5) { - weight_index |= (mode & 0x3) << 1; - } else { - weight_index |= (mode & 0xc) >> 1; - } - weight_index -= 2; - if ((mode_layout != 9) && ((mode & 0x200) != 0)) { - weight_index += 6; - } - return weight_index + 1; +uint DecodeMaxWeight(uint mode_layout, uint mode) { + const uint mode0_or = (mode & 0x3) << 1; + const uint mode5_or = (mode & 0xc) >> 1; + const uint cmp_moden = 0 - uint(mode_layout < 5); + const uint cmp_add6 = 0 - (uint(mode_layout != 9) & (mode >> 9) & 1); + return (((mode >> 4) & 1) + | (mode0_or & cmp_moden) | (mode5_or & ~cmp_moden)) + + (6 & cmp_add6) - 1; } -void DecompressBlock(ivec3 coord) { - uint mode = StreamBits(11); - if (IsError(mode)) { +void DecompressBlock(uvec4 local_buff, ivec3 coord) { + uint mode = StreamBits(local_buff, 11); + if (IsError(local_buff, mode)) { FillError(coord); return; } if ((mode & 0x1ff) == 0x1fc) { // params.void_extent_ldr = true; - FillVoidExtentLDR(coord); + FillVoidExtentLDR(local_buff, coord); return; } - const uvec2 size_params = DecodeBlockSize(mode); - if ((size_params.x > block_dims.x) || (size_params.y > block_dims.y)) { - FillError(coord); - return; - } - const uint num_partitions = StreamBits(2) + 1; + const uint num_partitions = StreamBits(local_buff, 2) + 1; const uint mode_layout = FindLayout(mode); const bool dual_plane = (mode_layout != 9) && ((mode & 0x400) != 0); - if (num_partitions > 4 || (num_partitions == 4 && dual_plane)) { - FillError(coord); - return; - } - if (GetNumWeightValues(size_params, dual_plane) > MAX_WEIGHT_VALUES) { + const uvec2 size_params = DecodeBlockSize(mode_layout, mode); + if (size_params.x > block_dims.x || size_params.y > block_dims.y + || num_partitions > 4 || (num_partitions == 4 && dual_plane) + || GetNumWeightValues(size_params, dual_plane) > MAX_WEIGHT_VALUES) { FillError(coord); return; } uint partition_index = 1; uvec4 color_endpoint_mode = uvec4(0); - uint ced_pointer = 0; uint base_cem = 0; if (num_partitions == 1) { - color_endpoint_mode.x = StreamBits(4); + color_endpoint_mode.x = StreamBits(local_buff, 4); partition_index = 0; } else { - partition_index = StreamBits(10); - base_cem = StreamBits(6); + partition_index = StreamBits(local_buff, 10); + base_cem = StreamBits(local_buff, 6); } const uint base_mode = base_cem & 3; - const uint max_weight = DecodeMaxWeight(mode); + const uint max_weight = DecodeMaxWeight(mode_layout, mode); const uint weight_bits = GetPackedBitSize(size_params, dual_plane, max_weight); - if (weight_bits < 24 || weight_bits > 96) { - FillError(coord); - return; - } - uint remaining_bits = 128 - weight_bits - total_bitsread; - uint extra_cem_bits = 0; - if (base_mode > 0) { - switch (num_partitions) { - case 2: - extra_cem_bits += 2; - break; - case 3: - extra_cem_bits += 5; - break; - case 4: - extra_cem_bits += 8; - break; - default: - FillError(coord); - return; - } - } - remaining_bits -= extra_cem_bits; + const uint extra_cem_bits = base_mode > 0 ? ((0x85200 >> (num_partitions << 2)) & 0x0f) : 0; const uint plane_selector_bits = dual_plane ? 2 : 0; + uint remaining_bits = 128 - weight_bits - total_bitsread; + remaining_bits -= extra_cem_bits; remaining_bits -= plane_selector_bits; if (remaining_bits > 128) { // Bad data, more remaining bits than 4 bytes // return early - FillError(coord); return; } // Read color data... const uint color_data_bits = remaining_bits; - while (remaining_bits > 0) { + for (uint i = 0; remaining_bits > 0 && i < 4; ++i) { const int nb = int(min(remaining_bits, 32U)); - const uint b = StreamBits(nb); - color_endpoint_data[ced_pointer] = uint(bitfieldExtract(b, 0, nb)); - ++ced_pointer; + color_endpoint_data[i] = StreamBits(local_buff, nb); remaining_bits -= nb; } - const uint plane_index = uint(StreamBits(plane_selector_bits)); + + // color_endpoint_mode assumed to be 0 on invalids/out of "range" + const uint plane_index = uint(StreamBits(local_buff, plane_selector_bits)); + const uvec4 cem_mask = (uvec4(0, 1, 2, 3) - num_partitions) >> 8; if (base_mode > 0) { - const uint extra_cem = StreamBits(extra_cem_bits); - uint cem = (extra_cem << 6) | base_cem; - cem >>= 2; - uvec4 C = uvec4(0); - for (uint i = 0; i < num_partitions; i++) { - C[i] = (cem & 1); - cem >>= 1; - } - uvec4 M = uvec4(0); - for (uint i = 0; i < num_partitions; i++) { - M[i] = cem & 3; - cem >>= 2; - } - for (uint i = 0; i < num_partitions; i++) { - color_endpoint_mode[i] = base_mode; - if (C[i] == 0) { - --color_endpoint_mode[i]; - } - color_endpoint_mode[i] <<= 2; - color_endpoint_mode[i] |= M[i]; - } + const uint extra_cem = StreamBits(local_buff, extra_cem_bits); + const uint cem = ((extra_cem << 6) | base_cem) >> 2; + const uint c0 = cem & ((1 << num_partitions) - 1); + const uint c1 = (cem >> num_partitions) & ((1 << (num_partitions << 1)) - 1); + const uvec4 c = (uvec4(c0) >> uvec4(0, 1, 2, 3)) & 1; + const uvec4 m = (uvec4(c1) >> uvec4(0, 2, 4, 6)) & 3; + color_endpoint_mode = (((uvec4(base_mode) - (1 - c)) << 2) | m) & cem_mask; } else if (num_partitions > 1) { - const uint cem = base_cem >> 2; - for (uint i = 0; i < num_partitions; i++) { - color_endpoint_mode[i] = cem; - } + color_endpoint_mode = uvec4(base_cem >> 2) & cem_mask; } uvec4 endpoints0[4]; @@ -1315,90 +922,46 @@ void DecompressBlock(ivec3 coord) { uint color_values[32]; uint colvals_index = 0; DecodeColorValues(color_endpoint_mode, num_partitions, color_data_bits, color_values); - for (uint i = 0; i < num_partitions; i++) { - ComputeEndpoints(endpoints0[i], endpoints1[i], color_endpoint_mode[i], color_values, - colvals_index); - } + for (uint i = 0; i < num_partitions; i++) + ComputeEndpoints(endpoints0[i], endpoints1[i], color_endpoint_mode[i], color_values, colvals_index); } color_endpoint_data = local_buff; color_endpoint_data = bitfieldReverse(color_endpoint_data).wzyx; const uint clear_byte_start = (weight_bits >> 3) + 1; - const uint byte_insert = ExtractBits(color_endpoint_data, int(clear_byte_start - 1) * 8, 8) & - uint(((1 << (weight_bits % 8)) - 1)); + const uint byte_insert = ExtractBits(color_endpoint_data, (clear_byte_start - 1) << 3, 8) & uint(((1 << (weight_bits & 7)) - 1)); const uint vec_index = (clear_byte_start - 1) >> 2; - color_endpoint_data[vec_index] = bitfieldInsert(color_endpoint_data[vec_index], byte_insert, - int((clear_byte_start - 1) % 4) * 8, 8); - for (uint i = clear_byte_start; i < 16; ++i) { - const uint idx = i >> 2; - color_endpoint_data[idx] = bitfieldInsert(color_endpoint_data[idx], 0, int(i % 4) * 8, 8); - } + color_endpoint_data[vec_index] = bitfieldInsert(color_endpoint_data[vec_index], byte_insert, int((clear_byte_start - 1) & 3) << 3, 8); + for (uint i = clear_byte_start; i < 16; ++i) + color_endpoint_data[i >> 2] = bitfieldInsert(color_endpoint_data[i >> 2], 0, int(i & 3) << 3, 8); // Re-init vector variables for next decode phase result_index = 0; color_bitsread = 0; - result_limit_reached = false; // The limit for the Unquantize phase, avoids decoding more data than needed. - result_vector_max_index = size_params.x * size_params.y; - if (dual_plane) { - result_vector_max_index *= 2; - } + result_vector_max_index = (size_params.x * size_params.y) << uint(dual_plane); DecodeIntegerSequence(max_weight, GetNumWeightValues(size_params, dual_plane)); - UnquantizeTexelWeights(size_params, dual_plane); for (uint j = 0; j < block_dims.y; j++) { for (uint i = 0; i < block_dims.x; i++) { - uint local_partition = 0; - if (num_partitions > 1) { - local_partition = Select2DPartition(partition_index, i, j, num_partitions); - } - const uint local_cem = color_endpoint_mode[local_partition]; - const uvec4 weight_vec = GetUnquantizedWeightVector(j, i, size_params, plane_index, dual_plane); - - vec4 p; - if (IsHDRColorEndpointMode(local_cem)) { - const uvec4 C0 = endpoints0[local_partition] << 4u; - const uvec4 C1 = endpoints1[local_partition] << 4u; - const uvec4 C = (C0 * (uvec4(64) - weight_vec) + C1 * weight_vec + uvec4(32)) / 64u; - const uvec4 E = (C & uvec4(0xF800u)) >> 11u; - const uvec4 M = C & uvec4(0x7FFu); - const uvec4 Mt_lo = 3u * M; - const uvec4 Mt_mid = 4u * M - 512u; - const uvec4 Mt_hi = 5u * M - 2048u; - const uvec4 Mt = - mix(Mt_lo, mix(Mt_mid, Mt_hi, greaterThanEqual(M, uvec4(1536u))), - greaterThanEqual(M, uvec4(512u))); - const uvec4 Cf = (E << 10u) + (Mt >> 3u); - const uvec4 half_bits = mix(Cf, uvec4(0x7BFFu), greaterThanEqual(Cf, uvec4(0x7C00u))); - p = vec4(unpackHalf2x16(half_bits.x).x, unpackHalf2x16(half_bits.y).x, - unpackHalf2x16(half_bits.z).x, unpackHalf2x16(half_bits.w).x); - - if (local_cem == 14u) { - const uint a0 = ReplicateByteTo16(uvec4(endpoints0[local_partition].x)).x; - const uint a1 = ReplicateByteTo16(uvec4(endpoints1[local_partition].x)).x; - const uint Ca = (a0 * (64u - weight_vec.x) + a1 * weight_vec.x + 32u) / 64u; - p.x = float(Ca) / 65535.0f; - } - } else { - const uvec4 C0 = ReplicateByteTo16(endpoints0[local_partition]); - const uvec4 C1 = ReplicateByteTo16(endpoints1[local_partition]); - const vec4 Cf = - vec4((C0 * (uvec4(64) - weight_vec) + C1 * weight_vec + uvec4(32)) / 64); - p = Cf / 65535.0f; - } - - imageStore(dest_image, coord + ivec3(i, j, 0), clamp(p, 0.0f, 1.0f).gbar); + const uint local_partition = Select2DPartition(partition_index, uvec2(i, j), num_partitions) & (0 - uint(num_partitions > 1)); + const uvec4 C0 = ReplicateByteTo16(endpoints0[local_partition]); + const uvec4 C1 = ReplicateByteTo16(endpoints1[local_partition]); + const uvec4 weight_vec = GetUnquantizedWeightVector(uvec2(i, j), size_params, plane_index, dual_plane); + const vec4 Cf = vec4((C0 * (uvec4(64) - weight_vec) + C1 * weight_vec + 32) >> 6); + const vec4 p = (Cf / 65535.0f); + imageStore(dest_image, coord + ivec3(i, j, 0), p.gbar); } } } uint SwizzleOffset(uvec2 pos) { - return ((pos.x & 32u) << 3u) | - ((pos.y & 6u) << 5u) | - ((pos.x & 16u) << 1u) | - ((pos.y & 1u) << 4u) | - (pos.x & 15u); + return ((pos.x & 32u) << 3u) + | ((pos.y & 6u) << 5u) + | ((pos.x & 16u) << 1u) + | ((pos.y & 1u) << 4u) + | (pos.x & 15u); } void main() { @@ -1418,6 +981,5 @@ void main() { if (any(greaterThanEqual(coord, imageSize(dest_image)))) { return; } - local_buff = astc_data[offset / 16]; - DecompressBlock(coord); + DecompressBlock(astc_data[offset >> 4], coord); } diff --git a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp index 2a3446ca66..9cb0f2aca2 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_2d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_2d.comp @@ -51,7 +51,7 @@ layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u1 layout(binding = BINDING_OUTPUT_IMAGE) uniform writeonly uimage2DArray output_image; -layout(local_size_x = 32, local_size_y = 32, local_size_z = 1) in; +layout(local_size_x = 32, local_size_y = 8, local_size_z = 1) in; const uint GOB_SIZE_X = 64; const uint GOB_SIZE_Y = 8; @@ -65,19 +65,11 @@ const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT + GOB_SIZE_Z_SHI const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1, GOB_SIZE_Y - 1); -uint SwizzleTable(uint pos) { - const uint t[8] = uint[]( - 0x12100200, 0x13110301, 0x16140604, 0x17150705, - 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d - ); - const uint i = pos >> 4; - const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; - return (h << 4) | (pos & 0xf); -} - uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + return ((pos.x & 32u) << 3u) | ((pos.y & 6u) << 5u) | + ((pos.x & 16u) << 1u) | ((pos.y & 1u) << 4u) | + (pos.x & 15u); } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp index 34e4f93307..dc23b32492 100644 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d.comp +++ b/src/video_core/host_shaders/block_linear_unswizzle_3d.comp @@ -53,7 +53,7 @@ layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u1 layout(binding = BINDING_OUTPUT_IMAGE) uniform writeonly uimage3D output_image; -layout(local_size_x = 16, local_size_y = 8, local_size_z = 8) in; +layout(local_size_x = 16, local_size_y = 8, local_size_z = 2) in; const uint GOB_SIZE_X = 64; const uint GOB_SIZE_Y = 8; @@ -67,19 +67,11 @@ const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT + GOB_SIZE_Z_SHI const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1, GOB_SIZE_Y - 1); -uint SwizzleTable(uint pos) { - const uint t[8] = uint[]( - 0x12100200, 0x13110301, 0x16140604, 0x17150705, - 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d - ); - const uint i = pos >> 4; - const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; - return (h << 4) | (pos & 0xf); -} - uint SwizzleOffset(uvec2 pos) { pos = pos & SWIZZLE_MASK; - return SwizzleTable(pos.y * 64 + pos.x); + return ((pos.x & 32u) << 3u) | ((pos.y & 6u) << 5u) | + ((pos.x & 16u) << 1u) | ((pos.y & 1u) << 4u) | + (pos.x & 15u); } uvec4 ReadTexel(uint offset) { diff --git a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp b/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp deleted file mode 100644 index e084837f25..0000000000 --- a/src/video_core/host_shaders/block_linear_unswizzle_3d_bcn.comp +++ /dev/null @@ -1,164 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project -// SPDX-License-Identifier: GPL-3.0-or-later - -#version 430 - -#ifdef VULKAN - #extension GL_EXT_shader_16bit_storage : require - #extension GL_EXT_shader_8bit_storage : require - #define HAS_EXTENDED_TYPES 1 - #define BEGIN_PUSH_CONSTANTS layout(push_constant) uniform PushConstants { - #define END_PUSH_CONSTANTS }; - #define UNIFORM(n) - #define BINDING_INPUT_BUFFER 0 - #define BINDING_OUTPUT_BUFFER 1 -#else - #extension GL_NV_gpu_shader5 : enable - #ifdef GL_NV_gpu_shader5 - #define HAS_EXTENDED_TYPES 1 - #else - #define HAS_EXTENDED_TYPES 0 - #endif - #define BEGIN_PUSH_CONSTANTS - #define END_PUSH_CONSTANTS - #define UNIFORM(n) layout(location = n) uniform - #define BINDING_INPUT_BUFFER 1 - #define BINDING_OUTPUT_BUFFER 0 -#endif - -// --- Push Constants / Uniforms --- -#ifdef VULKAN -layout(push_constant) uniform PushConstants { - uvec3 blocks_dim; // Offset 0 - uint bytes_per_block_log2; // Offset 12 - - uvec3 origin; // Offset 16 - uint slice_size; // Offset 28 - - uint block_size; // Offset 32 - uint x_shift; // Offset 36 - uint block_height; // Offset 40 - uint block_height_mask; // Offset 44 - - uint block_depth; // Offset 48 - uint block_depth_mask; // Offset 52 - int _pad; // Offset 56 - - ivec3 destination; // Offset 60 -} pc; -#else -BEGIN_PUSH_CONSTANTS - UNIFORM(0) uvec3 origin; - UNIFORM(1) ivec3 destination; - UNIFORM(2) uint bytes_per_block_log2; - UNIFORM(3) uint slice_size; - UNIFORM(4) uint block_size; - UNIFORM(5) uint x_shift; - UNIFORM(6) uint block_height; - UNIFORM(7) uint block_height_mask; - UNIFORM(8) uint block_depth; - UNIFORM(9) uint block_depth_mask; - UNIFORM(10) uvec3 blocks_dim; -END_PUSH_CONSTANTS -#define pc // Map pc prefix to nothing for OpenGL compatibility -#endif - -// --- Buffers --- -#if HAS_EXTENDED_TYPES -layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU8 { uint8_t u8data[]; }; -layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU16 { uint16_t u16data[]; }; -#endif -layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU32 { uint u32data[]; }; -layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU64 { uvec2 u64data[]; }; -layout(binding = BINDING_INPUT_BUFFER, std430) buffer InputBufferU128 { uvec4 u128data[]; }; - -layout(binding = BINDING_OUTPUT_BUFFER, std430) writeonly buffer OutputBuffer { - uint out_u32[]; -}; - -// --- Constants --- -layout(local_size_x = 8, local_size_y = 8, local_size_z = 4) in; - -const uint GOB_SIZE_X = 64; -const uint GOB_SIZE_Y = 8; -const uint GOB_SIZE_Z = 1; -const uint GOB_SIZE = GOB_SIZE_X * GOB_SIZE_Y * GOB_SIZE_Z; - -const uint GOB_SIZE_X_SHIFT = 6; -const uint GOB_SIZE_Y_SHIFT = 3; -const uint GOB_SIZE_Z_SHIFT = 0; -const uint GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT + GOB_SIZE_Z_SHIFT; -const uvec2 SWIZZLE_MASK = uvec2(GOB_SIZE_X - 1u, GOB_SIZE_Y - 1u); - -uint SwizzleTable(uint pos) { - const uint t[8] = uint[]( - 0x12100200, 0x13110301, 0x16140604, 0x17150705, - 0x1a180a08, 0x1b190b09, 0x1e1c0e0c, 0x1f1d0f0d - ); - const uint i = pos >> 4; - const uint h = (t[i / 4] >> ((i % 4) * 8)) & 0xff; - return (h << 4) | (pos & 0xf); -} - -// --- Helpers --- -uint SwizzleOffset(uvec2 pos) { - pos &= SWIZZLE_MASK; - return SwizzleTable(pos.y * 64u + pos.x); -} - -uvec4 ReadTexel(uint offset) { - uint bpl2 = pc.bytes_per_block_log2; - switch (bpl2) { -#if HAS_EXTENDED_TYPES - case 0u: return uvec4(u8data[offset], 0u, 0u, 0u); - case 1u: return uvec4(u16data[offset / 2u], 0u, 0u, 0u); -#else - case 0u: return uvec4(bitfieldExtract(u32data[offset / 4u], int((offset * 8u) & 24u), 8), 0u, 0u, 0u); - case 1u: return uvec4(bitfieldExtract(u32data[offset / 4u], int((offset * 8u) & 16u), 16), 0u, 0u, 0u); -#endif - case 2u: return uvec4(u32data[offset / 4u], 0u, 0u, 0u); - case 3u: return uvec4(u64data[offset / 8u], 0u, 0u); - case 4u: return u128data[offset / 16u]; - } - return uvec4(0u); -} - -void main() { - uvec3 block_coord = gl_GlobalInvocationID; - if (any(greaterThanEqual(block_coord, pc.blocks_dim))) { - return; - } - - uint bytes_per_block = 1u << pc.bytes_per_block_log2; - // Origin is in pixels, divide by 4 for block-space (e.g. BCn formats) - uvec3 pos; - pos.x = (block_coord.x + (pc.origin.x >> 2u)) * bytes_per_block; - pos.y = block_coord.y + (pc.origin.y >> 2u); - pos.z = block_coord.z + pc.origin.z; - - uint swizzle = SwizzleOffset(pos.xy); - uint block_y = pos.y >> GOB_SIZE_Y_SHIFT; - uint offset = 0u; - // Apply block-linear offsets - offset += (pos.z >> pc.block_depth) * pc.slice_size; - offset += (pos.z & pc.block_depth_mask) << (GOB_SIZE_SHIFT + pc.block_height); - offset += (block_y >> pc.block_height) * pc.block_size; - offset += (block_y & pc.block_height_mask) << GOB_SIZE_SHIFT; - offset += (pos.x >> GOB_SIZE_X_SHIFT) << pc.x_shift; - offset += swizzle; - - uvec4 texel = ReadTexel(offset); - - // Calculate linear output index - uint block_index = block_coord.x + - (block_coord.y * pc.blocks_dim.x) + - (block_coord.z * pc.blocks_dim.x * pc.blocks_dim.y); - uint out_idx = block_index * (bytes_per_block >> 2u); - - out_u32[out_idx] = texel.x; - out_u32[out_idx + 1u] = texel.y; - if (pc.bytes_per_block_log2 == 4u) { - out_u32[out_idx + 2u] = texel.z; - out_u32[out_idx + 3u] = texel.w; - } -} \ No newline at end of file diff --git a/src/video_core/host_shaders/pitch_unswizzle.comp b/src/video_core/host_shaders/pitch_unswizzle.comp index 7d23f594c7..dba874b414 100644 --- a/src/video_core/host_shaders/pitch_unswizzle.comp +++ b/src/video_core/host_shaders/pitch_unswizzle.comp @@ -33,7 +33,7 @@ BEGIN_PUSH_CONSTANTS UNIFORM(0) uvec2 origin; UNIFORM(1) ivec2 destination; -UNIFORM(2) uint bytes_per_block; +UNIFORM(2) uint bytes_per_block_log2; UNIFORM(3) uint pitch; END_PUSH_CONSTANTS @@ -47,26 +47,26 @@ layout(binding = BINDING_INPUT_BUFFER, std430) readonly buffer InputBufferU128 { layout(binding = BINDING_OUTPUT_IMAGE) writeonly uniform uimage2D output_image; -layout(local_size_x = 32, local_size_y = 32, local_size_z = 1) in; +layout(local_size_x = 32, local_size_y = 8, local_size_z = 1) in; uvec4 ReadTexel(uint offset) { - switch (bytes_per_block) { + switch (bytes_per_block_log2) { #if HAS_EXTENDED_TYPES - case 1: + case 0: return uvec4(u8data[offset], 0, 0, 0); - case 2: + case 1: return uvec4(u16data[offset / 2], 0, 0, 0); #else - case 1: + case 0: return uvec4(bitfieldExtract(u32data[offset / 4], int((offset * 8) & 24), 8), 0, 0, 0); - case 2: + case 1: return uvec4(bitfieldExtract(u32data[offset / 4], int((offset * 8) & 16), 16), 0, 0, 0); #endif - case 4: + case 2: return uvec4(u32data[offset / 4], 0, 0, 0); - case 8: + case 3: return uvec4(u64data[offset / 8], 0, 0); - case 16: + case 4: return u128data[offset / 16]; } return uvec4(0); @@ -76,7 +76,7 @@ void main() { uvec2 pos = gl_GlobalInvocationID.xy + origin; uint offset = 0; - offset += pos.x * bytes_per_block; + offset += pos.x << bytes_per_block_log2; offset += pos.y * pitch; const uvec4 texel = ReadTexel(offset); diff --git a/src/video_core/host_shaders/vulkan_color_clear.frag b/src/video_core/host_shaders/vulkan_color_clear.frag index 617bf01e17..d17ce12b1a 100644 --- a/src/video_core/host_shaders/vulkan_color_clear.frag +++ b/src/video_core/host_shaders/vulkan_color_clear.frag @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -7,8 +10,10 @@ layout (push_constant) uniform PushConstants { vec4 clear_color; }; -layout(location = 0) out vec4 color; +layout(location = 0) out vec4 colors[8]; void main() { - color = clear_color; + for (int index = 0; index < 8; ++index) { + colors[index] = clear_color; + } } diff --git a/src/video_core/host_shaders/vulkan_indirect_quads.comp b/src/video_core/host_shaders/vulkan_indirect_quads.comp new file mode 100644 index 0000000000..01bc2f961f --- /dev/null +++ b/src/video_core/host_shaders/vulkan_indirect_quads.comp @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#version 460 core + +layout (local_size_x = 32) in; + +layout (std430, set = 0, binding = 0) readonly buffer InputBuffer { + uint input_commands[]; +}; + +layout (std430, set = 0, binding = 1) writeonly buffer OutputBuffer { + uint output_commands[]; +}; + +layout (push_constant) uniform PushConstants { + uint num_draws; + uint src_stride; +}; + +void main() { + uint draw = gl_GlobalInvocationID.x; + if (draw >= num_draws) { + return; + } + uint src = draw * src_stride; + uint dst = draw * 5; + output_commands[dst + 0] = input_commands[src + 0] / 4 * 6; + output_commands[dst + 1] = input_commands[src + 1]; + output_commands[dst + 2] = input_commands[src + 2] / 4 * 6; + output_commands[dst + 3] = input_commands[src + 3]; + output_commands[dst + 4] = input_commands[src + 4]; +} diff --git a/src/video_core/memory_manager.cpp b/src/video_core/memory_manager.cpp index d5838a387d..b9b136f6b8 100644 --- a/src/video_core/memory_manager.cpp +++ b/src/video_core/memory_manager.cpp @@ -46,8 +46,8 @@ MemoryManager::MemoryManager(Core::System& system_, MaxwellDeviceMemoryManager& page_table_mask = page_table_size - 1; big_page_table_mask = big_page_table_size - 1; - big_page_table_dev.ResizeAndClear(big_page_table_size); big_entries.resize(big_page_table_size / 32, 0); + big_page_table_dev.resize(big_page_table_size); big_page_continuous.resize(big_page_table_size / continuous_bits, 0); entries.resize(page_table_size / 32, 0); } @@ -143,7 +143,7 @@ GPUVAddr MemoryManager::BigPageTableOp(GPUVAddr gpu_addr, [[maybe_unused]] DAddr const DAddr current_dev_addr = dev_addr + offset; const auto index = PageEntryIndex(current_gpu_addr, true); const u32 sub_value = static_cast(current_dev_addr >> cpu_page_bits); - big_page_table_dev.Set(index, sub_value); + big_page_table_dev[index] = sub_value; const bool is_continuous = ([&] { uintptr_t base_ptr{ reinterpret_cast(memory.GetPointer(current_dev_addr))}; diff --git a/src/video_core/memory_manager.h b/src/video_core/memory_manager.h index 08978567d2..f9fddd177a 100644 --- a/src/video_core/memory_manager.h +++ b/src/video_core/memory_manager.h @@ -17,7 +17,7 @@ #include "common/multi_level_page_table.h" #include "common/range_map.h" #include "common/scratch_buffer.h" -#include "common/sparse_large_vector.h" +#include "common/virtual_buffer.h" #include "video_core/invalidation_accumulator.h" #include "video_core/cache_types.h" #include "video_core/host1x/gpu_device_memory_manager.h" @@ -214,7 +214,7 @@ private: Common::MultiLevelPageTable page_table; Common::RangeMap kind_map; - Common::SparseLargeVector big_page_table_dev; + Common::VirtualBuffer big_page_table_dev; std::vector big_page_continuous; boost::container::small_vector, 32> page_stash{}; diff --git a/src/video_core/rasterizer_interface.h b/src/video_core/rasterizer_interface.h index 481efbf53b..d0e55da8b1 100644 --- a/src/video_core/rasterizer_interface.h +++ b/src/video_core/rasterizer_interface.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -75,7 +78,7 @@ public: virtual void SyncOperation(std::function&& func) = 0; /// Signal a GPU based syncpoint as a fence - virtual void SignalSyncPoint(u32 value) = 0; + virtual void SignalSyncPoint(u32 value, u32 count) = 0; /// Signal a GPU based reference as point virtual void SignalReference() = 0; diff --git a/src/video_core/renderer_null/null_rasterizer.cpp b/src/video_core/renderer_null/null_rasterizer.cpp index a5cda0f389..d2fb89119c 100644 --- a/src/video_core/renderer_null/null_rasterizer.cpp +++ b/src/video_core/renderer_null/null_rasterizer.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -69,10 +72,12 @@ void RasterizerNull::SignalFence(std::function&& func) { void RasterizerNull::SyncOperation(std::function&& func) { func(); } -void RasterizerNull::SignalSyncPoint(u32 value) { +void RasterizerNull::SignalSyncPoint(u32 value, u32 count) { auto& syncpoint_manager = m_gpu.Host1x().GetSyncpointManager(); - syncpoint_manager.IncrementGuest(value); - syncpoint_manager.IncrementHost(value); + for (u32 i = 0; i < count; ++i) { + syncpoint_manager.IncrementGuest(value); + syncpoint_manager.IncrementHost(value); + } } void RasterizerNull::SignalReference() {} void RasterizerNull::ReleaseFences(bool) {} diff --git a/src/video_core/renderer_null/null_rasterizer.h b/src/video_core/renderer_null/null_rasterizer.h index c7f5849c75..c98da41f15 100644 --- a/src/video_core/renderer_null/null_rasterizer.h +++ b/src/video_core/renderer_null/null_rasterizer.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -61,7 +64,7 @@ public: void ModifyGPUMemory(size_t as_id, GPUVAddr addr, u64 size) override; void SignalFence(std::function&& func) override; void SyncOperation(std::function&& func) override; - void SignalSyncPoint(u32 value) override; + void SignalSyncPoint(u32 value, u32 count) override; void SignalReference() override; void ReleaseFences(bool force) override; void FlushAndInvalidateRegion( diff --git a/src/video_core/renderer_null/renderer_null.cpp b/src/video_core/renderer_null/renderer_null.cpp index e6147d66cc..434fc172e3 100644 --- a/src/video_core/renderer_null/renderer_null.cpp +++ b/src/video_core/renderer_null/renderer_null.cpp @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -20,7 +23,7 @@ void RendererNull::Composite(std::span framebuff } m_gpu.RendererFrameEndNotify(); - render_window.OnFrameDisplayed(); + render_window.OnFrameDisplayed(1); } std::vector RendererNull::GetAppletCaptureBuffer() { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp index 1d77d28c46..35d47856eb 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.cpp +++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp @@ -615,8 +615,8 @@ void RasterizerOpenGL::SyncOperation(std::function&& func) { fence_manager.SyncOperation(std::move(func)); } -void RasterizerOpenGL::SignalSyncPoint(u32 value) { - fence_manager.SignalSyncPoint(value); +void RasterizerOpenGL::SignalSyncPoint(u32 value, u32 count) { + fence_manager.SignalSyncPoint(value, count); } void RasterizerOpenGL::SignalReference() { diff --git a/src/video_core/renderer_opengl/gl_rasterizer.h b/src/video_core/renderer_opengl/gl_rasterizer.h index 6eae51ff7d..338c5eb877 100644 --- a/src/video_core/renderer_opengl/gl_rasterizer.h +++ b/src/video_core/renderer_opengl/gl_rasterizer.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: 2015 Citra Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -106,7 +109,7 @@ public: void ModifyGPUMemory(size_t as_id, GPUVAddr addr, u64 size) override; void SignalFence(std::function&& func) override; void SyncOperation(std::function&& func) override; - void SignalSyncPoint(u32 value) override; + void SignalSyncPoint(u32 value, u32 count) override; void SignalReference() override; void ReleaseFences(bool force = true) override; void FlushAndInvalidateRegion( diff --git a/src/video_core/renderer_opengl/gl_texture_cache.cpp b/src/video_core/renderer_opengl/gl_texture_cache.cpp index 680cb2179f..2589e42109 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.cpp +++ b/src/video_core/renderer_opengl/gl_texture_cache.cpp @@ -235,9 +235,7 @@ void ApplySwizzle(GLuint handle, PixelFormat format, std::array swizzles, - u32 z_start, u32 z_count) { + std::span swizzles) { switch (image.info.type) { case ImageType::e2D: if (IsPixelFormatASTC(image.info.format)) { @@ -709,11 +689,6 @@ Image::Image(TextureCacheRuntime& runtime_, const VideoCommon::ImageInfo& info_, gl_internal_format = is_srgb ? GL_SRGB8_ALPHA8 : GL_RGBA8; gl_format = GL_RGBA; gl_type = GL_UNSIGNED_INT_8_8_8_8_REV; - - if (IsPixelFormatASTC(info.format) && IsAstcRecompressionEnabled()) { - gl_internal_format = SelectAstcFormat(info.format, is_srgb); - gl_format = GL_NONE; - } } else { const auto& tuple = MaxwellToGL::GetFormatTuple(info.format); gl_internal_format = tuple.internal_format; @@ -1110,10 +1085,6 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI if (True(image.flags & ImageFlagBits::Converted)) { const bool is_srgb = IsPixelFormatSRGB(info.format); internal_format = is_srgb ? GL_SRGB8_ALPHA8 : GL_RGBA8; - - if (IsPixelFormatASTC(info.format) && IsAstcRecompressionEnabled()) { - internal_format = SelectAstcFormat(info.format, is_srgb); - } } else { internal_format = MaxwellToGL::GetFormatTuple(format).internal_format; } diff --git a/src/video_core/renderer_opengl/gl_texture_cache.h b/src/video_core/renderer_opengl/gl_texture_cache.h index 8a830fa0c9..19a1b79cf8 100644 --- a/src/video_core/renderer_opengl/gl_texture_cache.h +++ b/src/video_core/renderer_opengl/gl_texture_cache.h @@ -75,9 +75,7 @@ public: void Finish(); - void FlushDeferredClear() {} - - StagingBufferMap UploadStagingBuffer(size_t size, bool deferred = false); + StagingBufferMap UploadStagingBuffer(size_t size); StagingBufferMap DownloadStagingBuffer(size_t size, bool deferred = false); @@ -121,8 +119,7 @@ public: Tegra::Engines::Fermi2D::Operation operation); void AccelerateImageUpload(Image& image, const StagingBufferMap& map, - std::span swizzles, - u32 z_start, u32 z_count); + std::span swizzles); void InsertUploadMemoryBarrier(); @@ -229,8 +226,6 @@ public: bool ScaleDown(bool ignore = false); - u64 allocation_tick; - private: void CopyBufferToImage(const VideoCommon::BufferImageCopy& copy, size_t buffer_offset); @@ -367,7 +362,6 @@ private: }; struct TextureCacheParams { - static constexpr bool ENABLE_VALIDATION = true; static constexpr bool FRAMEBUFFER_BLITS = true; static constexpr bool HAS_EMULATED_COPIES = true; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; diff --git a/src/video_core/renderer_opengl/renderer_opengl.cpp b/src/video_core/renderer_opengl/renderer_opengl.cpp index f9ec32e652..caf15294b2 100644 --- a/src/video_core/renderer_opengl/renderer_opengl.cpp +++ b/src/video_core/renderer_opengl/renderer_opengl.cpp @@ -151,7 +151,7 @@ void RendererOpenGL::Composite(std::span framebu rasterizer.TickFrame(); context->SwapBuffers(); - render_window.OnFrameDisplayed(); + render_window.OnFrameDisplayed(1); } void RendererOpenGL::AddTelemetryFields() { diff --git a/src/video_core/renderer_opengl/util_shaders.cpp b/src/video_core/renderer_opengl/util_shaders.cpp index 7ced8c77b4..ef3fc05f2a 100644 --- a/src/video_core/renderer_opengl/util_shaders.cpp +++ b/src/video_core/renderer_opengl/util_shaders.cpp @@ -4,6 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include #include @@ -116,7 +117,7 @@ void UtilShaders::ASTCDecode(Image& image, const StagingBufferMap& map, void UtilShaders::BlockLinearUpload2D(Image& image, const StagingBufferMap& map, std::span swizzles) { - static constexpr Extent3D WORKGROUP_SIZE{32, 32, 1}; + static constexpr Extent3D WORKGROUP_SIZE{32, 8, 1}; static constexpr GLuint BINDING_INPUT_BUFFER = 0; static constexpr GLuint BINDING_OUTPUT_IMAGE = 0; @@ -151,7 +152,7 @@ void UtilShaders::BlockLinearUpload2D(Image& image, const StagingBufferMap& map, void UtilShaders::BlockLinearUpload3D(Image& image, const StagingBufferMap& map, std::span swizzles) { - static constexpr Extent3D WORKGROUP_SIZE{16, 8, 8}; + static constexpr Extent3D WORKGROUP_SIZE{16, 8, 2}; static constexpr GLuint BINDING_INPUT_BUFFER = 0; static constexpr GLuint BINDING_OUTPUT_IMAGE = 0; @@ -189,12 +190,12 @@ void UtilShaders::BlockLinearUpload3D(Image& image, const StagingBufferMap& map, void UtilShaders::PitchUpload(Image& image, const StagingBufferMap& map, std::span swizzles) { - static constexpr Extent3D WORKGROUP_SIZE{32, 32, 1}; + static constexpr Extent3D WORKGROUP_SIZE{32, 8, 1}; static constexpr GLuint BINDING_INPUT_BUFFER = 0; static constexpr GLuint BINDING_OUTPUT_IMAGE = 0; static constexpr GLuint LOC_ORIGIN = 0; static constexpr GLuint LOC_DESTINATION = 1; - static constexpr GLuint LOC_BYTES_PER_BLOCK = 2; + static constexpr GLuint LOC_BYTES_PER_BLOCK_LOG2 = 2; static constexpr GLuint LOC_PITCH = 3; const u32 bytes_per_block = BytesPerBlock(image.info.format); @@ -208,7 +209,7 @@ void UtilShaders::PitchUpload(Image& image, const StagingBufferMap& map, glFlushMappedNamedBufferRange(map.buffer, map.offset, image.guest_size_bytes); glUniform2ui(LOC_ORIGIN, 0, 0); glUniform2i(LOC_DESTINATION, 0, 0); - glUniform1ui(LOC_BYTES_PER_BLOCK, bytes_per_block); + glUniform1ui(LOC_BYTES_PER_BLOCK_LOG2, static_cast(std::countr_zero(bytes_per_block))); glUniform1ui(LOC_PITCH, pitch); glBindImageTexture(BINDING_OUTPUT_IMAGE, image.StorageHandle(), 0, GL_FALSE, 0, GL_WRITE_ONLY, format); diff --git a/src/video_core/renderer_vulkan/blit_image.cpp b/src/video_core/renderer_vulkan/blit_image.cpp index ddd52e9426..7c91f53f32 100644 --- a/src/video_core/renderer_vulkan/blit_image.cpp +++ b/src/video_core/renderer_vulkan/blit_image.cpp @@ -279,6 +279,44 @@ constexpr VkPipelineDepthStencilStateCreateInfo PIPELINE_DEPTH_ONLY_STATE_CREATE .maxDepthBounds = 0.0f, }; +constexpr VkPipelineDepthStencilStateCreateInfo PIPELINE_DEPTH_STENCIL_STATE_DISABLED_CREATE_INFO{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .depthTestEnable = VK_FALSE, + .depthWriteEnable = VK_FALSE, + .depthCompareOp = VK_COMPARE_OP_ALWAYS, + .depthBoundsTestEnable = VK_FALSE, + .stencilTestEnable = VK_FALSE, + .front = VkStencilOpState{}, + .back = VkStencilOpState{}, + .minDepthBounds = 0.0f, + .maxDepthBounds = 0.0f, +}; + +using BlendAttachments = std::array; + +[[nodiscard]] VkPipelineColorBlendStateCreateInfo MakeColorBlendState( + const BlendAttachments& attachments, const RenderingFormats& formats) { + return VkPipelineColorBlendStateCreateInfo{ + .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .logicOpEnable = VK_FALSE, + .logicOp = VK_LOGIC_OP_CLEAR, + .attachmentCount = formats.num_colors, + .pAttachments = attachments.data(), + .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, + }; +} + +[[nodiscard]] VkPipelineMultisampleStateCreateInfo MakeMultisampleState( + VkSampleCountFlagBits samples) { + VkPipelineMultisampleStateCreateInfo multisample_ci = PIPELINE_MULTISAMPLE_STATE_CREATE_INFO; + multisample_ci.rasterizationSamples = samples; + return multisample_ci; +} + template inline constexpr VkSamplerCreateInfo SAMPLER_CREATE_INFO{ .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, @@ -452,45 +490,19 @@ VkExtent2D GetConversionExtent(const ImageView& src_image_view) { }; } -void TransitionImageLayout(vk::CommandBuffer& cmdbuf, VkImage image, VkImageLayout target_layout, - VkImageLayout source_layout = VK_IMAGE_LAYOUT_GENERAL) { - constexpr VkFlags flags{VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - const VkImageMemoryBarrier barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = flags, - .dstAccessMask = flags, - .oldLayout = source_layout, - .newLayout = target_layout, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = image, - .subresourceRange{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - }, - }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, barrier); -} - void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) { const VkImage image = image_view.ImageHandle(); const VkImageSubresourceRange subresource_range = SubresourceRangeFromView(image_view); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([image, subresource_range](vk::CommandBuffer cmdbuf) { - const VkImageMemoryBarrier barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -498,20 +510,27 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) .image = image, .subresourceRange = subresource_range, }; - cmdbuf.PipelineBarrier( - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_TRANSFER_BIT | - VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, - barrier); + cmdbuf.PipelineBarrier(barrier); }); } +void DiscardCoveredAttachments(Scheduler& scheduler, const Framebuffer* framebuffer, + const Region2D& region, bool writes_stencil) { + const VkExtent2D area = framebuffer->RenderArea(); + if ((std::min)(region.start.x, region.end.x) > 0 || + (std::min)(region.start.y, region.end.y) > 0 || + (std::max)(region.start.x, region.end.x) < static_cast(area.width) || + (std::max)(region.start.y, region.end.y) < static_cast(area.height) || + framebuffer->Attachments().layers != 1) { + return; + } + u32 attachments = 1u | Scheduler::DEPTH_ATTACHMENT_BIT; + if (writes_stencil) { + attachments |= Scheduler::STENCIL_ATTACHMENT_BIT; + } + scheduler.OverrideLoadOps(framebuffer, attachments, VK_ATTACHMENT_LOAD_OP_DONT_CARE, {}); +} + [[nodiscard]] VkSampleCountFlagBits SampleCountFlag(u32 num_samples) { switch (num_samples) { case 2: @@ -562,25 +581,6 @@ void RecordShaderReadBarrier(Scheduler& scheduler, const ImageView& image_view) }, }); } - -void BeginRenderPass(vk::CommandBuffer& cmdbuf, const Framebuffer* framebuffer) { - const VkRenderPass render_pass = framebuffer->RenderPass(); - const VkFramebuffer framebuffer_handle = framebuffer->Handle(); - const VkExtent2D render_area = framebuffer->RenderArea(); - const VkRenderPassBeginInfo renderpass_bi{ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, - .pNext = nullptr, - .renderPass = render_pass, - .framebuffer = framebuffer_handle, - .renderArea{ - .offset{}, - .extent = render_area, - }, - .clearValueCount = 0, - .pClearValues = nullptr, - }; - cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); -} } // Anonymous namespace BlitImageHelper::BlitImageHelper(const Device& device_, Scheduler& scheduler_, @@ -664,8 +664,10 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, const ImageV Tegra::Engines::Fermi2D::Operation operation) { const bool is_linear = filter == Tegra::Engines::Fermi2D::Filter::Bilinear; const BlitImagePipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .operation = operation, + .samples = dst_framebuffer->Samples(), + .target = 0, }; VkSampler sampler = *nearest_sampler; if (is_linear) { @@ -681,16 +683,19 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView const Region2D& dst_region, const Region2D& src_region, const Extent3D& src_size) { const BlitImagePipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .operation = Tegra::Engines::Fermi2D::Operation::SrcCopy, + .samples = dst_framebuffer->Samples(), + .target = 0, }; const VkPipelineLayout layout = *one_texture_pipeline_layout; const VkPipeline pipeline = FindOrEmplaceColorPipeline(key); - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([this, dst_framebuffer, src_image_view, src_image, src_sampler, dst_region, - src_region, src_size, pipeline, layout](vk::CommandBuffer cmdbuf) { - TransitionImageLayout(cmdbuf, src_image, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL); - BeginRenderPass(cmdbuf, dst_framebuffer); + if (scheduler.IsRenderPassImage(src_image)) { + scheduler.RequestOutsideRenderPassOperationContext(); + } + scheduler.RequestRenderpass(dst_framebuffer); + scheduler.Record([this, src_image_view, src_sampler, dst_region, src_region, src_size, + pipeline, layout](vk::CommandBuffer cmdbuf) { const VkDescriptorSet descriptor_set = one_texture_descriptor_allocator.Commit(); UpdateOneTextureDescriptorSet(device, descriptor_set, src_sampler, src_image_view); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); @@ -698,8 +703,8 @@ void BlitImageHelper::BlitColor(const Framebuffer* dst_framebuffer, VkImageView nullptr); BindBlitState(cmdbuf, layout, dst_region, src_region, src_size); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); }); + scheduler.InvalidateState(); } void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer, @@ -713,6 +718,7 @@ void BlitImageHelper::BlitImpl(const Framebuffer* dst_framebuffer, } RecordShaderReadBarrier(scheduler, src_image_view); + DiscardCoveredAttachments(scheduler, dst_framebuffer, dst_region, blit_stencil); scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record([this, dst_region, src_region, pipeline, layout, sampler, src_view, src_stencil_view, blit_stencil](vk::CommandBuffer cmdbuf) { @@ -738,7 +744,7 @@ void BlitImageHelper::BlitColorMSAA(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region) { const BlitMSAAPipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .samples = dst_framebuffer->Samples(), }; BlitImpl(dst_framebuffer, src_image_view, dst_region, src_region, @@ -752,7 +758,7 @@ void BlitImageHelper::BlitDepthStencilMSAA(const Framebuffer* dst_framebuffer, const bool blit_stencil = dst_framebuffer->HasAspectStencilBit() && device.IsExtShaderStencilExportSupported(); const BlitMSAAPipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .samples = dst_framebuffer->Samples(), }; VkImageView src_stencil_view = VK_NULL_HANDLE; @@ -767,7 +773,7 @@ void BlitImageHelper::BlitDepthStencilMSAA(const Framebuffer* dst_framebuffer, void BlitImageHelper::BlitDepth(const Framebuffer* dst_framebuffer, ImageView& src_image_view, const Region2D& dst_region, const Region2D& src_region) { BlitImpl(dst_framebuffer, src_image_view, dst_region, src_region, - FindOrEmplaceBlitDepthPipeline(dst_framebuffer->RenderPass()), *nearest_sampler, + FindOrEmplaceBlitDepthPipeline(dst_framebuffer->Formats()), *nearest_sampler, src_image_view.DepthView(), VK_NULL_HANDLE, false); } @@ -781,7 +787,7 @@ void BlitImageHelper::ResolveDepthStencil(const Framebuffer* dst_framebuffer, src_stencil_view = src_image_view.StencilView(); } BlitImpl(dst_framebuffer, src_image_view, dst_region, src_region, - FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer->RenderPass(), + FindOrEmplaceResolveDepthStencilPipeline(dst_framebuffer->Formats(), resolve_stencil), *nearest_sampler, src_image_view.DepthView(), src_stencil_view, resolve_stencil); } @@ -795,8 +801,10 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, ASSERT(operation == Tegra::Engines::Fermi2D::Operation::SrcCopy); const bool blit_stencil = device.IsExtShaderStencilExportSupported(); const BlitImagePipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .operation = operation, + .samples = dst_framebuffer->Samples(), + .target = 0, }; VkPipeline pipeline{}; VkImageView src_stencil_view = VK_NULL_HANDLE; @@ -804,7 +812,7 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, pipeline = FindOrEmplaceDepthStencilPipeline(key); src_stencil_view = src_image_view.StencilView(); } else { - pipeline = FindOrEmplaceBlitDepthPipeline(key.renderpass); + pipeline = FindOrEmplaceBlitDepthPipeline(key.formats); } BlitImpl(dst_framebuffer, src_image_view, dst_region, src_region, pipeline, *nearest_sampler, src_image_view.DepthView(), src_stencil_view, blit_stencil); @@ -812,26 +820,22 @@ void BlitImageHelper::BlitDepthStencil(const Framebuffer* dst_framebuffer, void BlitImageHelper::ConvertD32ToR32(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertDepthToColorPipeline(convert_d32_to_r32_pipeline, dst_framebuffer->RenderPass()); - Convert(*convert_d32_to_r32_pipeline, dst_framebuffer, src_image_view); + Convert(*convert_depth_to_float_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertR32ToD32(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertColorToDepthPipeline(convert_r32_to_d32_pipeline, dst_framebuffer->RenderPass()); - Convert(*convert_r32_to_d32_pipeline, dst_framebuffer, src_image_view); + Convert(*convert_float_to_depth_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD16ToR16(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertDepthToColorPipeline(convert_d16_to_r16_pipeline, dst_framebuffer->RenderPass()); - Convert(*convert_d16_to_r16_pipeline, dst_framebuffer, src_image_view); + Convert(*convert_depth_to_float_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertR16ToD16(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertColorToDepthPipeline(convert_r16_to_d16_pipeline, dst_framebuffer->RenderPass()); - Convert(*convert_r16_to_d16_pipeline, dst_framebuffer, src_image_view); + Convert(*convert_float_to_depth_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertABGR8ToD24S8(const Framebuffer* dst_framebuffer, @@ -841,45 +845,37 @@ void BlitImageHelper::ConvertABGR8ToD24S8(const Framebuffer* dst_framebuffer, LOG_WARNING(Render_Vulkan, "ConvertABGR8ToD24S8 requires shader_stencil_export, skipping"); return; } - ConvertPipelineDepthTargetEx(convert_abgr8_to_d24s8_pipeline, dst_framebuffer->RenderPass(), - convert_abgr8_to_d24s8_frag); - Convert(*convert_abgr8_to_d24s8_pipeline, dst_framebuffer, src_image_view); + Convert(*convert_abgr8_to_d24s8_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertABGR8ToD32F(const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { - ConvertPipelineDepthTargetEx(convert_abgr8_to_d32f_pipeline, dst_framebuffer->RenderPass(), - convert_abgr8_to_d32f_frag); - Convert(*convert_abgr8_to_d32f_pipeline, dst_framebuffer, src_image_view); + Convert(*convert_abgr8_to_d32f_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD32FToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_d32f_to_abgr8_pipeline, dst_framebuffer->RenderPass(), - convert_d32f_to_abgr8_frag); - ConvertDepthStencil(*convert_d32f_to_abgr8_pipeline, dst_framebuffer, src_image_view); + ConvertDepthStencil(*convert_d32f_to_abgr8_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertD24S8ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_d24s8_to_abgr8_pipeline, dst_framebuffer->RenderPass(), - convert_d24s8_to_abgr8_frag); - ConvertDepthStencil(*convert_d24s8_to_abgr8_pipeline, dst_framebuffer, src_image_view); + ConvertDepthStencil(*convert_d24s8_to_abgr8_frag, dst_framebuffer, src_image_view); } void BlitImageHelper::ConvertS8D24ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view) { - ConvertPipelineColorTargetEx(convert_s8d24_to_abgr8_pipeline, dst_framebuffer->RenderPass(), - convert_s8d24_to_abgr8_frag); - ConvertDepthStencil(*convert_s8d24_to_abgr8_pipeline, dst_framebuffer, src_image_view); + ConvertDepthStencil(*convert_s8d24_to_abgr8_frag, dst_framebuffer, src_image_view); } -void BlitImageHelper::ClearColor(const Framebuffer* dst_framebuffer, u8 color_mask, +void BlitImageHelper::ClearColor(const Framebuffer* dst_framebuffer, u32 target, u8 color_mask, const std::array& clear_color, const Region2D& dst_region) { const BlitImagePipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .operation = Tegra::Engines::Fermi2D::Operation::BlendPremult, + .samples = dst_framebuffer->Samples(), + .target = target, }; const VkPipeline pipeline = FindOrEmplaceClearColorPipeline(key); const VkPipelineLayout layout = *clear_color_pipeline_layout; @@ -902,11 +898,12 @@ void BlitImageHelper::ClearDepthStencil(const Framebuffer* dst_framebuffer, bool f32 clear_depth, u8 stencil_mask, u32 stencil_ref, u32 stencil_compare_mask, const Region2D& dst_region) { const BlitDepthStencilPipelineKey key{ - .renderpass = dst_framebuffer->RenderPass(), + .formats = dst_framebuffer->Formats(), .depth_clear = depth_clear, .stencil_mask = stencil_mask, .stencil_compare_mask = stencil_compare_mask, .stencil_ref = stencil_ref, + .samples = dst_framebuffer->Samples(), }; const VkPipeline pipeline = FindOrEmplaceClearStencilPipeline(key); const VkPipelineLayout layout = *clear_color_pipeline_layout; @@ -922,7 +919,7 @@ void BlitImageHelper::ClearDepthStencil(const Framebuffer* dst_framebuffer, bool scheduler.InvalidateState(); } -void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, +void BlitImageHelper::CopyMSAAImpl(const RenderingFormats& formats, VkPipeline pipeline, VkPipelineLayout layout, VkImage dst_image, VkFormat dst_vk_format, VkImage src_image, VkFormat src_vk_format, s32 scale_x, s32 scale_y, @@ -958,17 +955,15 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, .offset = dst_offset, .extent = dst_extent, }; - vk::Framebuffer framebuffer = device.GetLogical().CreateFramebuffer(VkFramebufferCreateInfo{ - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .renderPass = renderpass, - .attachmentCount = 1, - .pAttachments = dst_view.address(), - .width = static_cast(dst_offset.x) + dst_extent.width, - .height = static_cast(dst_offset.y) + dst_extent.height, - .layers = 1, - }); + const VkImageView dst_view_handle = *dst_view; + RenderingAttachments attachments = + MakeRenderingAttachments(formats, std::span(&dst_view_handle, formats.num_colors), + dst_view_handle, render_area, 1); + attachments.colors[0].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachments.depth.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + if (copy_stencil) { + attachments.stencil.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + } const MSAACopyPushConstants push_constants{ .dst_offset = {dst_offset.x, dst_offset.y}, .src_offset = {copy.src_offset.x, copy.src_offset.y}, @@ -979,10 +974,10 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, src_stencil_handle = *src_stencil_view; } scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([this, pipeline, layout, sampler, renderpass, - framebuffer_handle = *framebuffer, src_view_handle = *src_view, - src_stencil_handle, src = src_image, dst = dst_image, render_area, - aspect_info, push_constants](vk::CommandBuffer cmdbuf) { + scheduler.Record([this, pipeline, layout, sampler, attachments, + src_view_handle = *src_view, src_stencil_handle, src = src_image, + dst = dst_image, render_area, aspect_info, + push_constants](vk::CommandBuffer cmdbuf) { const VkImageSubresourceRange barrier_range{ .aspectMask = aspect_info.barrier_aspect, .baseMipLevel = 0, @@ -991,10 +986,12 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, .layerCount = VK_REMAINING_ARRAY_LAYERS, }; const std::array pre_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = aspect_info.pre_src_access, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = aspect_info.pre_dst_stages, .dstAccessMask = aspect_info.pre_src_dst_access, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, @@ -1003,10 +1000,12 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, .image = src, .subresourceRange = barrier_range, }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = aspect_info.pre_src_access, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = aspect_info.pre_dst_stages, .dstAccessMask = aspect_info.pre_dst_dst_access, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, @@ -1016,18 +1015,8 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, .subresourceRange = barrier_range, }, }; - cmdbuf.PipelineBarrier(aspect_info.pre_src_stages, aspect_info.pre_dst_stages, 0, - nullptr, nullptr, pre_barriers); - const VkRenderPassBeginInfo renderpass_bi{ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, - .pNext = nullptr, - .renderPass = renderpass, - .framebuffer = framebuffer_handle, - .renderArea = render_area, - .clearValueCount = 0, - .pClearValues = nullptr, - }; - cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); + cmdbuf.PipelineBarrier(0, {}, {}, pre_barriers); + BeginRendering(cmdbuf, attachments); VkDescriptorSet descriptor_set = VK_NULL_HANDLE; if (src_stencil_handle != VK_NULL_HANDLE) { descriptor_set = two_textures_descriptor_allocator.Commit(); @@ -1052,12 +1041,14 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, cmdbuf.SetScissor(0, render_area); cmdbuf.PushConstants(layout, VK_SHADER_STAGE_FRAGMENT_BIT, push_constants); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); - const VkImageMemoryBarrier post_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + cmdbuf.EndRendering(); + const VkImageMemoryBarrier2 post_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = aspect_info.post_src_stages, .srcAccessMask = aspect_info.post_src_access, - .dstAccessMask = aspect_info.post_dst_access, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1065,21 +1056,18 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, .image = dst, .subresourceRange = barrier_range, }; - cmdbuf.PipelineBarrier(aspect_info.post_src_stages, aspect_info.post_dst_stages, 0, - post_barrier); + cmdbuf.PipelineBarrier(post_barrier); }); msaa_copy_resources.push_back(MSAACopyResources{ .tick = scheduler.CurrentTick(), .src_view = std::move(src_view), .dst_view = std::move(dst_view), - .framebuffer = std::move(framebuffer), }); if (copy_stencil) { msaa_copy_resources.push_back(MSAACopyResources{ .tick = scheduler.CurrentTick(), .src_view = std::move(src_stencil_view), .dst_view = vk::ImageView{}, - .framebuffer = vk::Framebuffer{}, }); } } @@ -1087,10 +1075,9 @@ void BlitImageHelper::CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, scheduler.InvalidateState(); } -void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_image, - VideoCore::Surface::PixelFormat dst_format, VkImage src_image, - VideoCore::Surface::PixelFormat src_format, u32 num_samples, - std::span copies, +void BlitImageHelper::CopyMSAA(VkImage dst_image, VideoCore::Surface::PixelFormat dst_format, + VkImage src_image, VideoCore::Surface::PixelFormat src_format, + u32 num_samples, std::span copies, bool msaa_to_non_msaa) { const auto [samples_x, samples_y] = VideoCommon::SamplesLog2(static_cast(num_samples)); const s32 scale_x = 1 << samples_x; @@ -1099,14 +1086,10 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i if (msaa_to_non_msaa) { samples = VK_SAMPLE_COUNT_1_BIT; } - RenderPassKey renderpass_key{}; - renderpass_key.color_formats.fill(VideoCore::Surface::PixelFormat::Invalid); - renderpass_key.color_formats[0] = dst_format; - renderpass_key.depth_format = VideoCore::Surface::PixelFormat::Invalid; - renderpass_key.samples = samples; - const VkRenderPass renderpass = render_pass_cache.Get(renderpass_key); + const RenderingFormats formats = MakeRenderingFormats( + device, std::span(&dst_format, 1), VideoCore::Surface::PixelFormat::Invalid); const MSAACopyPipelineKey key{ - .renderpass = renderpass, + .formats = formats, .samples = samples, .msaa_to_non_msaa = msaa_to_non_msaa, .format_class = FormatClass(dst_format), @@ -1115,39 +1098,38 @@ void BlitImageHelper::CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_i .src_view_aspect = VK_IMAGE_ASPECT_COLOR_BIT, .attachment_aspect = VK_IMAGE_ASPECT_COLOR_BIT, .barrier_aspect = VK_IMAGE_ASPECT_COLOR_BIT, - .pre_src_access = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .pre_src_dst_access = VK_ACCESS_SHADER_READ_BIT, + .pre_src_dst_access = VK_ACCESS_2_SHADER_READ_BIT, .pre_dst_dst_access = - VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .pre_src_stages = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, .pre_dst_stages = - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, - .post_src_access = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .post_dst_access = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT, - .post_src_stages = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, - .post_dst_stages = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, + VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + .post_src_access = VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, + .post_src_stages = VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, }; const VkFormat src_vk_format = MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, src_format).format; const VkFormat dst_vk_format = MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, dst_format).format; - CopyMSAAImpl(renderpass, FindOrEmplaceMSAACopyPipeline(key), *msaa_copy_pipeline_layout, + CopyMSAAImpl(formats, FindOrEmplaceMSAACopyPipeline(key), *msaa_copy_pipeline_layout, dst_image, dst_vk_format, src_image, src_vk_format, scale_x, scale_y, copies, aspect_info, false); } -void BlitImageHelper::Convert(VkPipeline pipeline, const Framebuffer* dst_framebuffer, +void BlitImageHelper::Convert(VkShaderModule module, const Framebuffer* dst_framebuffer, const ImageView& src_image_view) { const VkPipelineLayout layout = *one_texture_pipeline_layout; + const VkPipeline pipeline = + FindOrEmplaceConvertPipeline(dst_framebuffer->Formats(), module, layout); const VkImageView src_view = src_image_view.Handle(Shader::TextureType::Color2D); const VkSampler sampler = *nearest_sampler; const VkExtent2D extent = GetConversionExtent(src_image_view); + const Region2D region{ + .start = {0, 0}, + .end = {static_cast(extent.width), static_cast(extent.height)}, + }; RecordShaderReadBarrier(scheduler, src_image_view); + DiscardCoveredAttachments(scheduler, dst_framebuffer, region, false); scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record([pipeline, layout, sampler, src_view, extent, this](vk::CommandBuffer cmdbuf) { const VkOffset2D offset{ @@ -1184,15 +1166,23 @@ void BlitImageHelper::Convert(VkPipeline pipeline, const Framebuffer* dst_frameb scheduler.InvalidateState(); } -void BlitImageHelper::ConvertDepthStencil(VkPipeline pipeline, const Framebuffer* dst_framebuffer, +void BlitImageHelper::ConvertDepthStencil(VkShaderModule module, + const Framebuffer* dst_framebuffer, ImageView& src_image_view) { const VkPipelineLayout layout = *two_textures_pipeline_layout; + const VkPipeline pipeline = + FindOrEmplaceConvertPipeline(dst_framebuffer->Formats(), module, layout); const VkImageView src_depth_view = src_image_view.DepthView(); const VkImageView src_stencil_view = src_image_view.StencilView(); const VkSampler sampler = *nearest_sampler; const VkExtent2D extent = GetConversionExtent(src_image_view); + const Region2D region{ + .start = {0, 0}, + .end = {static_cast(extent.width), static_cast(extent.height)}, + }; RecordShaderReadBarrier(scheduler, src_image_view); + DiscardCoveredAttachments(scheduler, dst_framebuffer, region, false); scheduler.RequestRenderpass(dst_framebuffer); scheduler.Record([pipeline, layout, sampler, src_depth_view, src_stencil_view, extent, this](vk::CommandBuffer cmdbuf) { @@ -1238,32 +1228,17 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe blit_color_keys.push_back(key); const std::array stages = MakeStages(*full_screen_vert, *blit_color_to_color_frag); - const VkPipelineColorBlendAttachmentState blend_attachment{ - .blendEnable = VK_FALSE, - .srcColorBlendFactor = VK_BLEND_FACTOR_ZERO, - .dstColorBlendFactor = VK_BLEND_FACTOR_ZERO, - .colorBlendOp = VK_BLEND_OP_ADD, - .srcAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, - .dstAlphaBlendFactor = VK_BLEND_FACTOR_ZERO, - .alphaBlendOp = VK_BLEND_OP_ADD, - .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | - VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, - }; + BlendAttachments blend_attachments{}; + blend_attachments[key.target] = PIPELINE_COLOR_BLEND_ATTACHMENT_STATE; // TODO: programmable blending - const VkPipelineColorBlendStateCreateInfo color_blend_create_info{ - .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .logicOpEnable = VK_FALSE, - .logicOp = VK_LOGIC_OP_CLEAR, - .attachmentCount = 1, - .pAttachments = &blend_attachment, - .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, - }; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info = + MakeColorBlendState(blend_attachments, key.formats); + const VkPipelineMultisampleStateCreateInfo multisample_ci = MakeMultisampleState(key.samples); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); blit_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1272,16 +1247,15 @@ VkPipeline BlitImageHelper::FindOrEmplaceColorPipeline(const BlitImagePipelineKe .pTessellationState = nullptr, .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, - .pDepthStencilState = nullptr, + .pMultisampleState = &multisample_ci, + .pDepthStencilState = &PIPELINE_DEPTH_STENCIL_STATE_DISABLED_CREATE_INFO, .pColorBlendState = &color_blend_create_info, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *one_texture_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *blit_color_pipelines.back(); } @@ -1292,10 +1266,12 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip } blit_depth_stencil_keys.push_back(key); const std::array stages = MakeStages(*full_screen_vert, *blit_depth_stencil_frag); + const VkPipelineMultisampleStateCreateInfo multisample_ci = MakeMultisampleState(key.samples); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); blit_depth_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1304,16 +1280,15 @@ VkPipeline BlitImageHelper::FindOrEmplaceDepthStencilPipeline(const BlitImagePip .pTessellationState = nullptr, .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pMultisampleState = &multisample_ci, .pDepthStencilState = &PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *two_textures_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *blit_depth_stencil_pipelines.back(); } @@ -1335,20 +1310,16 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, }; - const VkPipelineColorBlendStateCreateInfo color_blend_state_generic_create_info{ - .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .logicOpEnable = VK_FALSE, - .logicOp = VK_LOGIC_OP_CLEAR, - .attachmentCount = 1, - .pAttachments = &color_blend_attachment_state, - .blendConstants = {0.0f, 0.0f, 0.0f, 0.0f}, - }; + BlendAttachments blend_attachments{}; + blend_attachments[key.target] = color_blend_attachment_state; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info = + MakeColorBlendState(blend_attachments, key.formats); + const VkPipelineMultisampleStateCreateInfo multisample_ci = MakeMultisampleState(key.samples); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); clear_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1357,16 +1328,15 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearColorPipeline(const BlitImagePipel .pTessellationState = nullptr, .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, - .pDepthStencilState = &PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO, - .pColorBlendState = &color_blend_state_generic_create_info, + .pMultisampleState = &multisample_ci, + .pDepthStencilState = &PIPELINE_DEPTH_STENCIL_STATE_DISABLED_CREATE_INFO, + .pColorBlendState = &color_blend_create_info, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *clear_color_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *clear_color_pipelines.back(); } @@ -1401,10 +1371,15 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( .minDepthBounds = 0.0f, .maxDepthBounds = 0.0f, }; + const BlendAttachments blend_attachments{}; + const VkPipelineColorBlendStateCreateInfo color_blend_create_info = + MakeColorBlendState(blend_attachments, key.formats); + const VkPipelineMultisampleStateCreateInfo multisample_ci = MakeMultisampleState(key.samples); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); clear_stencil_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1413,16 +1388,15 @@ VkPipeline BlitImageHelper::FindOrEmplaceClearStencilPipeline( .pTessellationState = nullptr, .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, + .pMultisampleState = &multisample_ci, .pDepthStencilState = &depth_stencil_ci, - .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, + .pColorBlendState = &color_blend_create_info, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *clear_color_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *clear_stencil_pipelines.back(); } @@ -1445,9 +1419,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip .alphaToOneEnable = VK_FALSE, }; const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); blit_msaa_color_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1461,11 +1436,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPip .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *one_texture_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *blit_msaa_color_pipelines.back(); } @@ -1493,9 +1467,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthStencilMSAAPipeline( .alphaToOneEnable = VK_FALSE, }; const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1510,7 +1485,6 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthStencilMSAAPipeline( .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = blit_stencil ? *two_textures_pipeline_layout : *one_texture_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, @@ -1518,17 +1492,18 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthStencilMSAAPipeline( return *pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthPipeline(VkRenderPass renderpass) { - const auto it = std::ranges::find(blit_depth_keys, renderpass); +VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthPipeline(const RenderingFormats& formats) { + const auto it = std::ranges::find(blit_depth_keys, formats); if (it != blit_depth_keys.end()) { return *blit_depth_pipelines[std::distance(blit_depth_keys.begin(), it)]; } - blit_depth_keys.push_back(renderpass); + blit_depth_keys.push_back(formats); const std::array stages = MakeStages(*full_screen_vert, *blit_depth_frag); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = formats.CreateInfo(); blit_depth_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1542,7 +1517,6 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthPipeline(VkRenderPass renderpa .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *one_texture_pipeline_layout, - .renderPass = renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, @@ -1550,22 +1524,23 @@ VkPipeline BlitImageHelper::FindOrEmplaceBlitDepthPipeline(VkRenderPass renderpa return *blit_depth_pipelines.back(); } -VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass, - bool resolve_stencil) { +VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline( + const RenderingFormats& formats, bool resolve_stencil) { auto& keys = resolve_stencil ? resolve_depth_stencil_keys : resolve_depth_keys; auto& pipelines = resolve_stencil ? resolve_depth_stencil_pipelines : resolve_depth_pipelines; - const auto it = std::ranges::find(keys, renderpass); + const auto it = std::ranges::find(keys, formats); if (it != keys.end()) { return *pipelines[std::distance(keys.begin(), it)]; } - keys.push_back(renderpass); + keys.push_back(formats); const std::array stages = MakeStages(*full_screen_vert, resolve_stencil ? *blit_depth_stencil_msaa_frag : *blit_depth_msaa_frag); const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = formats.CreateInfo(); pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1580,18 +1555,16 @@ VkPipeline BlitImageHelper::FindOrEmplaceResolveDepthStencilPipeline(VkRenderPas .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = resolve_stencil ? *two_textures_pipeline_layout : *one_texture_pipeline_layout, - .renderPass = renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *pipelines.back(); } -void BlitImageHelper::CopyMSAADepth(RenderPassCache& render_pass_cache, VkImage dst_image, - VideoCore::Surface::PixelFormat dst_format, VkImage src_image, - VideoCore::Surface::PixelFormat src_format, u32 num_samples, - std::span copies, +void BlitImageHelper::CopyMSAADepth(VkImage dst_image, VideoCore::Surface::PixelFormat dst_format, + VkImage src_image, VideoCore::Surface::PixelFormat src_format, + u32 num_samples, std::span copies, bool copy_stencil, bool msaa_to_non_msaa) { const auto [samples_x, samples_y] = VideoCommon::SamplesLog2(static_cast(num_samples)); const s32 scale_x = 1 << samples_x; @@ -1600,13 +1573,9 @@ void BlitImageHelper::CopyMSAADepth(RenderPassCache& render_pass_cache, VkImage if (msaa_to_non_msaa) { samples = VK_SAMPLE_COUNT_1_BIT; } - RenderPassKey renderpass_key{}; - renderpass_key.color_formats.fill(VideoCore::Surface::PixelFormat::Invalid); - renderpass_key.depth_format = dst_format; - renderpass_key.samples = samples; - const VkRenderPass renderpass = render_pass_cache.Get(renderpass_key); + const RenderingFormats formats = MakeRenderingFormats(device, {}, dst_format); const MSAACopyPipelineKey key{ - .renderpass = renderpass, + .formats = formats, .samples = samples, .msaa_to_non_msaa = msaa_to_non_msaa, .format_class = MSAACopyFormatClass::Float, @@ -1624,27 +1593,19 @@ void BlitImageHelper::CopyMSAADepth(RenderPassCache& render_pass_cache, VkImage .src_view_aspect = VK_IMAGE_ASPECT_DEPTH_BIT, .attachment_aspect = attachment_aspect, .barrier_aspect = attachment_aspect, - .pre_src_access = - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - .pre_src_dst_access = VK_ACCESS_SHADER_READ_BIT, - .pre_dst_dst_access = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, - .pre_src_stages = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_TRANSFER_BIT, + .pre_src_dst_access = VK_ACCESS_2_SHADER_READ_BIT, + .pre_dst_dst_access = VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, .pre_dst_stages = - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT, - .post_src_access = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, - .post_dst_access = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT, - .post_src_stages = VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT, - .post_dst_stages = vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, + VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT, + .post_src_access = VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, + .post_src_stages = VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT, }; const VkFormat src_vk_format = MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, src_format).format; const VkFormat dst_vk_format = MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, dst_format).format; - CopyMSAAImpl(renderpass, FindOrEmplaceMSAACopyDepthPipeline(key, copy_stencil), layout, + CopyMSAAImpl(formats, FindOrEmplaceMSAACopyDepthPipeline(key, copy_stencil), layout, dst_image, dst_vk_format, src_image, src_vk_format, scale_x, scale_y, copies, aspect_info, copy_stencil); } @@ -1677,9 +1638,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline .alphaToOneEnable = VK_FALSE, }; const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); msaa_copy_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1693,11 +1655,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyPipeline(const MSAACopyPipeline .pColorBlendState = &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = *msaa_copy_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache())); + })); return *msaa_copy_pipelines.back(); } @@ -1755,9 +1716,10 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyDepthPipeline(const MSAACopyPip }; const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = key.formats.CreateInfo(); pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1772,7 +1734,6 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyDepthPipeline(const MSAACopyPip .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, .layout = copy_stencil ? *msaa_copy_depth_stencil_pipeline_layout : *msaa_copy_pipeline_layout, - .renderPass = key.renderpass, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, @@ -1780,25 +1741,32 @@ VkPipeline BlitImageHelper::FindOrEmplaceMSAACopyDepthPipeline(const MSAACopyPip return *pipelines.back(); } -void BlitImageHelper::ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) { - ConvertPipeline(pipeline, renderpass, false); -} - -void BlitImageHelper::ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass) { - ConvertPipeline(pipeline, renderpass, true); -} - -void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass, - vk::ShaderModule& module, bool single_texture, - bool is_target_depth) { - if (pipeline) { - return; +VkPipeline BlitImageHelper::FindOrEmplaceConvertPipeline(const RenderingFormats& formats, + VkShaderModule module, + VkPipelineLayout layout) { + const ConvertPipelineKey key{ + .formats = formats, + .module = module, + }; + const auto it = std::ranges::find(convert_keys, key); + if (it != convert_keys.end()) { + return *convert_pipelines[std::distance(convert_keys.begin(), it)]; } - const std::array stages = MakeStages(*full_screen_vert, *module); - const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); - pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ + convert_keys.push_back(key); + const VkPipelineDepthStencilStateCreateInfo* depth_stencil_ci = nullptr; + const VkPipelineColorBlendStateCreateInfo* color_blend_ci = + &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO; + if (formats.num_colors == 0) { + depth_stencil_ci = &PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO; + color_blend_ci = &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO; + } + const std::array stages = MakeStages(*full_screen_vert, module); + const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = + GetPipelineInputAssemblyStateCreateInfo(device); + const VkPipelineRenderingCreateInfo rendering_ci = formats.CreateInfo(); + convert_pipelines.push_back(device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -1808,59 +1776,15 @@ void BlitImageHelper::ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass ren .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, - .pDepthStencilState = is_target_depth ? &PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO : nullptr, - .pColorBlendState = is_target_depth ? &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO - : &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, + .pDepthStencilState = depth_stencil_ci, + .pColorBlendState = color_blend_ci, .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, - .layout = single_texture ? *one_texture_pipeline_layout : *two_textures_pipeline_layout, - .renderPass = renderpass, + .layout = layout, .subpass = 0, .basePipelineHandle = VK_NULL_HANDLE, .basePipelineIndex = 0, - }, device.StaticPipelineCache()); -} - -void BlitImageHelper::ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, - vk::ShaderModule& module) { - ConvertPipelineEx(pipeline, renderpass, module, false, false); -} - -void BlitImageHelper::ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, - vk::ShaderModule& module) { - ConvertPipelineEx(pipeline, renderpass, module, true, true); -} - -void BlitImageHelper::ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, - bool is_target_depth) { - if (pipeline) { - return; - } - VkShaderModule frag_shader = - is_target_depth ? *convert_float_to_depth_frag : *convert_depth_to_float_frag; - const std::array stages = MakeStages(*full_screen_vert, frag_shader); - const VkPipelineInputAssemblyStateCreateInfo input_assembly_ci = GetPipelineInputAssemblyStateCreateInfo(device); - pipeline = device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ - .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .stageCount = static_cast(stages.size()), - .pStages = stages.data(), - .pVertexInputState = &PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, - .pInputAssemblyState = &input_assembly_ci, - .pTessellationState = nullptr, - .pViewportState = &PIPELINE_VIEWPORT_STATE_CREATE_INFO, - .pRasterizationState = &PIPELINE_RASTERIZATION_STATE_CREATE_INFO, - .pMultisampleState = &PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, - .pDepthStencilState = is_target_depth ? &PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO : nullptr, - .pColorBlendState = is_target_depth ? &PIPELINE_COLOR_BLEND_STATE_EMPTY_CREATE_INFO - : &PIPELINE_COLOR_BLEND_STATE_GENERIC_CREATE_INFO, - .pDynamicState = &PIPELINE_DYNAMIC_STATE_CREATE_INFO, - .layout = *one_texture_pipeline_layout, - .renderPass = renderpass, - .subpass = 0, - .basePipelineHandle = VK_NULL_HANDLE, - .basePipelineIndex = 0, - }, device.StaticPipelineCache()); + })); + return *convert_pipelines.back(); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/blit_image.h b/src/video_core/renderer_vulkan/blit_image.h index 62e6743a23..bc1db246ff 100644 --- a/src/video_core/renderer_vulkan/blit_image.h +++ b/src/video_core/renderer_vulkan/blit_image.h @@ -11,6 +11,7 @@ #include "video_core/engines/fermi_2d.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" +#include "video_core/renderer_vulkan/vk_render_pass_cache.h" #include "video_core/surface.h" #include "video_core/texture_cache/types.h" #include "video_core/vulkan_common/vulkan_wrapper.h" @@ -24,25 +25,34 @@ using VideoCommon::Region2D; class Device; class Framebuffer; class ImageView; -class RenderPassCache; class StateTracker; class Scheduler; struct BlitImagePipelineKey { constexpr auto operator<=>(const BlitImagePipelineKey&) const noexcept = default; - VkRenderPass renderpass; + RenderingFormats formats; Tegra::Engines::Fermi2D::Operation operation; + VkSampleCountFlagBits samples; + u32 target; }; struct BlitDepthStencilPipelineKey { constexpr auto operator<=>(const BlitDepthStencilPipelineKey&) const noexcept = default; - VkRenderPass renderpass; + RenderingFormats formats; bool depth_clear; u8 stencil_mask; u32 stencil_compare_mask; u32 stencil_ref; + VkSampleCountFlagBits samples; +}; + +struct ConvertPipelineKey { + constexpr auto operator<=>(const ConvertPipelineKey&) const noexcept = default; + + RenderingFormats formats; + VkShaderModule module; }; enum class MSAACopyFormatClass : u32 { @@ -54,7 +64,7 @@ enum class MSAACopyFormatClass : u32 { struct MSAACopyPipelineKey { constexpr auto operator<=>(const MSAACopyPipelineKey&) const noexcept = default; - VkRenderPass renderpass; + RenderingFormats formats; VkSampleCountFlagBits samples; bool msaa_to_non_msaa; MSAACopyFormatClass format_class; @@ -63,7 +73,7 @@ struct MSAACopyPipelineKey { struct BlitMSAAPipelineKey { constexpr auto operator<=>(const BlitMSAAPipelineKey&) const noexcept = default; - VkRenderPass renderpass; + RenderingFormats formats; VkSampleCountFlagBits samples; }; @@ -117,38 +127,32 @@ public: void ConvertS8D24ToABGR8(const Framebuffer* dst_framebuffer, ImageView& src_image_view); - void ClearColor(const Framebuffer* dst_framebuffer, u8 color_mask, + void ClearColor(const Framebuffer* dst_framebuffer, u32 target, u8 color_mask, const std::array& clear_color, const Region2D& dst_region); void ClearDepthStencil(const Framebuffer* dst_framebuffer, bool depth_clear, f32 clear_depth, u8 stencil_mask, u32 stencil_ref, u32 stencil_compare_mask, const Region2D& dst_region); - void CopyMSAA(RenderPassCache& render_pass_cache, VkImage dst_image, - VideoCore::Surface::PixelFormat dst_format, VkImage src_image, + void CopyMSAA(VkImage dst_image, VideoCore::Surface::PixelFormat dst_format, VkImage src_image, VideoCore::Surface::PixelFormat src_format, u32 num_samples, std::span copies, bool msaa_to_non_msaa); - void CopyMSAADepth(RenderPassCache& render_pass_cache, VkImage dst_image, - VideoCore::Surface::PixelFormat dst_format, VkImage src_image, - VideoCore::Surface::PixelFormat src_format, u32 num_samples, - std::span copies, bool copy_stencil, - bool msaa_to_non_msaa); + void CopyMSAADepth(VkImage dst_image, VideoCore::Surface::PixelFormat dst_format, + VkImage src_image, VideoCore::Surface::PixelFormat src_format, + u32 num_samples, std::span copies, + bool copy_stencil, bool msaa_to_non_msaa); private: struct MSAACopyAspectInfo { VkImageAspectFlags src_view_aspect; VkImageAspectFlags attachment_aspect; VkImageAspectFlags barrier_aspect; - VkAccessFlags pre_src_access; - VkAccessFlags pre_src_dst_access; - VkAccessFlags pre_dst_dst_access; - VkPipelineStageFlags pre_src_stages; - VkPipelineStageFlags pre_dst_stages; - VkAccessFlags post_src_access; - VkAccessFlags post_dst_access; - VkPipelineStageFlags post_src_stages; - VkPipelineStageFlags post_dst_stages; + VkAccessFlags2 pre_src_dst_access; + VkAccessFlags2 pre_dst_dst_access; + VkPipelineStageFlags2 pre_dst_stages; + VkAccessFlags2 post_src_access; + VkPipelineStageFlags2 post_src_stages; }; void BlitImpl(const Framebuffer* dst_framebuffer, const ImageView& src_image_view, @@ -156,16 +160,16 @@ private: VkSampler sampler, VkImageView src_view, VkImageView src_stencil_view, bool blit_stencil); - void CopyMSAAImpl(VkRenderPass renderpass, VkPipeline pipeline, VkPipelineLayout layout, + void CopyMSAAImpl(const RenderingFormats& formats, VkPipeline pipeline, VkPipelineLayout layout, VkImage dst_image, VkFormat dst_vk_format, VkImage src_image, VkFormat src_vk_format, s32 scale_x, s32 scale_y, std::span copies, const MSAACopyAspectInfo& aspect_info, bool copy_stencil); - void Convert(VkPipeline pipeline, const Framebuffer* dst_framebuffer, + void Convert(VkShaderModule module, const Framebuffer* dst_framebuffer, const ImageView& src_image_view); - void ConvertDepthStencil(VkPipeline pipeline, const Framebuffer* dst_framebuffer, + void ConvertDepthStencil(VkShaderModule module, const Framebuffer* dst_framebuffer, ImageView& src_image_view); [[nodiscard]] VkPipeline FindOrEmplaceColorPipeline(const BlitImagePipelineKey& key); @@ -182,24 +186,12 @@ private: [[nodiscard]] VkPipeline FindOrEmplaceBlitColorMSAAPipeline(const BlitMSAAPipelineKey& key); [[nodiscard]] VkPipeline FindOrEmplaceBlitDepthStencilMSAAPipeline( const BlitMSAAPipelineKey& key, bool blit_stencil); - [[nodiscard]] VkPipeline FindOrEmplaceBlitDepthPipeline(VkRenderPass renderpass); - [[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline(VkRenderPass renderpass, - bool resolve_stencil); - - void ConvertPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass, bool is_target_depth); - - void ConvertDepthToColorPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass); - - void ConvertColorToDepthPipeline(vk::Pipeline& pipeline, VkRenderPass renderpass); - - void ConvertPipelineEx(vk::Pipeline& pipeline, VkRenderPass renderpass, - vk::ShaderModule& module, bool single_texture, bool is_target_depth); - - void ConvertPipelineColorTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, - vk::ShaderModule& module); - - void ConvertPipelineDepthTargetEx(vk::Pipeline& pipeline, VkRenderPass renderpass, - vk::ShaderModule& module); + [[nodiscard]] VkPipeline FindOrEmplaceBlitDepthPipeline(const RenderingFormats& formats); + [[nodiscard]] VkPipeline FindOrEmplaceResolveDepthStencilPipeline( + const RenderingFormats& formats, bool resolve_stencil); + [[nodiscard]] VkPipeline FindOrEmplaceConvertPipeline(const RenderingFormats& formats, + VkShaderModule module, + VkPipelineLayout layout); const Device& device; Scheduler& scheduler; @@ -260,32 +252,24 @@ private: std::vector msaa_copy_depth_stencil_pipelines; std::vector blit_msaa_color_keys; std::vector blit_msaa_color_pipelines; - std::vector blit_depth_keys; + std::vector blit_depth_keys; std::vector blit_depth_pipelines; std::vector blit_msaa_depth_keys; std::vector blit_msaa_depth_pipelines; std::vector blit_msaa_depth_stencil_keys; std::vector blit_msaa_depth_stencil_pipelines; - std::vector resolve_depth_keys; + std::vector resolve_depth_keys; std::vector resolve_depth_pipelines; - std::vector resolve_depth_stencil_keys; + std::vector resolve_depth_stencil_keys; std::vector resolve_depth_stencil_pipelines; struct MSAACopyResources { u64 tick; vk::ImageView src_view; vk::ImageView dst_view; - vk::Framebuffer framebuffer; }; std::deque msaa_copy_resources; - vk::Pipeline convert_d32_to_r32_pipeline; - vk::Pipeline convert_r32_to_d32_pipeline; - vk::Pipeline convert_d16_to_r16_pipeline; - vk::Pipeline convert_r16_to_d16_pipeline; - vk::Pipeline convert_abgr8_to_d24s8_pipeline; - vk::Pipeline convert_abgr8_to_d32f_pipeline; - vk::Pipeline convert_d32f_to_abgr8_pipeline; - vk::Pipeline convert_d24s8_to_abgr8_pipeline; - vk::Pipeline convert_s8d24_to_abgr8_pipeline; + std::vector convert_keys; + std::vector convert_pipelines; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp index 0ced890929..9a2dc76202 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.cpp @@ -88,6 +88,18 @@ bool ComputeAttachment0DualSourceBlend(const Maxwell& regs) { : uses_dual_source(regs.blend); } +bool WritesColorTarget(const Maxwell& regs, size_t index, bool static_masks) { + if (!static_masks || (index == 0 && regs.anti_alias_alpha_control.alpha_to_coverage != 0)) { + return true; + } + size_t mask_index = index; + if (regs.color_mask_common) { + mask_index = 0; + } + const auto& mask = regs.color_mask[mask_index]; + return mask.R || mask.G || mask.B || mask.A; +} + void RefreshXfbState(VideoCommon::TransformFeedbackState& state, const Maxwell& regs) { std::ranges::transform(regs.transform_feedback.controls, state.layouts.begin(), [](const auto& layout) { @@ -101,7 +113,8 @@ void RefreshXfbState(VideoCommon::TransformFeedbackState& state, const Maxwell& } } // Anonymous namespace -void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFeatures& features) { +void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFeatures& features, + u32 color_outputs) { const Maxwell& regs = maxwell3d.regs; const auto topology_ = maxwell3d.draw_manager.draw_state.topology; @@ -119,11 +132,14 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe xfb_enabled.Assign(regs.transform_feedback_enabled != 0); ndc_minus_one_to_one.Assign(regs.depth_mode == Maxwell::DepthMode::MinusOneToOne ? 1 : 0); polygon_mode.Assign(PackPolygonMode(regs.polygon_mode_front)); - tessellation_primitive.Assign(static_cast(regs.tessellation.params.domain_type.Value())); - tessellation_spacing.Assign(static_cast(regs.tessellation.params.spacing.Value())); - tessellation_clockwise.Assign(regs.tessellation.params.output_primitives.Value() == - Maxwell::Tessellation::OutputPrimitives::Triangles_CW); - patch_control_points_minus_one.Assign(regs.patch_vertices - 1); + if (topology_ == Maxwell::PrimitiveTopology::Patches) { + tessellation_primitive.Assign( + static_cast(regs.tessellation.params.domain_type.Value())); + tessellation_spacing.Assign(static_cast(regs.tessellation.params.spacing.Value())); + tessellation_clockwise.Assign(regs.tessellation.params.output_primitives.Value() == + Maxwell::Tessellation::OutputPrimitives::Triangles_CW); + patch_control_points_minus_one.Assign(regs.patch_vertices - 1); + } const bool can_collapse_topology_class = features.has_extended_dynamic_state && features.has_extended_dynamic_state_2; topology.Assign(can_collapse_topology_class @@ -138,8 +154,13 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe regs.alpha_test_enabled != 0 ? regs.alpha_test_func : Maxwell::ComparisonOp::Always_GL; alpha_test_func.Assign(PackComparisonOp(test_func)); early_z.Assign(regs.mandated_early_z != 0 ? 1 : 0); - depth_enabled.Assign(regs.zeta_enable != 0 ? 1 : 0); - depth_format.Assign(static_cast(regs.zeta.format)); + const bool drop_unused = features.has_dynamic_rendering_unused_attachments; + const bool depth_used = regs.depth_test_enable || regs.depth_bounds_enable || + regs.stencil_enable || extended_dynamic_state; + if (regs.zeta_enable != 0 && regs.zeta.Address() != 0 && (depth_used || !drop_unused)) { + depth_enabled.Assign(1); + depth_format.Assign(static_cast(regs.zeta.format)); + } y_negate.Assign(regs.window_origin.mode != Maxwell::WindowOrigin::Mode::UpperLeft ? 1 : 0); bool use_last_provoking_vertex = false; @@ -168,19 +189,39 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe smooth_lines.Assign(regs.line_anti_alias_enable != 0 ? 1 : 0); alpha_to_coverage_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_coverage != 0 ? 1 : 0); alpha_to_one_enabled.Assign(regs.anti_alias_alpha_control.alpha_to_one != 0 ? 1 : 0); - app_stage.Assign(maxwell3d.engine_state); - depth_bounds_min = static_cast(regs.depth_bounds[0]); - depth_bounds_max = static_cast(regs.depth_bounds[1]); - - line_stipple_factor = regs.line_stipple_params.factor; - line_stipple_pattern = regs.line_stipple_params.pattern; - - for (size_t i = 0; i < regs.rt.size(); ++i) { - color_formats[i] = static_cast(regs.rt[i].format); + line_stipple_factor = 0; + line_stipple_pattern = 0; + if (regs.line_stipple_enable || features.has_dynamic_state3_line_stipple_enable) { + line_stipple_factor = regs.line_stipple_params.factor; + line_stipple_pattern = regs.line_stipple_params.pattern; + } + + const bool static_masks = + !features.has_extended_dynamic_state_3_blend && !features.has_color_write_enable; + const auto previous_formats = color_formats; + color_formats.fill(static_cast(Tegra::RenderTargetFormat::NONE)); + const size_t num_targets = (std::min)(size_t{regs.rt_control.count}, color_formats.size()); + size_t num_formats = 0; + for (size_t index = 0; index < num_targets; ++index) { + const bool used = ((color_outputs >> index) & 1) != 0 && + WritesColorTarget(regs, index, static_masks); + if (regs.rt[index].Address() != 0 && (used || !drop_unused)) { + num_formats = index + 1; + } + } + for (size_t index = 0; index < num_formats; ++index) { + color_formats[index] = static_cast(regs.rt[index].format); + } + alpha_test_ref = 0; + if (regs.alpha_test_enabled != 0) { + alpha_test_ref = std::bit_cast(regs.alpha_test_ref); + } + point_size = 0; + if (topology_ == Maxwell::PrimitiveTopology::Points || + regs.IsShaderConfigEnabled(Maxwell::ShaderType::Geometry)) { + point_size = std::bit_cast(regs.point_size); } - alpha_test_ref = std::bit_cast(regs.alpha_test_ref); - point_size = std::bit_cast(regs.point_size); if (maxwell3d.dirty.flags[Dirty::VertexInput]) { if (features.has_dynamic_vertex_input) { @@ -241,11 +282,15 @@ void FixedPipelineState::Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFe dynamic_state.Refresh2(regs, topology_, extended_dynamic_state_2); } if (!extended_dynamic_state_3_blend) { - if (maxwell3d.dirty.flags[Dirty::Blending]) { + if (maxwell3d.dirty.flags[Dirty::Blending] || color_formats != previous_formats) { maxwell3d.dirty.flags[Dirty::Blending] = false; for (size_t index = 0; index < attachments.size(); ++index) { - attachments[index].Refresh(regs, index); auto& attachment = attachments[index]; + attachment.raw = 0; + if (color_formats[index] == static_cast(Tegra::RenderTargetFormat::NONE)) { + continue; + } + attachment.Refresh(regs, index); if (color_write_enable_dynamic && attachment.mask_r == 0 && attachment.mask_g == 0 && attachment.mask_b == 0 && attachment.mask_a == 0) { @@ -314,28 +359,34 @@ void FixedPipelineState::DynamicState::Refresh(const Maxwell& regs) { packed_front_face = 1 - packed_front_face; } - front.action_stencil_fail.Assign(PackStencilOp(regs.stencil_front_op.fail)); - front.action_depth_fail.Assign(PackStencilOp(regs.stencil_front_op.zfail)); - front.action_depth_pass.Assign(PackStencilOp(regs.stencil_front_op.zpass)); - front.test_func.Assign(PackComparisonOp(regs.stencil_front_op.func)); - if (regs.stencil_two_side_enable) { - back.action_stencil_fail.Assign(PackStencilOp(regs.stencil_back_op.fail)); - back.action_depth_fail.Assign(PackStencilOp(regs.stencil_back_op.zfail)); - back.action_depth_pass.Assign(PackStencilOp(regs.stencil_back_op.zpass)); - back.test_func.Assign(PackComparisonOp(regs.stencil_back_op.func)); - } else { - back.action_stencil_fail.Assign(front.action_stencil_fail); - back.action_depth_fail.Assign(front.action_depth_fail); - back.action_depth_pass.Assign(front.action_depth_pass); - back.test_func.Assign(front.test_func); + if (regs.stencil_enable) { + front.action_stencil_fail.Assign(PackStencilOp(regs.stencil_front_op.fail)); + front.action_depth_fail.Assign(PackStencilOp(regs.stencil_front_op.zfail)); + front.action_depth_pass.Assign(PackStencilOp(regs.stencil_front_op.zpass)); + front.test_func.Assign(PackComparisonOp(regs.stencil_front_op.func)); + if (regs.stencil_two_side_enable) { + back.action_stencil_fail.Assign(PackStencilOp(regs.stencil_back_op.fail)); + back.action_depth_fail.Assign(PackStencilOp(regs.stencil_back_op.zfail)); + back.action_depth_pass.Assign(PackStencilOp(regs.stencil_back_op.zpass)); + back.test_func.Assign(PackComparisonOp(regs.stencil_back_op.func)); + } else { + back.action_stencil_fail.Assign(front.action_stencil_fail); + back.action_depth_fail.Assign(front.action_depth_fail); + back.action_depth_pass.Assign(front.action_depth_pass); + back.test_func.Assign(front.test_func); + } } stencil_enable.Assign(regs.stencil_enable); depth_write_enable.Assign(regs.depth_write_enabled); depth_bounds_enable.Assign(regs.depth_bounds_enable); depth_test_enable.Assign(regs.depth_test_enable); front_face.Assign(packed_front_face); - depth_test_func.Assign(PackComparisonOp(regs.depth_test_func)); - cull_face.Assign(PackCullFace(regs.gl_cull_face)); + if (regs.depth_test_enable) { + depth_test_func.Assign(PackComparisonOp(regs.depth_test_func)); + } + if (regs.gl_cull_test_enabled) { + cull_face.Assign(PackCullFace(regs.gl_cull_face)); + } cull_enable.Assign(regs.gl_cull_test_enabled != 0 ? 1 : 0); } @@ -364,6 +415,9 @@ void FixedPipelineState::DynamicState::Refresh3(const Maxwell& regs, const DynamicFeatures& features) { if (!features.has_dynamic_state3_logic_op_enable) { logic_op_enable.Assign(regs.logic_op.enable != 0 ? 1 : 0); + if (!regs.logic_op.enable) { + logic_op.Assign(0); + } } if (!features.has_dynamic_state3_depth_clamp_enable) { depth_clamp_disabled.Assign(regs.viewport_clip_control.geometry_clip == diff --git a/src/video_core/renderer_vulkan/fixed_pipeline_state.h b/src/video_core/renderer_vulkan/fixed_pipeline_state.h index 36a4b4b7a2..6e114ec111 100644 --- a/src/video_core/renderer_vulkan/fixed_pipeline_state.h +++ b/src/video_core/renderer_vulkan/fixed_pipeline_state.h @@ -38,6 +38,7 @@ struct DynamicFeatures { bool has_provoking_vertex_first_mode; bool has_provoking_vertex_last_mode; bool has_provoking_vertex_tf_preserve; + bool has_dynamic_rendering_unused_attachments; }; struct FixedPipelineState { @@ -230,7 +231,6 @@ struct FixedPipelineState { BitField<14, 1, u32> smooth_lines; BitField<15, 1, u32> alpha_to_coverage_enabled; BitField<16, 1, u32> alpha_to_one_enabled; - BitField<17, 3, Tegra::Engines::Maxwell3D::EngineHint> app_stage; }; std::array color_formats; @@ -238,6 +238,8 @@ struct FixedPipelineState { u32 driver_version; u32 alpha_test_ref; u32 point_size; + u32 line_stipple_factor; + u32 line_stipple_pattern; std::array viewport_swizzles; union { @@ -254,13 +256,8 @@ struct FixedPipelineState { VideoCommon::TransformFeedbackState xfb_state; - u32 depth_bounds_min; - u32 depth_bounds_max; - - u32 line_stipple_factor; - u32 line_stipple_pattern; - - void Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFeatures& features); + void Refresh(Tegra::Engines::Maxwell3D& maxwell3d, DynamicFeatures& features, + u32 color_outputs); size_t Hash() const noexcept; diff --git a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp index 6c18ade926..12c311ad35 100644 --- a/src/video_core/renderer_vulkan/maxwell_to_vk.cpp +++ b/src/video_core/renderer_vulkan/maxwell_to_vk.cpp @@ -9,7 +9,6 @@ #include "common/assert.h" #include "common/common_types.h" #include "common/logging.h" -#include "common/settings.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/renderer_vulkan/maxwell_to_vk.h" #include "video_core/surface.h" @@ -71,12 +70,14 @@ VkSamplerAddressMode WrapMode(const Device& device, } ASSERT(false); return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; - case Tegra::Texture::WrapMode::MirrorOnceClampToEdge: - return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; case Tegra::Texture::WrapMode::MirrorOnceBorder: UNIMPLEMENTED(); - return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; + [[fallthrough]]; + case Tegra::Texture::WrapMode::MirrorOnceClampToEdge: case Tegra::Texture::WrapMode::MirrorOnceClampOGL: + if (!device.IsKhrSamplerMirrorClampToEdgeSupported()) { + return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; + } return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; default: UNIMPLEMENTED_MSG("Unimplemented wrap mode={}", wrap_mode); @@ -223,10 +224,6 @@ FormatInfo SurfaceFormat(const Device& device, FormatType format_type, bool with SURFACE_FORMAT_ELEM(VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK, 0, ETC2_RGB_SRGB) \ SURFACE_FORMAT_ELEM(VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK, 0, ETC2_RGBA_SRGB) \ SURFACE_FORMAT_ELEM(VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK, 0, ETC2_RGB_PTA_SRGB) \ - SURFACE_FORMAT_ELEM(VK_FORMAT_EAC_R11_UNORM_BLOCK, 0, EAC_R11_UNORM) \ - SURFACE_FORMAT_ELEM(VK_FORMAT_EAC_R11_SNORM_BLOCK, 0, EAC_R11_SNORM) \ - SURFACE_FORMAT_ELEM(VK_FORMAT_EAC_R11G11_UNORM_BLOCK, 0, EAC_R11G11_UNORM) \ - SURFACE_FORMAT_ELEM(VK_FORMAT_EAC_R11G11_SNORM_BLOCK, 0, EAC_R11G11_SNORM) \ /* Depth formats */ \ SURFACE_FORMAT_ELEM(VK_FORMAT_D32_SFLOAT, usage_attachable, D32_FLOAT) \ SURFACE_FORMAT_ELEM(VK_FORMAT_D16_UNORM, usage_attachable, D16_UNORM) \ @@ -246,21 +243,11 @@ FormatInfo SurfaceFormat(const Device& device, FormatType format_type, bool with bool const is_srgb = with_srgb && VideoCore::Surface::IsPixelFormatSRGB(pixel_format); // Transcode on hardware that doesn't support ASTC natively if (!device.IsOptimalAstcSupported() && VideoCore::Surface::IsPixelFormatASTC(pixel_format)) { - switch (Settings::values.astc_recompression.GetValue()) { - case Settings::AstcRecompression::Uncompressed: - if (is_srgb) { - tuple.format = VK_FORMAT_A8B8G8R8_SRGB_PACK32; - } else { - tuple.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; - tuple.usage |= usage_storage; - } - break; - case Settings::AstcRecompression::Bc1: - tuple.format = is_srgb ? VK_FORMAT_BC1_RGBA_SRGB_BLOCK : VK_FORMAT_BC1_RGBA_UNORM_BLOCK; - break; - case Settings::AstcRecompression::Bc3: - tuple.format = is_srgb ? VK_FORMAT_BC3_SRGB_BLOCK : VK_FORMAT_BC3_UNORM_BLOCK; - break; + if (is_srgb) { + tuple.format = VK_FORMAT_A8B8G8R8_SRGB_PACK32; + } else { + tuple.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; + tuple.usage |= usage_storage; } } if (!device.IsOptimalBcnSupported() && VideoCore::Surface::IsPixelFormatBCn(pixel_format)) { @@ -282,17 +269,7 @@ FormatInfo SurfaceFormat(const Device& device, FormatType format_type, bool with } } else if (!device.IsOptimalEtc2Supported() && VideoCore::Surface::IsPixelFormatETC2(pixel_format)) { // Transcode on hardware that doesn't support ETC2 natively - if (pixel_format == PixelFormat::EAC_R11_SNORM) { - tuple.format = VK_FORMAT_R8_SNORM; - } else if (pixel_format == PixelFormat::EAC_R11_UNORM) { - tuple.format = VK_FORMAT_R8_UNORM; - } else if (pixel_format == PixelFormat::EAC_R11G11_SNORM) { - tuple.format = VK_FORMAT_R8G8_SNORM; - } else if (pixel_format == PixelFormat::EAC_R11G11_UNORM) { - tuple.format = VK_FORMAT_R8G8_UNORM; - } else { - tuple.format = is_srgb ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 : VK_FORMAT_A8B8G8R8_UNORM_PACK32; - } + tuple.format = is_srgb ? VK_FORMAT_A8B8G8R8_SRGB_PACK32 : VK_FORMAT_A8B8G8R8_UNORM_PACK32; } bool const attachable = (tuple.usage & usage_attachable) != 0; bool const storage = (tuple.usage & usage_storage) != 0; diff --git a/src/video_core/renderer_vulkan/present/frame_gen.cpp b/src/video_core/renderer_vulkan/present/frame_gen.cpp index e6d857323b..186562a6e7 100644 --- a/src/video_core/renderer_vulkan/present/frame_gen.cpp +++ b/src/video_core/renderer_vulkan/present/frame_gen.cpp @@ -2,6 +2,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include +#include #include #include @@ -9,10 +10,12 @@ #include "common/fs/fs.h" #include "common/fs/path_util.h" #include "common/settings.h" +#include "video_core/host_shaders/vulkan_fidelityfx_fsr_vert_spv.h" +#include "video_core/host_shaders/vulkan_present_frag_spv.h" #include "video_core/renderer_vulkan/present/frame_gen.h" #include "video_core/renderer_vulkan/present/util.h" -#include "video_core/renderer_vulkan/vk_present_manager.h" #include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/renderer_vulkan/vk_shader_util.h" #include "video_core/vulkan_common/vulkan_device.h" namespace Vulkan { @@ -23,27 +26,13 @@ constexpr size_t COLOR_CHANNELS = 4; constexpr u64 LSFG_REQUIRED_FRAMES = 2; constexpr u32 LSFG_RECURRENCE_FRAMES = 2; -[[nodiscard]] f32 ManualFlowScale() { +[[nodiscard]] f32 ConfiguredFlowScale() { + if (Settings::values.frame_gen_flow_scale_auto.GetValue()) { + return std::clamp(1.0f / Settings::values.resolution_info.up_factor, 0.25f, 1.0f); + } return static_cast(Settings::values.frame_gen_flow_scale.GetValue()) / 100.0f; } -[[nodiscard]] f32 ConfiguredFlowScale(VkExtent2D guest_extent, VkExtent2D presented_extent) { - if (!Settings::values.frame_gen_flow_scale_auto.GetValue()) { - return ManualFlowScale(); - } - if (guest_extent.width == 0 || presented_extent.width == 0) { - return 1.0f; - } - - const f32 rendered_width = static_cast(guest_extent.width) * - Settings::values.resolution_info.up_factor; - const f32 ratio = rendered_width / static_cast(presented_extent.width); - - constexpr f32 FLOW_SCALE_STEPS = 20.0f; - const f32 stepped = std::ceil(ratio * FLOW_SCALE_STEPS) / FLOW_SCALE_STEPS; - return std::clamp(stepped, 0.25f, 1.0f); -} - bool IsBlueFirst(VkFormat format) { return format == VK_FORMAT_B8G8R8A8_UNORM || format == VK_FORMAT_B8G8R8A8_SRGB; } @@ -80,7 +69,7 @@ void WritePortablePixmap(const std::filesystem::path& path, const std::string& m void WriteGrayscalePgm(const std::filesystem::path& path, VkExtent2D extent, std::span pixels) { const size_t expected = static_cast(extent.width) * extent.height; - WritePortablePixmap(path, "P5", extent, pixels.subspan(0, std::min(expected, pixels.size()))); + WritePortablePixmap(path, "P5", extent, pixels.subspan(0, (std::min)(expected, pixels.size()))); } void WriteRaw(const std::filesystem::path& path, std::span pixels) { @@ -113,13 +102,17 @@ void WriteColorPpm(const std::filesystem::path& path, VkExtent2D extent, WritePortablePixmap(path, "P6", extent, rgb); } -VkImageMemoryBarrier MakeTransitionBarrier(VkImage image, VkAccessFlags src_access, - VkAccessFlags dst_access, VkImageLayout old_layout, - VkImageLayout new_layout) { - return VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, +VkImageMemoryBarrier2 MakeTransitionBarrier(VkImage image, VkPipelineStageFlags2 src_stage, + VkAccessFlags2 src_access, + VkPipelineStageFlags2 dst_stage, + VkAccessFlags2 dst_access, VkImageLayout old_layout, + VkImageLayout new_layout) { + return VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = src_stage, .srcAccessMask = src_access, + .dstStageMask = dst_stage, .dstAccessMask = dst_access, .oldLayout = old_layout, .newLayout = new_layout, @@ -136,57 +129,65 @@ VkImageMemoryBarrier MakeTransitionBarrier(VkImage image, VkAccessFlags src_acce }; } -VkImageCopy MakeCopyRegion(VkExtent2D extent) { - return VkImageCopy{ - .srcSubresource{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .mipLevel = 0, - .baseArrayLayer = 0, - .layerCount = 1, - }, - .srcOffset = {}, - .dstSubresource{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .mipLevel = 0, - .baseArrayLayer = 0, - .layerCount = 1, - }, - .dstOffset = {}, - .extent = {.width = extent.width, .height = extent.height, .depth = 1}, +void DrawSourceFrame(vk::CommandBuffer cmdbuf, VkPipeline pipeline, VkPipelineLayout layout, + VkDescriptorSet set, LsfgImage& destination, VkExtent2D extent) { + const VkImageMemoryBarrier2 before = MakeTransitionBarrier( + destination.Handle(), VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, VK_ACCESS_2_NONE, + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, + destination.Layout(), VK_IMAGE_LAYOUT_GENERAL); + cmdbuf.PipelineBarrier(before); + + const VkViewport viewport{ + .x = 0.0f, + .y = 0.0f, + .width = static_cast(extent.width), + .height = static_cast(extent.height), + .minDepth = 0.0f, + .maxDepth = 1.0f, }; -} - -void CopyPresentedFrame(vk::CommandBuffer cmdbuf, VkImage source, LsfgImage& destination, - VkExtent2D extent) { - const auto make_barrier = MakeTransitionBarrier; - - const std::array before{ - make_barrier(source, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT, - VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL), - make_barrier(destination.Handle(), VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, - destination.Layout(), VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL), + const VkRect2D scissor{ + .offset = {0, 0}, + .extent = extent, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, {}, {}, before); + BeginRendering(cmdbuf, destination.View(), extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, set, {}); + cmdbuf.SetViewport(0, viewport); + cmdbuf.SetScissor(0, scissor); + cmdbuf.Draw(3, 1, 0, 0); + cmdbuf.EndRendering(); - cmdbuf.CopyImage(source, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destination.Handle(), - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, MakeCopyRegion(extent)); - - const std::array after{ - make_barrier(source, VK_ACCESS_TRANSFER_READ_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL), - make_barrier(destination.Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL), - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, {}, {}, after); + const VkImageMemoryBarrier2 after = MakeTransitionBarrier( + destination.Handle(), VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_SHADER_READ_BIT, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_GENERAL); + cmdbuf.PipelineBarrier(after); destination.SetLayout(VK_IMAGE_LAYOUT_GENERAL); } +void UpdateSourceSet(const Device& device, VkDescriptorSet set, VkSampler sampler, + VkImageView view) { + const VkDescriptorImageInfo image_info{ + .sampler = sampler, + .imageView = view, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + }; + const VkWriteDescriptorSet write{ + .sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, + .pNext = nullptr, + .dstSet = set, + .dstBinding = 0, + .dstArrayElement = 0, + .descriptorCount = 1, + .descriptorType = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, + .pImageInfo = &image_info, + .pBufferInfo = nullptr, + .pTexelBufferView = nullptr, + }; + device.GetLogical().UpdateDescriptorSets(std::array{write}, {}); +} + } // Anonymous namespace FrameGen::FrameGen(MemoryAllocator& memory_allocator_, Scheduler& scheduler_) @@ -194,18 +195,19 @@ FrameGen::FrameGen(MemoryAllocator& memory_allocator_, Scheduler& scheduler_) FrameGen::~FrameGen() = default; -void FrameGen::Process(const Device& device, Frame* frame, VkFormat format, - VkExtent2D guest_extent) { +void FrameGen::Process(const Device& device, VkImageView source, VkExtent2D extent) { generated = false; if (!shaders) { shaders.emplace(device); + CreateInputPass(device); } if (unavailable || !Settings::values.frame_gen.GetValue()) { if (chain) { scheduler.Finish(); chain.reset(); + outputs = {}; } warm_streak = 0; return; @@ -216,37 +218,39 @@ void FrameGen::Process(const Device& device, Frame* frame, VkFormat format, return; } - if (!frame->storage_view) { + if (source == VK_NULL_HANDLE) { warm_streak = 0; return; } - peak_guest_extent.width = std::max(peak_guest_extent.width, guest_extent.width); - peak_guest_extent.height = std::max(peak_guest_extent.height, guest_extent.height); - - const VkExtent2D extent{.width = frame->width, .height = frame->height}; - const f32 flow_scale = ConfiguredFlowScale(peak_guest_extent, extent); + const f32 flow_scale = ConfiguredFlowScale(); + const size_t generations = Settings::FrameGenMaxGenerations(); if (!chain || built_extent.width != extent.width || built_extent.height != extent.height || - built_format != format || built_flow_scale != flow_scale) { - Rebuild(device, extent, format, flow_scale); + built_flow_scale != flow_scale || built_generations != generations) { + Rebuild(device, extent, flow_scale, generations); } const u64 count = frame_count++; last_count = count; - last_generations = plan.generations; + last_generations = (std::min)(plan.generations, built_generations); - const bool warm = plan.warm && count + 1 >= LSFG_REQUIRED_FRAMES; + const size_t input_slot = count % INPUT_SLOTS; + const bool warm = plan.warm && count + 1 >= LSFG_REQUIRED_FRAMES && + scheduler.IsFree(input_ticks[input_slot]); warm_streak = warm ? warm_streak + 1 : 0; generated = warm && warm_streak >= LSFG_RECURRENCE_FRAMES && plan.generations > 0; - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([this, source = *frame->image, extent, count, - dispatch = warm](vk::CommandBuffer cmdbuf) { - CopyPresentedFrame(cmdbuf, source, chain->Input(count), extent); - if (dispatch) { + if (warm) { + const VkDescriptorSet set = input_sets[input_slot]; + input_ticks[input_slot] = scheduler.CurrentTick(); + UpdateSourceSet(device, set, *input_sampler, source); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([this, set, extent, count](vk::CommandBuffer cmdbuf) { + DrawSourceFrame(cmdbuf, *input_pipeline, *input_layout, set, chain->Input(count), + extent); chain->DispatchShared(cmdbuf, count); - } - }); + }); + } const bool dump_requested = generated && Settings::values.frame_gen_dump_flow.GetValue(); if (!dump_requested) { @@ -270,30 +274,53 @@ size_t FrameGen::GeneratedFrameCount() const { return generated ? last_generations : 0; } -void FrameGen::GenerateInto(const Device& device, Frame* destination, size_t generation) { - chain->SetTarget(device, last_generations, generation, destination->index, - *destination->storage_view); +std::chrono::nanoseconds FrameGen::PaceStep() const { + const f32 step = plan.interval / static_cast(last_generations + 1); + return std::chrono::duration_cast(std::chrono::duration(step)); +} - const VkExtent2D extent{.width = destination->width, .height = destination->height}; +const LsfgImage& FrameGen::Generate(const Device& device, size_t generation) { + LsfgImage& output = outputs[generation]; + chain->SetTarget(device, last_generations, generation, output.View()); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, count = last_count, generation_count = last_generations, generation, - target = destination->index, image = *destination->image, - extent](vk::CommandBuffer cmdbuf) { - chain->DispatchGeneration(cmdbuf, count, generation_count, generation, target, image, - extent); + image = output.Handle(), extent = built_extent](vk::CommandBuffer cmdbuf) { + chain->DispatchGeneration(cmdbuf, count, generation_count, generation, image, extent); }); + return output; } -void FrameGen::Rebuild(const Device& device, VkExtent2D extent, VkFormat format, f32 flow_scale) { +void FrameGen::CreateInputPass(const Device& device) { + input_set_layout = + CreateWrappedDescriptorSetLayout(device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER}); + input_layout = CreateWrappedPipelineLayout(device, input_set_layout); + input_pool = CreateWrappedDescriptorPool(device, INPUT_SLOTS, INPUT_SLOTS); + std::array layouts; + layouts.fill(*input_set_layout); + input_sets = CreateWrappedDescriptorSets(input_pool, layouts); + input_vertex_shader = BuildShader(device, VULKAN_FIDELITYFX_FSR_VERT_SPV); + input_fragment_shader = BuildShader(device, VULKAN_PRESENT_FRAG_SPV); + input_pipeline = CreateWrappedPipeline(device, LSFG_DEFAULT_FORMAT, input_layout, + std::tie(input_vertex_shader, input_fragment_shader)); + input_sampler = CreateNearestNeighborSampler(device); +} + +void FrameGen::Rebuild(const Device& device, VkExtent2D extent, f32 flow_scale, + size_t generations) { scheduler.Finish(); chain.reset(); built_flow_scale = flow_scale; - chain.emplace(device, memory_allocator, *shaders, extent, format, built_flow_scale); + chain.emplace(device, memory_allocator, *shaders, extent, LSFG_DEFAULT_FORMAT, + built_flow_scale); + outputs = {}; + for (LsfgImage& output : std::span{outputs}.first(generations)) { + output = LsfgImage(device, memory_allocator, extent, LSFG_DEFAULT_FORMAT); + } built_extent = extent; - built_format = format; + built_generations = generations; frame_count = 0; warm_streak = 0; generated = false; diff --git a/src/video_core/renderer_vulkan/present/frame_gen.h b/src/video_core/renderer_vulkan/present/frame_gen.h index 19c6d04289..51cc142277 100644 --- a/src/video_core/renderer_vulkan/present/frame_gen.h +++ b/src/video_core/renderer_vulkan/present/frame_gen.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include "common/common_types.h" @@ -15,23 +16,27 @@ namespace Vulkan { class Device; class Scheduler; -struct Frame; class FrameGen { public: explicit FrameGen(MemoryAllocator& memory_allocator, Scheduler& scheduler); ~FrameGen(); - void Process(const Device& device, Frame* frame, VkFormat format, VkExtent2D guest_extent); + void Process(const Device& device, VkImageView source, VkExtent2D extent); [[nodiscard]] size_t WantedGenerations(size_t capacity); [[nodiscard]] size_t GeneratedFrameCount() const; - void GenerateInto(const Device& device, Frame* destination, size_t generation); + [[nodiscard]] std::chrono::nanoseconds PaceStep() const; + + [[nodiscard]] const LsfgImage& Generate(const Device& device, size_t generation); private: - void Rebuild(const Device& device, VkExtent2D extent, VkFormat format, f32 flow_scale); + static constexpr size_t INPUT_SLOTS = 4; + + void CreateInputPass(const Device& device); + void Rebuild(const Device& device, VkExtent2D extent, f32 flow_scale, size_t generations); void DumpDebugImages(u64 count); MemoryAllocator& memory_allocator; @@ -39,12 +44,21 @@ private: std::optional shaders; std::optional chain; + std::array outputs; + vk::DescriptorSetLayout input_set_layout; + vk::PipelineLayout input_layout; + vk::DescriptorPool input_pool; + vk::DescriptorSets input_sets; + vk::ShaderModule input_vertex_shader; + vk::ShaderModule input_fragment_shader; + vk::Pipeline input_pipeline; + vk::Sampler input_sampler; + std::array input_ticks{}; FrameGenPacer pacer; FrameGenPlan plan{}; - VkExtent2D peak_guest_extent{}; VkExtent2D built_extent{}; - VkFormat built_format{VK_FORMAT_UNDEFINED}; f32 built_flow_scale{}; + size_t built_generations{}; u64 frame_count{}; u64 last_count{}; size_t last_generations{}; diff --git a/src/video_core/renderer_vulkan/present/frame_gen_pacer.cpp b/src/video_core/renderer_vulkan/present/frame_gen_pacer.cpp index 78a8c83b5a..195a81be19 100644 --- a/src/video_core/renderer_vulkan/present/frame_gen_pacer.cpp +++ b/src/video_core/renderer_vulkan/present/frame_gen_pacer.cpp @@ -47,7 +47,7 @@ constexpr auto PROBE_STEP_DELAY = std::chrono::milliseconds(250); } // Anonymous namespace FrameGenPlan FrameGenPacer::Plan(size_t capacity) { - const size_t ceiling = std::min(capacity, Settings::FrameGenMaxGenerations()); + const size_t ceiling = (std::min)(capacity, Settings::FrameGenMaxGenerations()); if (ceiling == 0) { Reset(); return {}; @@ -74,7 +74,7 @@ FrameGenPlan FrameGenPacer::Plan(size_t capacity) { if (smoothed_interval > 0.0f) { f32 burst_threshold = BURST_CADENCE_RATIO / smoothed_interval; if (target_rate > 0.0f) { - burst_threshold = std::max(burst_threshold, target_rate * BURST_TARGET_RATIO); + burst_threshold = (std::max)(burst_threshold, target_rate * BURST_TARGET_RATIO); } if (1.0f / interval_seconds > burst_threshold) { DeferEvaluations(interval); @@ -109,15 +109,15 @@ FrameGenPlan FrameGenPacer::Plan(size_t capacity) { } if (target_rate == 0.0f) { - limit = std::min(Settings::FrameGenGenerations(), ceiling); + limit = (std::min)(Settings::FrameGenGenerations(), ceiling); output_credit = 0.0f; issued_generations = limit; - return {.generations = limit, .warm = limit > 0}; + return {.generations = limit, .warm = limit > 0, .interval = smoothed_interval}; } UpdateLimit(now, 1.0f / smoothed_interval, target_rate, ceiling); - const size_t allowed = std::min(limit, ceiling); + const size_t allowed = (std::min)(limit, ceiling); const f32 desired_outputs = smoothed_interval * target_rate; if (allowed == 0 || desired_outputs <= 1.0f) { output_credit = 0.0f; @@ -127,7 +127,7 @@ FrameGenPlan FrameGenPacer::Plan(size_t capacity) { output_credit += desired_outputs; const size_t outputs = std::max(1, static_cast(std::floor(output_credit + CREDIT_EPSILON))); - const size_t generations = std::min(outputs - 1, allowed); + const size_t generations = (std::min)(outputs - 1, allowed); output_credit -= static_cast(generations + 1); if (output_credit < 0.0f) { @@ -137,12 +137,12 @@ FrameGenPlan FrameGenPacer::Plan(size_t capacity) { } issued_generations = generations; - return {.generations = generations, .warm = true}; + return {.generations = generations, .warm = true, .interval = smoothed_interval}; } void FrameGenPacer::UpdateLimit(Clock::time_point now, f32 base_rate, f32 target_rate, size_t ceiling) { - limit = std::min(limit, ceiling); + limit = (std::min)(limit, ceiling); if (probe_until) { if (now < *probe_until) { @@ -152,9 +152,9 @@ void FrameGenPacer::UpdateLimit(Clock::time_point now, f32 base_rate, f32 target output_credit = 0.0f; const f32 previous_output = - std::min(target_rate, probe_base_rate * static_cast(probe_previous_limit + 1)); + (std::min)(target_rate, probe_base_rate * static_cast(probe_previous_limit + 1)); const f32 current_output = - std::min(target_rate, base_rate * static_cast(limit + 1)); + (std::min)(target_rate, base_rate * static_cast(limit + 1)); const bool throughput_regressed = current_output < previous_output * PROBE_THROUGHPUT_TOLERANCE; @@ -166,7 +166,7 @@ void FrameGenPacer::UpdateLimit(Clock::time_point now, f32 base_rate, f32 target if (throughput_regressed || collapsed_for_marginal_gain || emulation_slowed) { limit = probe_previous_limit; - probe_failures = std::min(probe_failures + 1, MAX_PROBE_FAILURES); + probe_failures = (std::min)(probe_failures + 1, MAX_PROBE_FAILURES); next_probe = now + ProbeBackoff(probe_failures); deficit_since.reset(); return; diff --git a/src/video_core/renderer_vulkan/present/frame_gen_pacer.h b/src/video_core/renderer_vulkan/present/frame_gen_pacer.h index 7f4b23920c..8c513af71c 100644 --- a/src/video_core/renderer_vulkan/present/frame_gen_pacer.h +++ b/src/video_core/renderer_vulkan/present/frame_gen_pacer.h @@ -13,6 +13,7 @@ namespace Vulkan { struct FrameGenPlan { size_t generations{}; bool warm{}; + f32 interval{}; }; class FrameGenPacer { diff --git a/src/video_core/renderer_vulkan/present/fsr.cpp b/src/video_core/renderer_vulkan/present/fsr.cpp index 894fb144f0..bb3138e7e6 100644 --- a/src/video_core/renderer_vulkan/present/fsr.cpp +++ b/src/video_core/renderer_vulkan/present/fsr.cpp @@ -31,7 +31,6 @@ FSR::FSR(const Device& device, MemoryAllocator& memory_allocator, size_t image_c , m_extent{extent} { CreateImages(device); - CreateRenderPasses(device); CreateSampler(device); CreateShaders(device); CreateDescriptorPool(device); @@ -42,20 +41,9 @@ FSR::FSR(const Device& device, MemoryAllocator& memory_allocator, size_t image_c } void FSR::CreateImages(const Device& device) { - m_dynamic_images.resize(m_image_count); - for (auto& images : m_dynamic_images) { - images.images[Easu] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - images.images[Rcas] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - images.image_views[Easu] = CreateWrappedImageView(device, images.images[Easu], VK_FORMAT_R16G16B16A16_SFLOAT); - images.image_views[Rcas] = CreateWrappedImageView(device, images.images[Rcas], VK_FORMAT_R16G16B16A16_SFLOAT); - } -} - -void FSR::CreateRenderPasses(const Device& device) { - m_renderpass = CreateWrappedRenderPass(device, VK_FORMAT_R16G16B16A16_SFLOAT); - for (auto& images : m_dynamic_images) { - images.framebuffers[Easu] = CreateWrappedFramebuffer(device, m_renderpass, images.image_views[Easu], m_extent); - images.framebuffers[Rcas] = CreateWrappedFramebuffer(device, m_renderpass, images.image_views[Rcas], m_extent); + for (size_t stage = 0; stage < MaxFsrStage; ++stage) { + m_images[stage] = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); + m_image_views[stage] = CreateWrappedImageView(device, m_images[stage], VK_FORMAT_R16G16B16A16_SFLOAT); } } @@ -88,8 +76,8 @@ void FSR::CreateDescriptorSetLayout(const Device& device) { void FSR::CreateDescriptorSets(const Device& device) { std::vector layouts(MaxFsrStage, *m_descriptor_set_layout); - for (auto& images : m_dynamic_images) - images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, layouts); + for (size_t i = 0; i < m_image_count; ++i) + m_descriptor_sets.push_back(CreateWrappedDescriptorSets(m_descriptor_pool, layouts)); } void FSR::CreatePipelineLayouts(const Device& device) { @@ -112,51 +100,39 @@ void FSR::CreatePipelineLayouts(const Device& device) { } void FSR::CreatePipelines(const Device& device) { - m_easu_pipeline = CreateWrappedPipeline(device, m_renderpass, m_pipeline_layout, + m_easu_pipeline = CreateWrappedPipeline(device, VK_FORMAT_R16G16B16A16_SFLOAT, + m_pipeline_layout, std::tie(m_vert_shader, m_easu_shader)); - m_rcas_pipeline = CreateWrappedPipeline(device, m_renderpass, m_pipeline_layout, + m_rcas_pipeline = CreateWrappedPipeline(device, VK_FORMAT_R16G16B16A16_SFLOAT, + m_pipeline_layout, std::tie(m_vert_shader, m_rcas_shader)); } void FSR::UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index) { - Images& images = m_dynamic_images[image_index]; + const vk::DescriptorSets& descriptor_sets = m_descriptor_sets[image_index]; std::vector image_infos; image_infos.reserve(2); std::vector updates{ - CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, images.descriptor_sets[Easu], 0), - CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Easu], images.descriptor_sets[Rcas], 0) + CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, descriptor_sets[Easu], 0), + CreateWriteDescriptorSet(image_infos, *m_sampler, *m_image_views[Easu], descriptor_sets[Rcas], 0) }; device.GetLogical().UpdateDescriptorSets(updates, {}); } -void FSR::UploadImages(const Device& device, Scheduler& scheduler) { - if (!m_images_ready) { - m_images_ready = true; - scheduler.Record([&](vk::CommandBuffer cmdbuf) { - for (auto& image : m_dynamic_images) { - ClearColorImage(cmdbuf, *image.images[Easu]); - ClearColorImage(cmdbuf, *image.images[Rcas]); - } - }); - scheduler.Finish(); - } -} - VkImageView FSR::Draw(const Device& device, Scheduler& scheduler, size_t image_index, VkImage source_image, VkImageView source_image_view, VkExtent2D input_image_extent, const Common::Rectangle& crop_rect) { - Images& images = m_dynamic_images[image_index]; + const vk::DescriptorSets& descriptor_sets = m_descriptor_sets[image_index]; - VkImage easu_image = *images.images[Easu]; - VkImage rcas_image = *images.images[Rcas]; - VkDescriptorSet easu_descriptor_set = images.descriptor_sets[Easu]; - VkDescriptorSet rcas_descriptor_set = images.descriptor_sets[Rcas]; - VkFramebuffer easu_framebuffer = *images.framebuffers[Easu]; - VkFramebuffer rcas_framebuffer = *images.framebuffers[Rcas]; + VkImage easu_image = *m_images[Easu]; + VkImage rcas_image = *m_images[Rcas]; + VkDescriptorSet easu_descriptor_set = descriptor_sets[Easu]; + VkDescriptorSet rcas_descriptor_set = descriptor_sets[Rcas]; + VkImageView easu_view = *m_image_views[Easu]; + VkImageView rcas_view = *m_image_views[Rcas]; VkPipeline easu_pipeline = *m_easu_pipeline; VkPipeline rcas_pipeline = *m_rcas_pipeline; VkPipelineLayout pipeline_layout = *m_pipeline_layout; - VkRenderPass renderpass = *m_renderpass; VkExtent2D extent = m_extent; const f32 input_image_width = static_cast(input_image_extent.width); @@ -179,35 +155,36 @@ VkImageView FSR::Draw(const Device& device, Scheduler& scheduler, size_t image_i static_cast(Settings::values.fsr_sharpening_slider.GetValue()) / 100.0f; FsrRcasCon(rcas_con.data(), sharpening); - UploadImages(device, scheduler); UpdateDescriptorSets(device, source_image_view, image_index); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([=](vk::CommandBuffer cmdbuf) { TransitionImageLayout(cmdbuf, source_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, easu_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, renderpass, easu_framebuffer, extent); + TransitionImageLayout(cmdbuf, easu_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, easu_view, extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, easu_pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, easu_descriptor_set, {}); cmdbuf.PushConstants(pipeline_layout, VK_SHADER_STAGE_FRAGMENT_BIT, easu_con); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, easu_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, rcas_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, renderpass, rcas_framebuffer, extent); + TransitionImageLayout(cmdbuf, rcas_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, rcas_view, extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, rcas_pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline_layout, 0, rcas_descriptor_set, {}); cmdbuf.PushConstants(pipeline_layout, VK_SHADER_STAGE_FRAGMENT_BIT, rcas_con); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, rcas_image, VK_IMAGE_LAYOUT_GENERAL); }); - return *images.image_views[Rcas]; + return rcas_view; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/fsr.h b/src/video_core/renderer_vulkan/present/fsr.h index 7762480757..373a985339 100644 --- a/src/video_core/renderer_vulkan/present/fsr.h +++ b/src/video_core/renderer_vulkan/present/fsr.h @@ -24,7 +24,6 @@ public: private: void CreateImages(const Device& device); - void CreateRenderPasses(const Device& device); void CreateSampler(const Device& device); void CreateShaders(const Device& device); void CreateDescriptorPool(const Device& device); @@ -33,7 +32,6 @@ private: void CreatePipelineLayouts(const Device& device); void CreatePipelines(const Device& device); - void UploadImages(const Device& device, Scheduler& scheduler); void UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index); MemoryAllocator& m_memory_allocator; @@ -54,17 +52,11 @@ private: vk::ShaderModule m_rcas_shader; vk::Pipeline m_easu_pipeline; vk::Pipeline m_rcas_pipeline; - vk::RenderPass m_renderpass; vk::Sampler m_sampler; - struct Images { - vk::DescriptorSets descriptor_sets; - std::array images; - std::array image_views; - std::array framebuffers; - }; - std::vector m_dynamic_images; - bool m_images_ready{}; + std::array m_images; + std::array m_image_views; + std::vector m_descriptor_sets; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/fxaa.cpp b/src/video_core/renderer_vulkan/present/fxaa.cpp index 62f79d490c..6eb07b004f 100644 --- a/src/video_core/renderer_vulkan/present/fxaa.cpp +++ b/src/video_core/renderer_vulkan/present/fxaa.cpp @@ -21,7 +21,6 @@ FXAA::FXAA(const Device& device, MemoryAllocator& allocator, size_t image_count, , m_image_count(u32(image_count)) { CreateImages(device, allocator); - CreateRenderPasses(device); CreateSampler(device); CreateShaders(device); CreateDescriptorPool(device); @@ -34,20 +33,8 @@ FXAA::FXAA(const Device& device, MemoryAllocator& allocator, size_t image_count, FXAA::~FXAA() = default; void FXAA::CreateImages(const Device& device, MemoryAllocator& allocator) { - for (u32 i = 0; i < m_image_count; i++) { - Image& image = m_dynamic_images.emplace_back(); - image.image = CreateWrappedImage(allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - image.image_view = CreateWrappedImageView(device, image.image, VK_FORMAT_R16G16B16A16_SFLOAT); - } -} - -void FXAA::CreateRenderPasses(const Device& device) { - m_renderpass = CreateWrappedRenderPass(device, VK_FORMAT_R16G16B16A16_SFLOAT); - - for (auto& image : m_dynamic_images) { - image.framebuffer = - CreateWrappedFramebuffer(device, m_renderpass, image.image_view, m_extent); - } + m_image = CreateWrappedImage(allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); + m_image_view = CreateWrappedImageView(device, m_image, VK_FORMAT_R16G16B16A16_SFLOAT); } void FXAA::CreateSampler(const Device& device) { @@ -73,8 +60,8 @@ void FXAA::CreateDescriptorSetLayouts(const Device& device) { void FXAA::CreateDescriptorSets(const Device& device) { VkDescriptorSetLayout layout = *m_descriptor_set_layout; - for (auto& images : m_dynamic_images) { - images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, {layout}); + for (u32 i = 0; i < m_image_count; i++) { + m_descriptor_sets.push_back(CreateWrappedDescriptorSets(m_descriptor_pool, {layout})); } } @@ -83,65 +70,48 @@ void FXAA::CreatePipelineLayouts(const Device& device) { } void FXAA::CreatePipelines(const Device& device) { - m_pipeline = CreateWrappedPipeline(device, m_renderpass, m_pipeline_layout, + m_pipeline = CreateWrappedPipeline(device, VK_FORMAT_R16G16B16A16_SFLOAT, m_pipeline_layout, std::tie(m_vertex_shader, m_fragment_shader)); } void FXAA::UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index) { - Image& image = m_dynamic_images[image_index]; + const VkDescriptorSet descriptor_set = m_descriptor_sets[image_index][0]; std::vector image_infos; std::vector updates; image_infos.reserve(2); - updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, image.descriptor_sets[0], 0)); - updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, image.descriptor_sets[0], 1)); + updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, descriptor_set, 0)); + updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, descriptor_set, 1)); device.GetLogical().UpdateDescriptorSets(updates, {}); } -void FXAA::UploadImages(const Device& device, Scheduler& scheduler) { - if (m_images_ready) { - return; - } - - scheduler.Record([&](vk::CommandBuffer cmdbuf) { - for (auto& image : m_dynamic_images) { - ClearColorImage(cmdbuf, *image.image); - } - }); - scheduler.Finish(); - - m_images_ready = true; -} - void FXAA::Draw(const Device& device, Scheduler& scheduler, size_t image_index, VkImage* inout_image, VkImageView* inout_image_view) { - const Image& image{m_dynamic_images[image_index]}; const VkImage input_image{*inout_image}; - const VkImage output_image{*image.image}; - const VkDescriptorSet descriptor_set{image.descriptor_sets[0]}; - const VkFramebuffer framebuffer{*image.framebuffer}; - const VkRenderPass renderpass{*m_renderpass}; + const VkImage output_image{*m_image}; + const VkImageView output_view{*m_image_view}; + const VkDescriptorSet descriptor_set{m_descriptor_sets[image_index][0]}; const VkPipeline pipeline{*m_pipeline}; const VkPipelineLayout layout{*m_pipeline_layout}; const VkExtent2D extent{m_extent}; - UploadImages(device, scheduler); UpdateDescriptorSets(device, *inout_image_view, image_index); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([=](vk::CommandBuffer cmdbuf) { TransitionImageLayout(cmdbuf, input_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, renderpass, framebuffer, extent); + TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, output_view, extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set, {}); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL); }); - *inout_image = *image.image; - *inout_image_view = *image.image_view; + *inout_image = output_image; + *inout_image_view = output_view; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/fxaa.h b/src/video_core/renderer_vulkan/present/fxaa.h index 7eaf31f81c..7f5599db16 100644 --- a/src/video_core/renderer_vulkan/present/fxaa.h +++ b/src/video_core/renderer_vulkan/present/fxaa.h @@ -25,7 +25,6 @@ public: private: void CreateImages(const Device& device, MemoryAllocator& allocator); - void CreateRenderPasses(const Device& device); void CreateSampler(const Device& device); void CreateShaders(const Device& device); void CreateDescriptorPool(const Device& device); @@ -34,15 +33,10 @@ private: void CreatePipelineLayouts(const Device& device); void CreatePipelines(const Device& device); void UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index); - void UploadImages(const Device& device, Scheduler& scheduler); - struct Image { - vk::DescriptorSets descriptor_sets{}; - vk::Framebuffer framebuffer{}; - vk::Image image{}; - vk::ImageView image_view{}; - }; - std::vector m_dynamic_images{}; + vk::Image m_image{}; + vk::ImageView m_image_view{}; + std::vector m_descriptor_sets{}; const VkExtent2D m_extent; const u32 m_image_count; vk::ShaderModule m_vertex_shader{}; @@ -51,9 +45,7 @@ private: vk::DescriptorSetLayout m_descriptor_set_layout{}; vk::PipelineLayout m_pipeline_layout{}; vk::Pipeline m_pipeline{}; - vk::RenderPass m_renderpass{}; vk::Sampler m_sampler{}; - bool m_images_ready{}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/layer.cpp b/src/video_core/renderer_vulkan/present/layer.cpp index 1868142f51..f7152a31eb 100644 --- a/src/video_core/renderer_vulkan/present/layer.cpp +++ b/src/video_core/renderer_vulkan/present/layer.cpp @@ -60,25 +60,34 @@ VkFormat GetFormat(const Tegra::FramebufferConfig& framebuffer) { } // Anonymous namespace -Layer::Layer(const Device& device, MemoryAllocator& memory_allocator_, Scheduler& scheduler_, Tegra::MaxwellDeviceMemoryManager& device_memory_, size_t image_count_, VkExtent2D output_size, VkDescriptorSetLayout layout, const PresentFilters& filters_) +Layer::Layer(const Device& device, MemoryAllocator& memory_allocator_, Scheduler& scheduler_, + Tegra::MaxwellDeviceMemoryManager& device_memory_, size_t image_count_, + size_t generation_count_, VkExtent2D output_size, VkDescriptorSetLayout layout, + const PresentFilters& filters_) : memory_allocator(memory_allocator_) , scheduler(scheduler_) , device_memory(device_memory_) , filters(filters_) , image_count(image_count_) + , generation_count(generation_count_) + , generation_ticks(generation_count_) { CreateDescriptorPool(device); CreateDescriptorSets(device, layout); + const size_t slot_count = image_count + generation_count; if (filters.get_scaling_filter() == Settings::ScalingFilter::Fsr) { - sr_filter.emplace(device, memory_allocator, image_count, output_size); + sr_filter.emplace(device, memory_allocator, slot_count, output_size); } else if (filters.get_scaling_filter() == Settings::ScalingFilter::Sgsr) { - sr_filter.emplace(device, memory_allocator, image_count, output_size, false); + sr_filter.emplace(device, memory_allocator, slot_count, output_size, false); } else if (filters.get_scaling_filter() == Settings::ScalingFilter::SgsrEdge) { - sr_filter.emplace(device, memory_allocator, image_count, output_size, true); + sr_filter.emplace(device, memory_allocator, slot_count, output_size, true); } } Layer::~Layer() { + for (const u64 tick : generation_ticks) { + scheduler.Wait(tick); + } ReleaseRawImages(); } @@ -115,24 +124,36 @@ void Layer::ConfigureDraw(const Device& device, PresentPushConstants* out_push_c VkImage source_image = texture_info ? texture_info->image : *raw_images[image_index]; VkImageView source_image_view = texture_info ? texture_info->image_view : *raw_image_views[image_index]; + const VkExtent2D render_extent{ + .width = scaled_width, + .height = scaled_height, + }; + const VkExtent2D processed_extent{ + .width = Settings::values.resolution_info.ScaleUp(raw_width), + .height = Settings::values.resolution_info.ScaleUp(raw_height), + }; + VkExtent2D source_extent = render_extent; if (auto* fxaa = std::get_if(&anti_alias)) { fxaa->Draw(device, scheduler, image_index, &source_image, &source_image_view); + source_extent = processed_extent; } else if (auto* smaa = std::get_if(&anti_alias)) { smaa->Draw(device, scheduler, image_index, &source_image, &source_image_view); + source_extent = processed_extent; } #ifdef HAS_RESHADE if (post_process.has_value()) { post_process->Draw(device, scheduler, image_index, &source_image, &source_image_view); + source_extent = processed_extent; } #endif auto crop_rect = Tegra::NormalizeCrop(framebuffer, texture_width, texture_height); - const VkExtent2D render_extent{ - .width = scaled_width, - .height = scaled_height, - }; + generation_source = source_image_view; + generation_extent = source_extent; + generation_render_extent = render_extent; + generation_crop = crop_rect; if (auto* fsr = std::get_if(&sr_filter)) { source_image_view = fsr->Draw(device, scheduler, image_index, source_image, source_image_view, render_extent, crop_rect); @@ -149,12 +170,43 @@ void Layer::ConfigureDraw(const Device& device, PresentPushConstants* out_push_c *out_descriptor_set = descriptor_sets[image_index]; } +void Layer::ConfigureGenerated(const Device& device, PresentPushConstants* out_push_constants, + VkDescriptorSet* out_descriptor_set, VkSampler sampler, + size_t generation, VkImage image, VkImageView view, + const Layout::FramebufferLayout& layout) { + const size_t slot = image_count + generation; + generation_ticks[generation] = scheduler.CurrentTick(); + + VkImageView source_image_view = view; + Common::Rectangle crop_rect = generation_crop; + if (auto* fsr = std::get_if(&sr_filter)) { + source_image_view = + fsr->Draw(device, scheduler, slot, image, view, generation_render_extent, crop_rect); + crop_rect = {0, 0, 1, 1}; + } else if (auto* sgsr = std::get_if(&sr_filter)) { + source_image_view = + sgsr->Draw(device, scheduler, slot, image, view, generation_render_extent, crop_rect); + crop_rect = {0, 0, 1, 1}; + } + + SetMatrixData(device, *out_push_constants, layout); + SetVertexData(device, *out_push_constants, layout, crop_rect); + + UpdateDescriptorSet(device, source_image_view, sampler, slot); + *out_descriptor_set = descriptor_sets[slot]; +} + +bool Layer::IsGenerationFree(size_t generation) const { + return generation < generation_count && scheduler.IsFree(generation_ticks[generation]); +} + void Layer::CreateDescriptorPool(const Device& device) { - descriptor_pool = CreateWrappedDescriptorPool(device, image_count, image_count); + const size_t slot_count = image_count + generation_count; + descriptor_pool = CreateWrappedDescriptorPool(device, slot_count, slot_count); } void Layer::CreateDescriptorSets(const Device& device, VkDescriptorSetLayout layout) { - const std::vector layouts(image_count, layout); + const std::vector layouts(image_count + generation_count, layout); descriptor_sets = CreateWrappedDescriptorSets(descriptor_pool, layouts); } @@ -343,7 +395,9 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i buffer.Flush(); // Ensure host writes are visible before the GPU copy. } - const VkBufferImageCopy copy{ + const VkBufferImageCopy2 copy{ + .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2, + .pNext = nullptr, .bufferOffset = image_offset, .bufferRowLength = 0, .bufferImageHeight = 0, @@ -364,11 +418,13 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i }; scheduler.Record([this, copy, index = image_index](vk::CommandBuffer cmdbuf) { const VkImage image = *raw_images[index]; - const VkImageMemoryBarrier base_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 base_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = 0, - .dstAccessMask = 0, + .srcStageMask = VK_PIPELINE_STAGE_2_NONE, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstStageMask = VK_PIPELINE_STAGE_2_NONE, + .dstAccessMask = VK_ACCESS_2_NONE, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -382,23 +438,24 @@ void Layer::UpdateRawImage(const Tegra::FramebufferConfig& framebuffer, size_t i .layerCount = 1, }, }; - VkImageMemoryBarrier read_barrier = base_barrier; - read_barrier.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + VkImageMemoryBarrier2 read_barrier = base_barrier; + read_barrier.srcStageMask = VK_PIPELINE_STAGE_2_HOST_BIT; + read_barrier.dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + read_barrier.dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT; read_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; read_barrier.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - VkImageMemoryBarrier write_barrier = base_barrier; - write_barrier.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; - write_barrier.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + VkImageMemoryBarrier2 write_barrier = base_barrier; + write_barrier.srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + write_barrier.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT; + write_barrier.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT; + write_barrier.dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT; write_barrier.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, - read_barrier); + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.CopyBufferToImage(*buffer, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, write_barrier); + cmdbuf.PipelineBarrier(write_barrier); }); } diff --git a/src/video_core/renderer_vulkan/present/layer.h b/src/video_core/renderer_vulkan/present/layer.h index f290ba5bb4..422eca6d2f 100644 --- a/src/video_core/renderer_vulkan/present/layer.h +++ b/src/video_core/renderer_vulkan/present/layer.h @@ -51,7 +51,7 @@ class Layer final { public: explicit Layer(const Device& device, MemoryAllocator& memory_allocator, Scheduler& scheduler, Tegra::MaxwellDeviceMemoryManager& device_memory, size_t image_count, - VkExtent2D output_size, VkDescriptorSetLayout layout, + size_t generation_count, VkExtent2D output_size, VkDescriptorSetLayout layout, const PresentFilters& filters); ~Layer(); @@ -61,6 +61,21 @@ public: const Tegra::FramebufferConfig& framebuffer, const Layout::FramebufferLayout& layout); + void ConfigureGenerated(const Device& device, PresentPushConstants* out_push_constants, + VkDescriptorSet* out_descriptor_set, VkSampler sampler, + size_t generation, VkImage image, VkImageView view, + const Layout::FramebufferLayout& layout); + + [[nodiscard]] bool IsGenerationFree(size_t generation) const; + + [[nodiscard]] VkImageView GenerationSource() const { + return generation_source; + } + + [[nodiscard]] VkExtent2D GenerationExtent() const { + return generation_extent; + } + private: void CreateDescriptorPool(const Device& device); void CreateDescriptorSets(const Device& device, VkDescriptorSetLayout layout); @@ -88,6 +103,7 @@ private: Tegra::MaxwellDeviceMemoryManager& device_memory; const PresentFilters& filters; const size_t image_count{}; + const size_t generation_count{}; vk::DescriptorPool descriptor_pool{}; vk::DescriptorSets descriptor_sets{}; @@ -108,6 +124,11 @@ private: VkExtent2D post_process_extent{}; #endif std::vector resource_ticks{}; + std::vector generation_ticks{}; + VkImageView generation_source{}; + VkExtent2D generation_extent{}; + VkExtent2D generation_render_extent{}; + Common::Rectangle generation_crop{}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/lsfg_chain.cpp b/src/video_core/renderer_vulkan/present/lsfg_chain.cpp index 972b13a1c4..a7ecfc5d71 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_chain.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_chain.cpp @@ -15,9 +15,37 @@ namespace Vulkan { namespace { constexpr u32 FIXED_DESCRIPTOR_SETS = 64; -constexpr u32 DESCRIPTOR_SETS_PER_SLOT = 112; +constexpr u32 DESCRIPTOR_SETS_PER_SLOT = 100; constexpr size_t FIRST_DELTA_LEVEL = 4; +[[nodiscard]] constexpr size_t DeltaStartWave(size_t index) { + if (index == 0) { + return 0; + } + return (FIRST_DELTA_LEVEL + index) * LSFG_GAMMA_STAGES; +} + +constexpr size_t GENERATION_WAVES = + DeltaStartWave(LSFG_DELTA_INSTANCES - 1) + LSFG_DELTA_STAGES; + +template +void ForEachWaveStage(std::array& gamma, + std::array& delta, size_t wave, + Func&& func) { + for (size_t i = 0; i < gamma.size(); ++i) { + const size_t start = i * LSFG_GAMMA_STAGES; + if (wave >= start && wave - start < LSFG_GAMMA_STAGES) { + func(gamma[i], wave - start); + } + } + for (size_t i = 0; i < delta.size(); ++i) { + const size_t start = DeltaStartWave(i); + if (wave >= start && wave - start < LSFG_DELTA_STAGES) { + func(delta[i], wave - start); + } + } +} + } // Anonymous namespace LsfgChain::LsfgChain(const Device& device, MemoryAllocator& memory_allocator, @@ -47,7 +75,7 @@ LsfgChain::LsfgChain(const Device& device, MemoryAllocator& memory_allocator, const size_t level = LSFG_MIP_LEVELS - 1 - i; gamma[i] = LsfgGamma(device, memory_allocator, shaders, resources, descriptor_pool, alpha[level].Outputs(), - beta.Output(std::min(level, LSFG_BETA_OUTPUTS - 1)), + beta.Output((std::min)(level, LSFG_BETA_OUTPUTS - 1)), i == 0 ? nullptr : &gamma[i - 1].Output()); if (i < FIRST_DELTA_LEVEL) { @@ -88,16 +116,20 @@ void LsfgChain::DispatchShared(vk::CommandBuffer cmdbuf, u64 frame_count) { } void LsfgChain::DispatchGeneration(vk::CommandBuffer cmdbuf, u64 frame_count, - size_t generation_count, size_t generation, u32 target, - VkImage image, VkExtent2D extent) { + size_t generation_count, size_t generation, VkImage image, + VkExtent2D extent) { const size_t slot = LsfgGenerationSlot(generation_count, generation); - for (size_t i = 0; i < LSFG_MIP_LEVELS; ++i) { - gamma[i].Dispatch(cmdbuf, frame_count, slot); - if (i >= FIRST_DELTA_LEVEL) { - delta[i - FIRST_DELTA_LEVEL].Dispatch(cmdbuf, frame_count, slot); - } + for (size_t wave = 0; wave < GENERATION_WAVES; ++wave) { + LsfgBarriers barriers(cmdbuf); + ForEachWaveStage(gamma, delta, wave, [&barriers, frame_count](auto& pass, size_t stage) { + pass.PushBarriers(barriers, frame_count, stage); + }); + barriers.Build(); + ForEachWaveStage(gamma, delta, wave, [cmdbuf, frame_count, slot](auto& pass, size_t stage) { + pass.DispatchStage(cmdbuf, frame_count, slot, stage); + }); } - generate.Dispatch(cmdbuf, frame_count, slot, target, image, extent); + generate.Dispatch(cmdbuf, frame_count, slot, image, extent); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/lsfg_chain.h b/src/video_core/renderer_vulkan/present/lsfg_chain.h index c8879f74a9..4561d3a092 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_chain.h +++ b/src/video_core/renderer_vulkan/present/lsfg_chain.h @@ -35,11 +35,11 @@ public: void DispatchShared(vk::CommandBuffer cmdbuf, u64 frame_count); void DispatchGeneration(vk::CommandBuffer cmdbuf, u64 frame_count, size_t generation_count, - size_t generation, u32 target, VkImage image, VkExtent2D extent); + size_t generation, VkImage image, VkExtent2D extent); - void SetTarget(const Device& device, size_t generation_count, size_t generation, u32 target, + void SetTarget(const Device& device, size_t generation_count, size_t generation, VkImageView view) { - generate.SetTarget(device, LsfgGenerationSlot(generation_count, generation), target, view); + generate.SetTarget(device, LsfgGenerationSlot(generation_count, generation), view); } [[nodiscard]] LsfgImage& Input(u64 frame_count) { diff --git a/src/video_core/renderer_vulkan/present/lsfg_common.cpp b/src/video_core/renderer_vulkan/present/lsfg_common.cpp index d2df2755e5..0e34f782e7 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_common.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_common.cpp @@ -44,7 +44,8 @@ vk::Image CreateChainImage(MemoryAllocator& memory_allocator, VkExtent2D extent, .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | - VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, + VK_IMAGE_USAGE_STORAGE_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -67,12 +68,14 @@ vk::Buffer CreateUniformBuffer(MemoryAllocator& memory_allocator, VkDeviceSize s return memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::Upload); } -VkImageMemoryBarrier MakeBarrier(const LsfgImage& image, VkAccessFlags src_access, - VkAccessFlags dst_access) { - return VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, +VkImageMemoryBarrier2 MakeBarrier(const LsfgImage& image, VkAccessFlags2 src_access, + VkAccessFlags2 dst_access) { + return VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .srcAccessMask = src_access, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, .dstAccessMask = dst_access, .oldLayout = image.Layout(), .newLayout = VK_IMAGE_LAYOUT_GENERAL, @@ -93,24 +96,24 @@ VkImageMemoryBarrier MakeBarrier(const LsfgImage& image, VkAccessFlags src_acces LsfgImage::LsfgImage(const Device& device, MemoryAllocator& memory_allocator, VkExtent2D extent_, VkFormat format_) - : extent{std::max(1u, extent_.width), std::max(1u, extent_.height)}, format{format_} { + : extent{(std::max)(1u, extent_.width), (std::max)(1u, extent_.height)}, format{format_} { image = CreateChainImage(memory_allocator, extent, format); view = CreateWrappedImageView(device, image, format); } -LsfgBarriers& LsfgBarriers::Push(LsfgImage& image, VkAccessFlags src_access, - VkAccessFlags dst_access) { +LsfgBarriers& LsfgBarriers::Push(LsfgImage& image, VkAccessFlags2 src_access, + VkAccessFlags2 dst_access) { barriers.push_back(MakeBarrier(image, src_access, dst_access)); image.SetLayout(VK_IMAGE_LAYOUT_GENERAL); return *this; } LsfgBarriers& LsfgBarriers::WriteToRead(LsfgImage& image) { - return Push(image, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + return Push(image, VK_ACCESS_2_SHADER_WRITE_BIT, VK_ACCESS_2_SHADER_READ_BIT); } LsfgBarriers& LsfgBarriers::ReadToWrite(LsfgImage& image) { - return Push(image, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_SHADER_WRITE_BIT); + return Push(image, VK_ACCESS_2_SHADER_READ_BIT, VK_ACCESS_2_SHADER_WRITE_BIT); } LsfgBarriers& LsfgBarriers::WriteToRead(LsfgImage* image) { @@ -122,11 +125,13 @@ LsfgBarriers& LsfgBarriers::ReadToWrite(LsfgImage* image) { } LsfgBarriers& LsfgBarriers::DiscardToWrite(VkImage image) { - barriers.push_back(VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + barriers.push_back(VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -199,8 +204,7 @@ void LsfgBarriers::Build() { if (barriers.empty()) { return; } - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, {}, {}, barriers); + cmdbuf.PipelineBarrier(0, {}, {}, barriers); barriers.clear(); } @@ -283,7 +287,6 @@ LsfgPass::LsfgPass(const Device& device, const LsfgShaders& shaders, u32 shader_ for (const auto& [count, type] : bindings) { types.insert(types.end(), count, type); } - descriptor_count = static_cast(types.size()); descriptor_set_layout = CreateWrappedDescriptorSetLayout( device, std::span{types}, VK_SHADER_STAGE_COMPUTE_BIT); diff --git a/src/video_core/renderer_vulkan/present/lsfg_common.h b/src/video_core/renderer_vulkan/present/lsfg_common.h index d1e3271a6d..68e4d16f57 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_common.h +++ b/src/video_core/renderer_vulkan/present/lsfg_common.h @@ -27,7 +27,6 @@ constexpr VkFormat LSFG_FLOW_FORMAT = VK_FORMAT_R8_UNORM; constexpr VkFormat LSFG_MOTION_FORMAT = VK_FORMAT_R16G16B16A16_SFLOAT; constexpr size_t LSFG_HISTORY_SLOTS = 3; -constexpr size_t LSFG_MAX_TARGETS = 7; constexpr size_t LSFG_MAX_GENERATIONS = 3; constexpr size_t LSFG_GENERATION_SLOTS = LSFG_MAX_GENERATIONS * (LSFG_MAX_GENERATIONS + 1) / 2; @@ -147,10 +146,10 @@ public: void Build(); private: - LsfgBarriers& Push(LsfgImage& image, VkAccessFlags src_access, VkAccessFlags dst_access); + LsfgBarriers& Push(LsfgImage& image, VkAccessFlags2 src_access, VkAccessFlags2 dst_access); vk::CommandBuffer cmdbuf; - std::vector barriers; + std::vector barriers; }; class LsfgDescriptorWriter { @@ -204,10 +203,6 @@ public: return *descriptor_set_layout; } - [[nodiscard]] u32 DescriptorCount() const { - return descriptor_count; - } - void Bind(vk::CommandBuffer cmdbuf, VkDescriptorSet set) const; void BindPipeline(vk::CommandBuffer cmdbuf) const; void BindSet(vk::CommandBuffer cmdbuf, VkDescriptorSet set) const; @@ -216,7 +211,6 @@ private: vk::DescriptorSetLayout descriptor_set_layout; vk::PipelineLayout pipeline_layout; vk::Pipeline pipeline; - u32 descriptor_count{}; }; [[nodiscard]] vk::DescriptorPool CreateLsfgDescriptorPool(const Device& device, u32 max_sets); diff --git a/src/video_core/renderer_vulkan/present/lsfg_delta.cpp b/src/video_core/renderer_vulkan/present/lsfg_delta.cpp index dc6eec383f..9ced1f5a47 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_delta.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_delta.cpp @@ -196,88 +196,68 @@ LsfgDelta::LsfgDelta(const Device& device, MemoryAllocator& memory_allocator, } } -void LsfgDelta::Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot) { +void LsfgDelta::PushBarriers(LsfgBarriers& barriers, u64 frame_count, size_t stage) { + auto& history = (*inputs)[frame_count % LSFG_HISTORY_SLOTS]; + auto& previous_history = (*inputs)[(frame_count + 2) % LSFG_HISTORY_SLOTS]; + switch (stage) { + case 0: + barriers.WriteToReadAll(previous_history) + .WriteToReadAll(history) + .WriteToRead(previous_gamma) + .ReadToWriteAll(temp1); + break; + case 1: + case 3: + barriers.WriteToReadAll(temp1).ReadToWriteAll(temp2); + break; + case 2: + barriers.WriteToReadAll(temp2).ReadToWriteAll(temp1); + break; + case 4: + barriers.WriteToReadAll(temp2) + .WriteToRead(previous_gamma) + .WriteToRead(*flow_input) + .ReadToWrite(out_image1); + break; + case 5: + barriers.WriteToReadAll(previous_history) + .WriteToReadAll(history) + .WriteToRead(previous_gamma) + .WriteToRead(previous1) + .ReadToWriteAll(temp2); + break; + case 6: + case 8: + barriers.WriteToReadAll(temp2).ReadToWrite(temp1[0]).ReadToWrite(temp1[1]); + break; + case 7: + barriers.WriteToRead(temp1[0]).WriteToRead(temp1[1]).ReadToWriteAll(temp2); + break; + default: + barriers.WriteToRead(temp1[0]) + .WriteToRead(temp1[1]) + .WriteToRead(previous2) + .ReadToWrite(out_image2); + break; + } +} + +void LsfgDelta::DispatchStage(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, + size_t stage) const { const Generation& pass = generations[slot]; + const size_t history = frame_count % LSFG_HISTORY_SLOTS; + VkDescriptorSet set = pass.first_descriptor_sets[history]; + if (stage == 5) { + set = pass.sixth_descriptor_sets[history]; + } else if (stage > 5) { + set = pass.descriptor_sets[stage - 2]; + } else if (stage != 0) { + set = pass.descriptor_sets[stage - 1]; + } + passes[stage].Bind(cmdbuf, set); const VkExtent2D extent = temp1[0].Extent(); - const u32 groups_x = GroupCount(extent.width); - const u32 groups_y = GroupCount(extent.height); - - const size_t history = frame_count % LSFG_HISTORY_SLOTS; - const size_t previous_history = (frame_count + 2) % LSFG_HISTORY_SLOTS; - - LsfgBarriers(cmdbuf) - .WriteToReadAll((*inputs)[previous_history]) - .WriteToReadAll((*inputs)[history]) - .WriteToRead(previous_gamma) - .ReadToWriteAll(temp1) - .Build(); - passes[0].Bind(cmdbuf, pass.first_descriptor_sets[history]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf).WriteToReadAll(temp1).ReadToWriteAll(temp2).Build(); - passes[1].Bind(cmdbuf, pass.descriptor_sets[0]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf).WriteToReadAll(temp2).ReadToWriteAll(temp1).Build(); - passes[2].Bind(cmdbuf, pass.descriptor_sets[1]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf).WriteToReadAll(temp1).ReadToWriteAll(temp2).Build(); - passes[3].Bind(cmdbuf, pass.descriptor_sets[2]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToReadAll(temp2) - .WriteToRead(previous_gamma) - .WriteToRead(*flow_input) - .ReadToWrite(out_image1) - .Build(); - passes[4].Bind(cmdbuf, pass.descriptor_sets[3]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToReadAll((*inputs)[previous_history]) - .WriteToReadAll((*inputs)[history]) - .WriteToRead(previous_gamma) - .WriteToRead(previous1) - .ReadToWriteAll(temp2) - .Build(); - passes[5].Bind(cmdbuf, pass.sixth_descriptor_sets[history]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToReadAll(temp2) - .ReadToWrite(temp1[0]) - .ReadToWrite(temp1[1]) - .Build(); - passes[6].Bind(cmdbuf, pass.descriptor_sets[4]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToRead(temp1[0]) - .WriteToRead(temp1[1]) - .ReadToWriteAll(temp2) - .Build(); - passes[7].Bind(cmdbuf, pass.descriptor_sets[5]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToReadAll(temp2) - .ReadToWrite(temp1[0]) - .ReadToWrite(temp1[1]) - .Build(); - passes[8].Bind(cmdbuf, pass.descriptor_sets[6]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToRead(temp1[0]) - .WriteToRead(temp1[1]) - .WriteToRead(previous2) - .ReadToWrite(out_image2) - .Build(); - passes[9].Bind(cmdbuf, pass.descriptor_sets[7]); - cmdbuf.Dispatch(groups_x, groups_y, 1); + cmdbuf.Dispatch(GroupCount(extent.width), GroupCount(extent.height), 1); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/lsfg_delta.h b/src/video_core/renderer_vulkan/present/lsfg_delta.h index 1bbcfe6745..a3beda0b77 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_delta.h +++ b/src/video_core/renderer_vulkan/present/lsfg_delta.h @@ -27,7 +27,9 @@ public: LsfgImageHistory& inputs, LsfgImage& flow_input, LsfgImage* previous_gamma, LsfgImage* previous1, LsfgImage* previous2); - void Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot); + void PushBarriers(LsfgBarriers& barriers, u64 frame_count, size_t stage); + + void DispatchStage(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, size_t stage) const; [[nodiscard]] LsfgImage& Output1() { return out_image1; diff --git a/src/video_core/renderer_vulkan/present/lsfg_gamma.cpp b/src/video_core/renderer_vulkan/present/lsfg_gamma.cpp index 067d124b6f..fff0b57d0e 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_gamma.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_gamma.cpp @@ -133,53 +133,43 @@ LsfgGamma::LsfgGamma(const Device& device, MemoryAllocator& memory_allocator, } } -void LsfgGamma::Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot) { +void LsfgGamma::PushBarriers(LsfgBarriers& barriers, u64 frame_count, size_t stage) { + switch (stage) { + case 0: + barriers.WriteToReadAll((*inputs)[(frame_count + 2) % LSFG_HISTORY_SLOTS]) + .WriteToReadAll((*inputs)[frame_count % LSFG_HISTORY_SLOTS]) + .WriteToRead(previous) + .ReadToWriteAll(temp1); + break; + case 1: + barriers.WriteToReadAll(temp1).ReadToWriteAll(temp2); + break; + case 2: + barriers.WriteToReadAll(temp2).ReadToWrite(temp1[0]).ReadToWrite(temp1[1]); + break; + case 3: + barriers.WriteToRead(temp1[0]).WriteToRead(temp1[1]).ReadToWriteAll(temp2); + break; + default: + barriers.WriteToReadAll(temp2) + .WriteToRead(previous) + .WriteToRead(*flow_input) + .ReadToWrite(out_image); + break; + } +} + +void LsfgGamma::DispatchStage(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, + size_t stage) const { const Generation& pass = generations[slot]; + VkDescriptorSet set = pass.first_descriptor_sets[frame_count % LSFG_HISTORY_SLOTS]; + if (stage != 0) { + set = pass.descriptor_sets[stage - 1]; + } + passes[stage].Bind(cmdbuf, set); const VkExtent2D extent = temp1[0].Extent(); - const u32 groups_x = GroupCount(extent.width); - const u32 groups_y = GroupCount(extent.height); - - const size_t history = frame_count % LSFG_HISTORY_SLOTS; - const size_t previous_history = (frame_count + 2) % LSFG_HISTORY_SLOTS; - - LsfgBarriers(cmdbuf) - .WriteToReadAll((*inputs)[previous_history]) - .WriteToReadAll((*inputs)[history]) - .WriteToRead(previous) - .ReadToWriteAll(temp1) - .Build(); - passes[0].Bind(cmdbuf, pass.first_descriptor_sets[history]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf).WriteToReadAll(temp1).ReadToWriteAll(temp2).Build(); - passes[1].Bind(cmdbuf, pass.descriptor_sets[0]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToReadAll(temp2) - .ReadToWrite(temp1[0]) - .ReadToWrite(temp1[1]) - .Build(); - passes[2].Bind(cmdbuf, pass.descriptor_sets[1]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToRead(temp1[0]) - .WriteToRead(temp1[1]) - .ReadToWriteAll(temp2) - .Build(); - passes[3].Bind(cmdbuf, pass.descriptor_sets[2]); - cmdbuf.Dispatch(groups_x, groups_y, 1); - - LsfgBarriers(cmdbuf) - .WriteToReadAll(temp2) - .WriteToRead(previous) - .WriteToRead(*flow_input) - .ReadToWrite(out_image) - .Build(); - passes[4].Bind(cmdbuf, pass.descriptor_sets[3]); - cmdbuf.Dispatch(groups_x, groups_y, 1); + cmdbuf.Dispatch(GroupCount(extent.width), GroupCount(extent.height), 1); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/lsfg_gamma.h b/src/video_core/renderer_vulkan/present/lsfg_gamma.h index 848c380bfe..cda230d449 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_gamma.h +++ b/src/video_core/renderer_vulkan/present/lsfg_gamma.h @@ -26,7 +26,9 @@ public: LsfgResources& resources, vk::DescriptorPool& descriptor_pool, LsfgImageHistory& inputs, LsfgImage& flow_input, LsfgImage* previous); - void Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot); + void PushBarriers(LsfgBarriers& barriers, u64 frame_count, size_t stage); + + void DispatchStage(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, size_t stage) const; [[nodiscard]] LsfgImage& Output() { return out_image; diff --git a/src/video_core/renderer_vulkan/present/lsfg_generate.cpp b/src/video_core/renderer_vulkan/present/lsfg_generate.cpp index 4473b59a2f..f26c27c92c 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_generate.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_generate.cpp @@ -22,12 +22,15 @@ constexpr u32 DISPATCH_TILE_SHIFT = 4; return (size + (1u << DISPATCH_TILE_SHIFT) - 1) >> DISPATCH_TILE_SHIFT; } -VkImageMemoryBarrier MakeTargetBarrier(VkImage image, VkAccessFlags src_access, - VkAccessFlags dst_access, VkImageLayout old_layout) { - return VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, +VkImageMemoryBarrier2 MakeTargetBarrier(VkImage image, VkPipelineStageFlags2 src_stage, + VkAccessFlags2 src_access, VkPipelineStageFlags2 dst_stage, + VkAccessFlags2 dst_access, VkImageLayout old_layout) { + return VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = src_stage, .srcAccessMask = src_access, + .dstStageMask = dst_stage, .dstAccessMask = dst_access, .oldLayout = old_layout, .newLayout = VK_IMAGE_LAYOUT_GENERAL, @@ -63,25 +66,21 @@ LsfgGenerate::LsfgGenerate(const Device& device, const LsfgShaders& shaders, edge_sampler = resources.GetSampler(VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE, VK_COMPARE_OP_ALWAYS, false); - const std::vector layouts( - LSFG_GENERATION_SLOTS * LSFG_MAX_TARGETS * 2, pass.SetLayout()); + const std::vector layouts(LSFG_GENERATION_SLOTS * 2, pass.SetLayout()); owned_sets = CreateWrappedDescriptorSets(descriptor_pool, layouts); size_t next = 0; for (size_t slot = 0; slot < LSFG_GENERATION_SLOTS; ++slot) { - Generation& target = generations[slot]; - target.buffer = resources.GetBuffer(LsfgSlotTimestamp(slot)); - - for (auto& entry : target.targets) { - for (auto& set : entry.descriptor_sets) { - set = owned_sets[next++]; - } + Generation& entry = generations[slot]; + entry.buffer = resources.GetBuffer(LsfgSlotTimestamp(slot)); + for (auto& set : entry.descriptor_sets) { + set = owned_sets[next++]; } } } -void LsfgGenerate::SetTarget(const Device& device, size_t slot, u32 target, VkImageView view) { - Target& entry = generations[slot].targets[target]; +void LsfgGenerate::SetTarget(const Device& device, size_t slot, VkImageView view) { + Generation& entry = generations[slot]; if (entry.view == view) { return; } @@ -89,7 +88,7 @@ void LsfgGenerate::SetTarget(const Device& device, size_t slot, u32 target, VkIm for (size_t i = 0; i < entry.descriptor_sets.size(); ++i) { LsfgDescriptorWriter(entry.descriptor_sets[i]) - .AddUniformBuffer(generations[slot].buffer, LsfgResources::BufferSize()) + .AddUniformBuffer(entry.buffer, LsfgResources::BufferSize()) .AddSampler(sampler) .AddSampler(edge_sampler) .AddSampledImage((*frames)[1 - i]) @@ -102,9 +101,9 @@ void LsfgGenerate::SetTarget(const Device& device, size_t slot, u32 target, VkIm } } -void LsfgGenerate::Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, u32 target, +void LsfgGenerate::Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, VkImage image, VkExtent2D extent) { - const Target& entry = generations[slot].targets[target]; + const Generation& entry = generations[slot]; LsfgBarriers(cmdbuf) .WriteToReadAll(*frames) @@ -117,14 +116,11 @@ void LsfgGenerate::Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t sl pass.Bind(cmdbuf, entry.descriptor_sets[frame_count % entry.descriptor_sets.size()]); cmdbuf.Dispatch(GroupCount(extent.width), GroupCount(extent.height), 1); - const std::array after{MakeTargetBarrier( - image, VK_ACCESS_SHADER_WRITE_BIT, - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT, - VK_IMAGE_LAYOUT_GENERAL)}; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, {}, {}, after); + const std::array after{MakeTargetBarrier(image, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + VK_ACCESS_2_SHADER_WRITE_BIT, + vk::PIPELINE_STAGE_IMAGE_USERS, + vk::ACCESS_IMAGE_USERS, VK_IMAGE_LAYOUT_GENERAL)}; + cmdbuf.PipelineBarrier(0, {}, {}, after); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/lsfg_generate.h b/src/video_core/renderer_vulkan/present/lsfg_generate.h index 75cbd8d2e4..fedad97bdd 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_generate.h +++ b/src/video_core/renderer_vulkan/present/lsfg_generate.h @@ -23,19 +23,15 @@ public: vk::DescriptorPool& descriptor_pool, LsfgImagePair& frames, LsfgImage& motion, LsfgImage& detail1, LsfgImage& detail2); - void SetTarget(const Device& device, size_t slot, u32 target, VkImageView view); + void SetTarget(const Device& device, size_t slot, VkImageView view); - void Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, u32 target, - VkImage image, VkExtent2D extent); + void Dispatch(vk::CommandBuffer cmdbuf, u64 frame_count, size_t slot, VkImage image, + VkExtent2D extent); private: - struct Target { + struct Generation { std::array descriptor_sets{}; VkImageView view{}; - }; - - struct Generation { - std::array targets{}; VkBuffer buffer{}; }; diff --git a/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp b/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp index 3f31f58d68..c5cd1c7c86 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_mipmaps.cpp @@ -40,8 +40,10 @@ LsfgMipmaps::LsfgMipmaps(const Device& device, MemoryAllocator& memory_allocator const VkExtent2D input_extent = (*frames)[0].Extent(); flow_extent = VkExtent2D{ - .width = std::max(1u, static_cast(static_cast(input_extent.width) * flow_scale)), - .height = std::max(1u, static_cast(static_cast(input_extent.height) * flow_scale)), + .width = (std::max)(1u, + static_cast(static_cast(input_extent.width) * flow_scale)), + .height = (std::max)(1u, + static_cast(static_cast(input_extent.height) * flow_scale)), }; for (size_t i = 0; i < LSFG_MIP_LEVELS; ++i) { diff --git a/src/video_core/renderer_vulkan/present/lsfg_shaders.cpp b/src/video_core/renderer_vulkan/present/lsfg_shaders.cpp index 51c7e4c106..9aebbc3cd9 100644 --- a/src/video_core/renderer_vulkan/present/lsfg_shaders.cpp +++ b/src/video_core/renderer_vulkan/present/lsfg_shaders.cpp @@ -10,7 +10,7 @@ namespace Vulkan { LsfgShaders::LsfgShaders(const Device& device) { if (!device.IsVulkanMemoryModelSupported() || !device.HasNullDescriptor() || - !device.IsFloat16Supported()) { + !device.IsNativeFloat16Enabled()) { return; } diff --git a/src/video_core/renderer_vulkan/present/post_process.cpp b/src/video_core/renderer_vulkan/present/post_process.cpp index 07d75b7025..7b5b81d7b5 100644 --- a/src/video_core/renderer_vulkan/present/post_process.cpp +++ b/src/video_core/renderer_vulkan/present/post_process.cpp @@ -126,58 +126,7 @@ VkPrimitiveTopology ToTopology(reshadefx::primitive_topology topology) { } } -vk::RenderPass CreateFxRenderPass(const Device& device, VkFormat format, bool clear) { - VkAttachmentLoadOp load_op = VK_ATTACHMENT_LOAD_OP_LOAD; - VkImageLayout initial_layout = VK_IMAGE_LAYOUT_GENERAL; - if (clear) { - load_op = VK_ATTACHMENT_LOAD_OP_CLEAR; - initial_layout = VK_IMAGE_LAYOUT_UNDEFINED; - } - - const VkAttachmentDescription attachment{ - .flags = 0, - .format = format, - .samples = VK_SAMPLE_COUNT_1_BIT, - .loadOp = load_op, - .storeOp = VK_ATTACHMENT_STORE_OP_STORE, - .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE, - .stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE, - .initialLayout = initial_layout, - .finalLayout = VK_IMAGE_LAYOUT_GENERAL, - }; - - const VkAttachmentReference reference{ - .attachment = 0, - .layout = VK_IMAGE_LAYOUT_GENERAL, - }; - - const VkSubpassDescription subpass{ - .flags = 0, - .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, - .inputAttachmentCount = 0, - .pInputAttachments = nullptr, - .colorAttachmentCount = 1, - .pColorAttachments = &reference, - .pResolveAttachments = nullptr, - .pDepthStencilAttachment = nullptr, - .preserveAttachmentCount = 0, - .pPreserveAttachments = nullptr, - }; - - return device.GetLogical().CreateRenderPass(VkRenderPassCreateInfo{ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .attachmentCount = 1, - .pAttachments = &attachment, - .subpassCount = 1, - .pSubpasses = &subpass, - .dependencyCount = 0, - .pDependencies = nullptr, - }); -} - -vk::Pipeline CreateFxPipeline(const Device& device, vk::RenderPass& renderpass, +vk::Pipeline CreateFxPipeline(const Device& device, VkFormat format, vk::PipelineLayout& layout, VkShaderModule vertex_shader, VkShaderModule fragment_shader, const reshadefx::pass& pass) { @@ -298,9 +247,10 @@ vk::Pipeline CreateFxPipeline(const Device& device, vk::RenderPass& renderpass, .pDynamicStates = dynamic_states.data(), }; + const VkPipelineRenderingCreateInfo rendering_ci = ColorRenderingInfo(format); return device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(stages.size()), .pStages = stages.data(), @@ -314,7 +264,7 @@ vk::Pipeline CreateFxPipeline(const Device& device, vk::RenderPass& renderpass, .pColorBlendState = &color_blend, .pDynamicState = &dynamic_state, .layout = *layout, - .renderPass = *renderpass, + .renderPass = VK_NULL_HANDLE, .subpass = 0, .basePipelineHandle = nullptr, .basePipelineIndex = 0, @@ -530,7 +480,9 @@ bool PostProcessChain::BuildEffects(const Device& device, MemoryAllocator& alloc for (const auto& pass : technique->passes) { Pass out; out.num_vertices = pass.num_vertices; - out.clear = pass.clear_render_targets != 0; + if (pass.clear_render_targets != 0) { + out.load_op = VK_ATTACHMENT_LOAD_OP_CLEAR; + } out.target_texture = NO_TEXTURE; out.extent = m_extent; @@ -596,25 +548,9 @@ bool PostProcessChain::BuildEffects(const Device& device, MemoryAllocator& alloc continue; } - out.renderpass = CreateFxRenderPass(device, target_format, out.clear); - out.pipeline = CreateFxPipeline(device, out.renderpass, out.pipeline_layout, + out.pipeline = CreateFxPipeline(device, target_format, out.pipeline_layout, *vertex_shader->second, *fragment_shader->second, pass); - if (out.writes_backbuffer) { - out.backbuffer_slot = static_cast(effect.backbuffer_pass_count % 2); - ++effect.backbuffer_pass_count; - for (u32 image = 0; image < m_image_count; ++image) { - for (size_t slot = 0; slot < 2; ++slot) { - out.framebuffers.push_back(CreateWrappedFramebuffer( - device, out.renderpass, m_frames[image].views[slot], out.extent)); - } - } - } else { - out.framebuffers.push_back( - CreateWrappedFramebuffer(device, out.renderpass, - effect.textures[out.target_texture].view, out.extent)); - } - effect.passes.push_back(std::move(out)); } @@ -851,19 +787,20 @@ void PostProcessChain::Draw(const Device& device, Scheduler& scheduler, size_t i UpdateDescriptors(device, effect, pass, image_index, current_view); - VkFramebuffer framebuffer{}; VkImage target_image{}; + VkImageView target_view{}; if (pass.writes_backbuffer) { const u32 target_slot = (slot + 1) % 2; - framebuffer = *pass.framebuffers[image_index * 2 + target_slot]; target_image = *frame.images[target_slot]; + target_view = *frame.views[target_slot]; } else { - framebuffer = *pass.framebuffers[0]; - target_image = *effect.textures[pass.target_texture].image; + const Texture& texture = effect.textures[pass.target_texture]; + target_image = *texture.image; + target_view = *texture.view; } const VkImage source_image = current_image; - const VkRenderPass renderpass = *pass.renderpass; + const VkAttachmentLoadOp load_op = pass.load_op; const VkPipeline pipeline = *pass.pipeline; const VkPipelineLayout layout = *pass.pipeline_layout; const VkDescriptorSet uniform_set = effect.uniform_sets[image_index]; @@ -875,12 +812,12 @@ void PostProcessChain::Draw(const Device& device, Scheduler& scheduler, size_t i scheduler.Record([=](vk::CommandBuffer cmdbuf) { TransitionImageLayout(cmdbuf, source_image, VK_IMAGE_LAYOUT_GENERAL); TransitionImageLayout(cmdbuf, target_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, renderpass, framebuffer, extent); + BeginRendering(cmdbuf, target_view, extent, load_op); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, std::array{uniform_set, sampler_set}, {}); cmdbuf.Draw(vertices, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, target_image, VK_IMAGE_LAYOUT_GENERAL); }); diff --git a/src/video_core/renderer_vulkan/present/post_process.h b/src/video_core/renderer_vulkan/present/post_process.h index 9d42bd4751..cc65cfe172 100644 --- a/src/video_core/renderer_vulkan/present/post_process.h +++ b/src/video_core/renderer_vulkan/present/post_process.h @@ -77,19 +77,16 @@ private: }; struct Pass { - vk::RenderPass renderpass{}; vk::Pipeline pipeline{}; vk::DescriptorSetLayout sampler_layout{}; vk::PipelineLayout pipeline_layout{}; vk::DescriptorSets sampler_sets{}; std::vector sampler_bindings{}; - std::vector framebuffers{}; size_t target_texture{}; VkExtent2D extent{}; u32 num_vertices{3}; - bool clear{}; + VkAttachmentLoadOp load_op{VK_ATTACHMENT_LOAD_OP_LOAD}; bool writes_backbuffer{}; - u32 backbuffer_slot{}; }; struct Effect { @@ -105,8 +102,6 @@ private: vk::DescriptorSetLayout uniform_layout{}; vk::DescriptorPool descriptor_pool{}; vk::DescriptorSets uniform_sets{}; - size_t backbuffer_pass_count{}; - u32 backbuffer_slots{1}; }; struct FrameImages { diff --git a/src/video_core/renderer_vulkan/present/sgsr.cpp b/src/video_core/renderer_vulkan/present/sgsr.cpp index fa37225e43..1787b43e80 100644 --- a/src/video_core/renderer_vulkan/present/sgsr.cpp +++ b/src/video_core/renderer_vulkan/present/sgsr.cpp @@ -27,16 +27,8 @@ SGSR::SGSR(const Device& device, MemoryAllocator& memory_allocator, size_t image , m_extent{extent} , m_edge_dir{edge_dir} { - // Not finished yet initializing at ctor time? - m_dynamic_images.resize(m_image_count); - for (auto& images : m_dynamic_images) { - images.image = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - images.image_view = CreateWrappedImageView(device, images.image, VK_FORMAT_R16G16B16A16_SFLOAT); - } - - m_renderpass = CreateWrappedRenderPass(device, VK_FORMAT_R16G16B16A16_SFLOAT); - for (auto& images : m_dynamic_images) - images.framebuffer = CreateWrappedFramebuffer(device, m_renderpass, images.image_view, m_extent); + m_image = CreateWrappedImage(m_memory_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); + m_image_view = CreateWrappedImageView(device, m_image, VK_FORMAT_R16G16B16A16_SFLOAT); m_sampler = CreateBilinearSampler(device); m_vert_shader = BuildShader(device, SGSR1_SHADER_VERT_SPV); @@ -48,8 +40,8 @@ SGSR::SGSR(const Device& device, MemoryAllocator& memory_allocator, size_t image m_descriptor_set_layout = CreateWrappedDescriptorSetLayout(device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER}); VkDescriptorSetLayout layout = *m_descriptor_set_layout; - for (auto& images : m_dynamic_images) - images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, layout); + for (size_t i = 0; i < m_image_count; ++i) + m_descriptor_sets.push_back(CreateWrappedDescriptorSets(m_descriptor_pool, layout)); const VkPushConstantRange range{ .stageFlags = VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, @@ -66,38 +58,26 @@ SGSR::SGSR(const Device& device, MemoryAllocator& memory_allocator, size_t image .pPushConstantRanges = &range, }; m_pipeline_layout = device.GetLogical().CreatePipelineLayout(ci); - m_stage_pipeline = CreateWrappedPipeline(device, m_renderpass, m_pipeline_layout, std::tie(m_vert_shader, m_stage_shader)); + m_stage_pipeline = CreateWrappedPipeline(device, VK_FORMAT_R16G16B16A16_SFLOAT, + m_pipeline_layout, + std::tie(m_vert_shader, m_stage_shader)); } void SGSR::UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index) { - Images& images = m_dynamic_images[image_index]; std::vector image_infos; std::vector updates; image_infos.reserve(1); - updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, images.descriptor_sets[0], 0)); + updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, m_descriptor_sets[image_index][0], 0)); device.GetLogical().UpdateDescriptorSets(updates, {}); } -void SGSR::UploadImages(const Device& device, Scheduler& scheduler) { - if (!m_images_ready) { - scheduler.Record([&](vk::CommandBuffer cmdbuf) { - for (auto& image : m_dynamic_images) - ClearColorImage(cmdbuf, *image.image); - }); - scheduler.Finish(); - m_images_ready = true; - } -} - VkImageView SGSR::Draw(const Device& device, Scheduler& scheduler, size_t image_index, VkImage source_image, VkImageView source_image_view, VkExtent2D input_image_extent, const Common::Rectangle& crop_rect) { - Images& images = m_dynamic_images[image_index]; - auto const output_image = *images.image; - auto const descriptor_set = images.descriptor_sets[0]; - auto const framebuffer = *images.framebuffer; + auto const output_image = *m_image; + auto const output_view = *m_image_view; + auto const descriptor_set = m_descriptor_sets[image_index][0]; auto const pipeline = *m_stage_pipeline; VkPipelineLayout layout = *m_pipeline_layout; - VkRenderPass renderpass = *m_renderpass; VkExtent2D extent = m_extent; const f32 input_image_width = f32(input_image_extent.width); @@ -125,22 +105,22 @@ VkImageView SGSR::Draw(const Device& device, Scheduler& scheduler, size_t image_ viewport_con[7] = std::bit_cast((std::min)(crop_rect.top, crop_rect.bottom)); viewport_con[8] = std::bit_cast(edge_sharpness); - UploadImages(device, scheduler); UpdateDescriptorSets(device, source_image_view, image_index); scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([=](vk::CommandBuffer cmdbuf) { TransitionImageLayout(cmdbuf, source_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, renderpass, framebuffer, extent); + TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, output_view, extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, descriptor_set, {}); cmdbuf.PushConstants(layout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, viewport_con); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL); }); - return *images.image_view; + return output_view; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/sgsr.h b/src/video_core/renderer_vulkan/present/sgsr.h index 42ba00898f..52a645b09f 100644 --- a/src/video_core/renderer_vulkan/present/sgsr.h +++ b/src/video_core/renderer_vulkan/present/sgsr.h @@ -19,7 +19,6 @@ public: VkImageView Draw(const Device& device, Scheduler& scheduler, size_t image_index, VkImage source_image, VkImageView source_image_view, VkExtent2D input_image_extent, const Common::Rectangle& crop_rect); private: void Initialize(const Device& device); - void UploadImages(const Device& device, Scheduler& scheduler); void UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index); MemoryAllocator& m_memory_allocator; @@ -32,17 +31,11 @@ private: vk::ShaderModule m_vert_shader; vk::ShaderModule m_stage_shader; vk::Pipeline m_stage_pipeline; - vk::RenderPass m_renderpass; vk::Sampler m_sampler; - struct Images { - vk::DescriptorSets descriptor_sets; - vk::Image image; - vk::ImageView image_view; - vk::Framebuffer framebuffer; - }; - std::vector m_dynamic_images; - bool m_images_ready{}; + vk::Image m_image; + vk::ImageView m_image_view; + std::vector m_descriptor_sets; bool m_edge_dir{}; }; diff --git a/src/video_core/renderer_vulkan/present/smaa.cpp b/src/video_core/renderer_vulkan/present/smaa.cpp index 28b0427fa0..e4258d4b5c 100644 --- a/src/video_core/renderer_vulkan/present/smaa.cpp +++ b/src/video_core/renderer_vulkan/present/smaa.cpp @@ -32,7 +32,6 @@ SMAA::SMAA(const Device& device, MemoryAllocator& allocator, size_t image_count, , m_image_count(u32(image_count)) { CreateImages(device); - CreateRenderPasses(device); CreateSampler(device); CreateShaders(device); CreateDescriptorPool(device); @@ -56,42 +55,18 @@ void SMAA::CreateImages(const Device& device) { m_static_image_views[Search] = CreateWrappedImageView(device, m_static_images[Search], VK_FORMAT_R8_UNORM); - for (u32 i = 0; i < m_image_count; i++) { - Images& images = m_dynamic_images.emplace_back(); + m_dynamic_images[Blend] = + CreateWrappedImage(m_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); + m_dynamic_images[Edges] = CreateWrappedImage(m_allocator, m_extent, VK_FORMAT_R16G16_SFLOAT); + m_dynamic_images[Output] = + CreateWrappedImage(m_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - images.images[Blend] = - CreateWrappedImage(m_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - images.images[Edges] = CreateWrappedImage(m_allocator, m_extent, VK_FORMAT_R16G16_SFLOAT); - images.images[Output] = - CreateWrappedImage(m_allocator, m_extent, VK_FORMAT_R16G16B16A16_SFLOAT); - - images.image_views[Blend] = - CreateWrappedImageView(device, images.images[Blend], VK_FORMAT_R16G16B16A16_SFLOAT); - images.image_views[Edges] = - CreateWrappedImageView(device, images.images[Edges], VK_FORMAT_R16G16_SFLOAT); - images.image_views[Output] = - CreateWrappedImageView(device, images.images[Output], VK_FORMAT_R16G16B16A16_SFLOAT); - } -} - -void SMAA::CreateRenderPasses(const Device& device) { - m_renderpasses[EdgeDetection] = CreateWrappedRenderPass(device, VK_FORMAT_R16G16_SFLOAT); - m_renderpasses[BlendingWeightCalculation] = - CreateWrappedRenderPass(device, VK_FORMAT_R16G16B16A16_SFLOAT); - m_renderpasses[NeighborhoodBlending] = - CreateWrappedRenderPass(device, VK_FORMAT_R16G16B16A16_SFLOAT); - - for (auto& images : m_dynamic_images) { - images.framebuffers[EdgeDetection] = CreateWrappedFramebuffer( - device, m_renderpasses[EdgeDetection], images.image_views[Edges], m_extent); - - images.framebuffers[BlendingWeightCalculation] = - CreateWrappedFramebuffer(device, m_renderpasses[BlendingWeightCalculation], - images.image_views[Blend], m_extent); - - images.framebuffers[NeighborhoodBlending] = CreateWrappedFramebuffer( - device, m_renderpasses[NeighborhoodBlending], images.image_views[Output], m_extent); - } + m_dynamic_image_views[Blend] = + CreateWrappedImageView(device, m_dynamic_images[Blend], VK_FORMAT_R16G16B16A16_SFLOAT); + m_dynamic_image_views[Edges] = + CreateWrappedImageView(device, m_dynamic_images[Edges], VK_FORMAT_R16G16_SFLOAT); + m_dynamic_image_views[Output] = + CreateWrappedImageView(device, m_dynamic_images[Output], VK_FORMAT_R16G16B16A16_SFLOAT); } void SMAA::CreateSampler(const Device& device) { @@ -143,8 +118,8 @@ void SMAA::CreateDescriptorSets(const Device& device) { std::ranges::transform(m_descriptor_set_layouts, layouts.begin(), [](auto& layout) { return *layout; }); - for (auto& images : m_dynamic_images) { - images.descriptor_sets = CreateWrappedDescriptorSets(m_descriptor_pool, layouts); + for (u32 i = 0; i < m_image_count; i++) { + m_descriptor_sets.push_back(CreateWrappedDescriptorSets(m_descriptor_pool, layouts)); } } @@ -155,36 +130,43 @@ void SMAA::CreatePipelineLayouts(const Device& device) { } void SMAA::CreatePipelines(const Device& device) { + static constexpr std::array formats{ + VK_FORMAT_R16G16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + VK_FORMAT_R16G16B16A16_SFLOAT, + }; for (size_t i = 0; i < MaxSMAAStage; i++) { m_pipelines[i] = - CreateWrappedPipeline(device, m_renderpasses[i], m_pipeline_layouts[i], + CreateWrappedPipeline(device, formats[i], m_pipeline_layouts[i], std::tie(m_vertex_shaders[i], m_fragment_shaders[i])); } } void SMAA::UpdateDescriptorSets(const Device& device, VkImageView image_view, size_t image_index) { - Images& images = m_dynamic_images[image_index]; + const vk::DescriptorSets& descriptor_sets = m_descriptor_sets[image_index]; std::vector image_infos; std::vector updates; image_infos.reserve(6); updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, - images.descriptor_sets[EdgeDetection], 0)); + descriptor_sets[EdgeDetection], 0)); - updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Edges], - images.descriptor_sets[BlendingWeightCalculation], + updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, + *m_dynamic_image_views[Edges], + descriptor_sets[BlendingWeightCalculation], 0)); updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, *m_static_image_views[Area], - images.descriptor_sets[BlendingWeightCalculation], + descriptor_sets[BlendingWeightCalculation], 1)); updates.push_back( CreateWriteDescriptorSet(image_infos, *m_sampler, *m_static_image_views[Search], - images.descriptor_sets[BlendingWeightCalculation], 2)); + descriptor_sets[BlendingWeightCalculation], 2)); updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, image_view, - images.descriptor_sets[NeighborhoodBlending], 0)); - updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, *images.image_views[Blend], - images.descriptor_sets[NeighborhoodBlending], 1)); + descriptor_sets[NeighborhoodBlending], 0)); + updates.push_back(CreateWriteDescriptorSet(image_infos, *m_sampler, + *m_dynamic_image_views[Blend], + descriptor_sets[NeighborhoodBlending], 1)); device.GetLogical().UpdateDescriptorSets(updates, {}); } @@ -202,36 +184,26 @@ void SMAA::UploadImages(const Device& device, Scheduler& scheduler) { UploadImage(device, m_allocator, scheduler, m_static_images[Search], search_extent, VK_FORMAT_R8_UNORM, ARRAY_TO_SPAN(searchTexBytes)); - scheduler.Record([&](vk::CommandBuffer cmdbuf) { - for (auto& images : m_dynamic_images) { - for (size_t i = 0; i < MaxDynamicImage; i++) { - ClearColorImage(cmdbuf, *images.images[i]); - } - } - }); - scheduler.Finish(); - m_images_ready = true; } void SMAA::Draw(const Device& device, Scheduler& scheduler, size_t image_index, VkImage* inout_image, VkImageView* inout_image_view) { - Images& images = m_dynamic_images[image_index]; + const vk::DescriptorSets& descriptor_sets = m_descriptor_sets[image_index]; VkImage input_image = *inout_image; - VkImage output_image = *images.images[Output]; - VkImage edges_image = *images.images[Edges]; - VkImage blend_image = *images.images[Blend]; + VkImage output_image = *m_dynamic_images[Output]; + VkImage edges_image = *m_dynamic_images[Edges]; + VkImage blend_image = *m_dynamic_images[Blend]; - VkDescriptorSet edge_detection_descriptor_set = images.descriptor_sets[EdgeDetection]; + VkDescriptorSet edge_detection_descriptor_set = descriptor_sets[EdgeDetection]; VkDescriptorSet blending_weight_calculation_descriptor_set = - images.descriptor_sets[BlendingWeightCalculation]; + descriptor_sets[BlendingWeightCalculation]; VkDescriptorSet neighborhood_blending_descriptor_set = - images.descriptor_sets[NeighborhoodBlending]; + descriptor_sets[NeighborhoodBlending]; - VkFramebuffer edge_detection_framebuffer = *images.framebuffers[EdgeDetection]; - VkFramebuffer blending_weight_calculation_framebuffer = - *images.framebuffers[BlendingWeightCalculation]; - VkFramebuffer neighborhood_blending_framebuffer = *images.framebuffers[NeighborhoodBlending]; + VkImageView edges_view = *m_dynamic_image_views[Edges]; + VkImageView blend_view = *m_dynamic_image_views[Blend]; + VkImageView output_view = *m_dynamic_image_views[Output]; UploadImages(device, scheduler); UpdateDescriptorSets(device, *inout_image_view, image_index); @@ -239,43 +211,43 @@ void SMAA::Draw(const Device& device, Scheduler& scheduler, size_t image_index, scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([=, this](vk::CommandBuffer cmdbuf) { TransitionImageLayout(cmdbuf, input_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, edges_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, *m_renderpasses[EdgeDetection], edge_detection_framebuffer, - m_extent); + TransitionImageLayout(cmdbuf, edges_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, edges_view, m_extent, VK_ATTACHMENT_LOAD_OP_CLEAR); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, *m_pipelines[EdgeDetection]); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, *m_pipeline_layouts[EdgeDetection], 0, edge_detection_descriptor_set, {}); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, edges_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, blend_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, *m_renderpasses[BlendingWeightCalculation], - blending_weight_calculation_framebuffer, m_extent); + TransitionImageLayout(cmdbuf, blend_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, blend_view, m_extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, *m_pipelines[BlendingWeightCalculation]); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, *m_pipeline_layouts[BlendingWeightCalculation], 0, blending_weight_calculation_descriptor_set, {}); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, blend_image, VK_IMAGE_LAYOUT_GENERAL); - TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL); - BeginRenderPass(cmdbuf, *m_renderpasses[NeighborhoodBlending], - neighborhood_blending_framebuffer, m_extent); + TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, output_view, m_extent, VK_ATTACHMENT_LOAD_OP_DONT_CARE); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, *m_pipelines[NeighborhoodBlending]); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_GRAPHICS, *m_pipeline_layouts[NeighborhoodBlending], 0, neighborhood_blending_descriptor_set, {}); cmdbuf.Draw(3, 1, 0, 0); - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); TransitionImageLayout(cmdbuf, output_image, VK_IMAGE_LAYOUT_GENERAL); }); - *inout_image = *images.images[Output]; - *inout_image_view = *images.image_views[Output]; + *inout_image = *m_dynamic_images[Output]; + *inout_image_view = *m_dynamic_image_views[Output]; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/smaa.h b/src/video_core/renderer_vulkan/present/smaa.h index a547161395..f9c87c9c6a 100644 --- a/src/video_core/renderer_vulkan/present/smaa.h +++ b/src/video_core/renderer_vulkan/present/smaa.h @@ -47,7 +47,6 @@ private: }; void CreateImages(const Device& device); - void CreateRenderPasses(const Device& device); void CreateSampler(const Device& device); void CreateShaders(const Device& device); void CreateDescriptorPool(const Device& device); @@ -68,18 +67,13 @@ private: std::array m_vertex_shaders{}; std::array m_fragment_shaders{}; std::array m_pipelines{}; - std::array m_renderpasses{}; std::array m_static_images{}; std::array m_static_image_views{}; - struct Images { - vk::DescriptorSets descriptor_sets{}; - std::array images{}; - std::array image_views{}; - std::array framebuffers{}; - }; - std::vector m_dynamic_images{}; + std::array m_dynamic_images{}; + std::array m_dynamic_image_views{}; + std::vector m_descriptor_sets{}; bool m_images_ready{}; vk::Sampler m_sampler{}; diff --git a/src/video_core/renderer_vulkan/present/util.cpp b/src/video_core/renderer_vulkan/present/util.cpp index 2ea7870f01..63e8e5f5e8 100644 --- a/src/video_core/renderer_vulkan/present/util.cpp +++ b/src/video_core/renderer_vulkan/present/util.cpp @@ -49,13 +49,13 @@ vk::Image CreateWrappedImage(MemoryAllocator& allocator, VkExtent2D dimensions, void TransitionImageLayout(vk::CommandBuffer& cmdbuf, VkImage image, VkImageLayout target_layout, VkImageLayout source_layout) { - constexpr VkFlags flags{VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - const VkImageMemoryBarrier barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = flags, - .dstAccessMask = flags, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = source_layout, .newLayout = target_layout, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -69,8 +69,7 @@ void TransitionImageLayout(vk::CommandBuffer& cmdbuf, VkImage image, VkImageLayo .layerCount = 1, }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, barrier); + cmdbuf.PipelineBarrier(barrier); } void UploadImage(const Device& device, MemoryAllocator& allocator, Scheduler& scheduler, @@ -90,7 +89,9 @@ void UploadImage(const Device& device, MemoryAllocator& allocator, Scheduler& sc std::ranges::copy(initial_contents, upload_buffer.Mapped().begin()); upload_buffer.Flush(); - const std::array regions{{{ + const VkBufferImageCopy2 region{ + .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2, + .pNext = nullptr, .bufferOffset = 0, .bufferRowLength = dimensions.width, .bufferImageHeight = dimensions.height, @@ -100,14 +101,14 @@ void UploadImage(const Device& device, MemoryAllocator& allocator, Scheduler& sc .layerCount = 1}, .imageOffset{}, .imageExtent{.width = dimensions.width, .height = dimensions.height, .depth = 1}, - }}}; + }; scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([&](vk::CommandBuffer cmdbuf) { TransitionImageLayout(cmdbuf, *image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_UNDEFINED); cmdbuf.CopyBufferToImage(*upload_buffer, *image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - regions); + region); TransitionImageLayout(cmdbuf, *image, VK_IMAGE_LAYOUT_GENERAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL); }); @@ -116,11 +117,13 @@ void UploadImage(const Device& device, MemoryAllocator& allocator, Scheduler& sc void DownloadColorImage(vk::CommandBuffer& cmdbuf, VkImage image, VkBuffer buffer, VkExtent3D extent) { - const VkImageMemoryBarrier read_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -134,11 +137,13 @@ void DownloadColorImage(vk::CommandBuffer& cmdbuf, VkImage image, VkBuffer buffe .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - const VkImageMemoryBarrier image_write_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 image_write_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -152,13 +157,17 @@ void DownloadColorImage(vk::CommandBuffer& cmdbuf, VkImage image, VkBuffer buffe .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - static constexpr VkMemoryBarrier memory_write_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 memory_write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_HOST, + .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT, }; - const VkBufferImageCopy copy{ + const VkBufferImageCopy2 copy{ + .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2, + .pNext = nullptr, .bufferOffset = 0, .bufferRowLength = 0, .bufferImageHeight = 0, @@ -171,11 +180,9 @@ void DownloadColorImage(vk::CommandBuffer& cmdbuf, VkImage image, VkBuffer buffe .imageOffset{.x = 0, .y = 0, .z = 0}, .imageExtent{extent}, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, - read_barrier); + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.CopyImageToBuffer(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, buffer, copy); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, - memory_write_barrier, nullptr, image_write_barrier); + cmdbuf.PipelineBarrier(0, memory_write_barrier, {}, image_write_barrier); } vk::ImageView CreateWrappedImageView(const Device& device, vk::Image& image, VkFormat format) { @@ -195,77 +202,6 @@ vk::ImageView CreateWrappedImageView(const Device& device, vk::Image& image, VkF }); } -vk::RenderPass CreateWrappedRenderPass(const Device& device, VkFormat format, - VkImageLayout initial_layout) { - const VkAttachmentDescription attachment{ - .flags = VK_ATTACHMENT_DESCRIPTION_MAY_ALIAS_BIT, - .format = format, - .samples = VK_SAMPLE_COUNT_1_BIT, - .loadOp = initial_layout == VK_IMAGE_LAYOUT_UNDEFINED ? VK_ATTACHMENT_LOAD_OP_DONT_CARE - : VK_ATTACHMENT_LOAD_OP_LOAD, - .storeOp = VK_ATTACHMENT_STORE_OP_STORE, - .stencilLoadOp = VK_ATTACHMENT_LOAD_OP_LOAD, - .stencilStoreOp = VK_ATTACHMENT_STORE_OP_STORE, - .initialLayout = initial_layout, - .finalLayout = VK_IMAGE_LAYOUT_GENERAL, - }; - - constexpr VkAttachmentReference color_attachment_ref{ - .attachment = 0, - .layout = VK_IMAGE_LAYOUT_GENERAL, - }; - - const VkSubpassDescription subpass_description{ - .flags = 0, - .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, - .inputAttachmentCount = 0, - .pInputAttachments = nullptr, - .colorAttachmentCount = 1, - .pColorAttachments = &color_attachment_ref, - .pResolveAttachments = nullptr, - .pDepthStencilAttachment = nullptr, - .preserveAttachmentCount = 0, - .pPreserveAttachments = nullptr, - }; - - constexpr VkSubpassDependency dependency{ - .srcSubpass = VK_SUBPASS_EXTERNAL, - .dstSubpass = 0, - .srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, - .dstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, - .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .dependencyFlags = 0, - }; - - return device.GetLogical().CreateRenderPass(VkRenderPassCreateInfo{ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .attachmentCount = 1, - .pAttachments = &attachment, - .subpassCount = 1, - .pSubpasses = &subpass_description, - .dependencyCount = 1, - .pDependencies = &dependency, - }); -} - -vk::Framebuffer CreateWrappedFramebuffer(const Device& device, vk::RenderPass& render_pass, - vk::ImageView& dest_image, VkExtent2D extent) { - return device.GetLogical().CreateFramebuffer(VkFramebufferCreateInfo{ - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .renderPass = *render_pass, - .attachmentCount = 1, - .pAttachments = dest_image.address(), - .width = extent.width, - .height = extent.height, - .layers = 1, - }); -} - vk::Sampler CreateWrappedSampler(const Device& device, VkFilter filter) { return device.GetLogical().CreateSampler(VkSamplerCreateInfo{ .sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO, @@ -396,8 +332,20 @@ vk::PipelineLayout CreateWrappedPipelineLayout(const Device& device, }); } +VkPipelineRenderingCreateInfo ColorRenderingInfo(const VkFormat& format) { + return { + .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, + .pNext = nullptr, + .viewMask = 0, + .colorAttachmentCount = 1, + .pColorAttachmentFormats = &format, + .depthAttachmentFormat = VK_FORMAT_UNDEFINED, + .stencilAttachmentFormat = VK_FORMAT_UNDEFINED, + }; +} + static vk::Pipeline CreateWrappedPipelineImpl( - const Device& device, vk::RenderPass& renderpass, vk::PipelineLayout& layout, + const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders, VkPipelineColorBlendAttachmentState blending) { const std::array shader_stages{{ @@ -501,9 +449,10 @@ static vk::Pipeline CreateWrappedPipelineImpl( .pDynamicStates = dynamic_states.data(), }; + const VkPipelineRenderingCreateInfo rendering_ci = ColorRenderingInfo(format); return device.GetLogical().CreateGraphicsPipeline(VkGraphicsPipelineCreateInfo{ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = 0, .stageCount = static_cast(shader_stages.size()), .pStages = shader_stages.data(), @@ -517,14 +466,14 @@ static vk::Pipeline CreateWrappedPipelineImpl( .pColorBlendState = &color_blend_ci, .pDynamicState = &dynamic_state_ci, .layout = *layout, - .renderPass = *renderpass, + .renderPass = VK_NULL_HANDLE, .subpass = 0, .basePipelineHandle = 0, .basePipelineIndex = 0, - }, device.StaticPipelineCache()); + }); } -vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass, +vk::Pipeline CreateWrappedPipeline(const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders) { constexpr VkPipelineColorBlendAttachmentState color_blend_attachment_disabled{ @@ -539,12 +488,12 @@ vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderp VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, }; - return CreateWrappedPipelineImpl(device, renderpass, layout, shaders, + return CreateWrappedPipelineImpl(device, format, layout, shaders, color_blend_attachment_disabled); } vk::Pipeline CreateWrappedPremultipliedBlendingPipeline( - const Device& device, vk::RenderPass& renderpass, vk::PipelineLayout& layout, + const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders) { constexpr VkPipelineColorBlendAttachmentState color_blend_attachment_premultiplied{ .blendEnable = VK_TRUE, @@ -558,12 +507,12 @@ vk::Pipeline CreateWrappedPremultipliedBlendingPipeline( VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, }; - return CreateWrappedPipelineImpl(device, renderpass, layout, shaders, + return CreateWrappedPipelineImpl(device, format, layout, shaders, color_blend_attachment_premultiplied); } vk::Pipeline CreateWrappedCoverageBlendingPipeline( - const Device& device, vk::RenderPass& renderpass, vk::PipelineLayout& layout, + const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders) { constexpr VkPipelineColorBlendAttachmentState color_blend_attachment_coverage{ .blendEnable = VK_TRUE, @@ -577,7 +526,7 @@ vk::Pipeline CreateWrappedCoverageBlendingPipeline( VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, }; - return CreateWrappedPipelineImpl(device, renderpass, layout, shaders, + return CreateWrappedPipelineImpl(device, format, layout, shaders, color_blend_attachment_coverage); } @@ -699,21 +648,35 @@ void ClearColorImage(vk::CommandBuffer& cmdbuf, VkImage image) { cmdbuf.ClearColorImage(image, VK_IMAGE_LAYOUT_GENERAL, {}, subresources); } -void BeginRenderPass(vk::CommandBuffer& cmdbuf, VkRenderPass render_pass, VkFramebuffer framebuffer, - VkExtent2D extent) { - const VkRenderPassBeginInfo renderpass_bi{ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, +void BeginRendering(vk::CommandBuffer& cmdbuf, VkImageView view, VkExtent2D extent, + VkAttachmentLoadOp load_op, const VkClearValue& clear_value) { + const VkRenderingAttachmentInfo attachment{ + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, .pNext = nullptr, - .renderPass = render_pass, - .framebuffer = framebuffer, + .imageView = view, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = VK_RESOLVE_MODE_NONE, + .resolveImageView = VK_NULL_HANDLE, + .resolveImageLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .loadOp = load_op, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = clear_value, + }; + cmdbuf.BeginRendering(VkRenderingInfo{ + .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, + .pNext = nullptr, + .flags = 0, .renderArea{ .offset{}, .extent = extent, }, - .clearValueCount = 0, - .pClearValues = nullptr, - }; - cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); + .layerCount = 1, + .viewMask = 0, + .colorAttachmentCount = 1, + .pColorAttachments = &attachment, + .pDepthAttachment = nullptr, + .pStencilAttachment = nullptr, + }); const VkViewport viewport{ .x = 0.0f, diff --git a/src/video_core/renderer_vulkan/present/util.h b/src/video_core/renderer_vulkan/present/util.h index ecc263286d..9eb862b0a8 100644 --- a/src/video_core/renderer_vulkan/present/util.h +++ b/src/video_core/renderer_vulkan/present/util.h @@ -27,10 +27,6 @@ void DownloadColorImage(vk::CommandBuffer& cmdbuf, VkImage image, VkBuffer buffe void ClearColorImage(vk::CommandBuffer& cmdbuf, VkImage image); vk::ImageView CreateWrappedImageView(const Device& device, vk::Image& image, VkFormat format); -vk::RenderPass CreateWrappedRenderPass(const Device& device, VkFormat format, - VkImageLayout initial_layout = VK_IMAGE_LAYOUT_GENERAL); -vk::Framebuffer CreateWrappedFramebuffer(const Device& device, vk::RenderPass& render_pass, - vk::ImageView& dest_image, VkExtent2D extent); vk::Sampler CreateWrappedSampler(const Device& device, VkFilter filter = VK_FILTER_LINEAR); vk::ShaderModule CreateWrappedShaderModule(const Device& device, std::span code); vk::DescriptorPool CreateWrappedDescriptorPool(const Device& device, size_t max_descriptors, @@ -47,16 +43,17 @@ vk::DescriptorSets CreateWrappedDescriptorSets(vk::DescriptorPool& pool, vk::Span layouts); vk::PipelineLayout CreateWrappedPipelineLayout(const Device& device, vk::DescriptorSetLayout& layout); -vk::Pipeline CreateWrappedPipeline(const Device& device, vk::RenderPass& renderpass, +[[nodiscard]] VkPipelineRenderingCreateInfo ColorRenderingInfo(const VkFormat& format); +vk::Pipeline CreateWrappedPipeline(const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders); vk::Pipeline CreateWrappedComputePipeline(const Device& device, vk::PipelineLayout& layout, VkShaderModule shader); vk::Pipeline CreateWrappedPremultipliedBlendingPipeline( - const Device& device, vk::RenderPass& renderpass, vk::PipelineLayout& layout, + const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders); vk::Pipeline CreateWrappedCoverageBlendingPipeline( - const Device& device, vk::RenderPass& renderpass, vk::PipelineLayout& layout, + const Device& device, VkFormat format, vk::PipelineLayout& layout, std::tuple shaders); VkWriteDescriptorSet CreateWriteDescriptorSet(std::vector& images, VkSampler sampler, VkImageView view, @@ -65,7 +62,7 @@ vk::Sampler CreateBilinearSampler(const Device& device); vk::Sampler CreateNearestNeighborSampler(const Device& device); vk::Sampler CreateCubicSampler(const Device& device, VkCubicFilterWeightsQCOM qcom_weights); -void BeginRenderPass(vk::CommandBuffer& cmdbuf, VkRenderPass render_pass, VkFramebuffer framebuffer, - VkExtent2D extent); +void BeginRendering(vk::CommandBuffer& cmdbuf, VkImageView view, VkExtent2D extent, + VkAttachmentLoadOp load_op, const VkClearValue& clear_value = {}); } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/window_adapt_pass.cpp b/src/video_core/renderer_vulkan/present/window_adapt_pass.cpp index 549b0660c1..9cd99206c7 100644 --- a/src/video_core/renderer_vulkan/present/window_adapt_pass.cpp +++ b/src/video_core/renderer_vulkan/present/window_adapt_pass.cpp @@ -25,8 +25,7 @@ WindowAdaptPass::WindowAdaptPass(const Device& device, VkFormat frame_format, vk CreateDescriptorSetLayout(device); CreatePipelineLayout(device); CreateVertexShader(device); - CreateRenderPass(device, frame_format); - CreatePipelines(device); + CreatePipelines(device, frame_format); } WindowAdaptPass::~WindowAdaptPass() = default; @@ -35,66 +34,70 @@ void WindowAdaptPass::Draw(const Device& device, RasterizerVulkan& rasterizer, S std::list& layers, std::span configs, const Layout::FramebufferLayout& layout, Frame* dst) { - - const VkFramebuffer host_framebuffer{*dst->framebuffer}; - const VkRenderPass renderpass{*render_pass}; - const VkPipelineLayout graphics_pipeline_layout{*pipeline_layout}; - const VkExtent2D render_area{ - .width = dst->width, - .height = dst->height, - }; - const size_t layer_count = configs.size(); - std::vector push_constants(layer_count); - std::vector descriptor_sets(layer_count); - std::vector graphics_pipelines(layer_count); + layer_push_constants.resize(layer_count); + layer_descriptor_sets.resize(layer_count); + layer_pipelines.resize(layer_count); auto layer_it = layers.begin(); for (size_t i = 0; i < layer_count; i++) { switch (configs[i].blending) { case Tegra::BlendMode::Opaque: default: - graphics_pipelines[i] = *opaque_pipeline; + layer_pipelines[i] = *opaque_pipeline; break; case Tegra::BlendMode::Premultiplied: - graphics_pipelines[i] = *premultiplied_pipeline; + layer_pipelines[i] = *premultiplied_pipeline; break; case Tegra::BlendMode::Coverage: - graphics_pipelines[i] = *coverage_pipeline; + layer_pipelines[i] = *coverage_pipeline; break; } - layer_it->ConfigureDraw(device, &push_constants[i], &descriptor_sets[i], rasterizer, *sampler, - image_index, configs[i], layout); + layer_it->ConfigureDraw(device, &layer_push_constants[i], &layer_descriptor_sets[i], + rasterizer, *sampler, image_index, configs[i], layout); layer_it++; } - scheduler.Record([=](vk::CommandBuffer cmdbuf) { + Record(scheduler, dst, layer_push_constants, layer_descriptor_sets); +} + +void WindowAdaptPass::DrawGenerated(const Device& device, Scheduler& scheduler, Layer& layer, + size_t generation, VkImage image, VkImageView view, + const Layout::FramebufferLayout& layout, Frame* dst) { + std::vector push_constants = layer_push_constants; + std::vector descriptor_sets = layer_descriptor_sets; + layer.ConfigureGenerated(device, &push_constants.front(), &descriptor_sets.front(), *sampler, + generation, image, view, layout); + Record(scheduler, dst, std::move(push_constants), std::move(descriptor_sets)); +} + +void WindowAdaptPass::Record(Scheduler& scheduler, Frame* dst, + std::vector push_constants, + std::vector descriptor_sets) const { + const VkImage host_image{*dst->image}; + const VkImageView host_view{*dst->image_view}; + const VkPipelineLayout graphics_pipeline_layout{*pipeline_layout}; + const VkExtent2D render_area{ + .width = dst->width, + .height = dst->height, + }; + scheduler.Record([host_image, host_view, graphics_pipeline_layout, render_area, + graphics_pipelines = layer_pipelines, + push_constants = std::move(push_constants), + descriptor_sets = std::move(descriptor_sets)](vk::CommandBuffer cmdbuf) { const f32 bg_red = Settings::values.bg_red.GetValue() / 255.0f; const f32 bg_green = Settings::values.bg_green.GetValue() / 255.0f; const f32 bg_blue = Settings::values.bg_blue.GetValue() / 255.0f; - const VkClearAttachment clear_attachment{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .colorAttachment = 0, - .clearValue = - { - .color = {.float32 = {bg_red, bg_green, bg_blue, 1.0f}}, - }, - }; - const VkClearRect clear_rect{ - .rect = - { - .offset = {0, 0}, - .extent = render_area, - }, - .baseArrayLayer = 0, - .layerCount = 1, + const VkClearValue clear_value{ + .color = {.float32 = {bg_red, bg_green, bg_blue, 1.0f}}, }; - BeginRenderPass(cmdbuf, renderpass, host_framebuffer, render_area); - cmdbuf.ClearAttachments({clear_attachment}, {clear_rect}); + TransitionImageLayout(cmdbuf, host_image, VK_IMAGE_LAYOUT_GENERAL, + VK_IMAGE_LAYOUT_UNDEFINED); + BeginRendering(cmdbuf, host_view, render_area, VK_ATTACHMENT_LOAD_OP_CLEAR, clear_value); - for (size_t i = 0; i < layer_count; i++) { + for (size_t i = 0; i < graphics_pipelines.size(); i++) { cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_GRAPHICS, graphics_pipelines[i]); cmdbuf.PushConstants(graphics_pipeline_layout, VK_SHADER_STAGE_VERTEX_BIT, push_constants[i]); @@ -103,7 +106,7 @@ void WindowAdaptPass::Draw(const Device& device, RasterizerVulkan& rasterizer, S cmdbuf.Draw(4, 1, 0, 0); } - cmdbuf.EndRenderPass(); + cmdbuf.EndRendering(); }); } @@ -111,10 +114,6 @@ VkDescriptorSetLayout WindowAdaptPass::GetDescriptorSetLayout() { return *descriptor_set_layout; } -VkRenderPass WindowAdaptPass::GetRenderPass() { - return *render_pass; -} - void WindowAdaptPass::CreateDescriptorSetLayout(const Device& device) { descriptor_set_layout = CreateWrappedDescriptorSetLayout(device, {VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER}); @@ -142,17 +141,13 @@ void WindowAdaptPass::CreateVertexShader(const Device& device) { vertex_shader = BuildShader(device, VULKAN_PRESENT_VERT_SPV); } -void WindowAdaptPass::CreateRenderPass(const Device& device, VkFormat frame_format) { - render_pass = CreateWrappedRenderPass(device, frame_format, VK_IMAGE_LAYOUT_UNDEFINED); -} - -void WindowAdaptPass::CreatePipelines(const Device& device) { - opaque_pipeline = CreateWrappedPipeline(device, render_pass, pipeline_layout, +void WindowAdaptPass::CreatePipelines(const Device& device, VkFormat frame_format) { + opaque_pipeline = CreateWrappedPipeline(device, frame_format, pipeline_layout, std::tie(vertex_shader, fragment_shader)); premultiplied_pipeline = CreateWrappedPremultipliedBlendingPipeline( - device, render_pass, pipeline_layout, std::tie(vertex_shader, fragment_shader)); + device, frame_format, pipeline_layout, std::tie(vertex_shader, fragment_shader)); coverage_pipeline = CreateWrappedCoverageBlendingPipeline( - device, render_pass, pipeline_layout, std::tie(vertex_shader, fragment_shader)); + device, frame_format, pipeline_layout, std::tie(vertex_shader, fragment_shader)); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/present/window_adapt_pass.h b/src/video_core/renderer_vulkan/present/window_adapt_pass.h index 58d6fc01c0..d33cacaee9 100644 --- a/src/video_core/renderer_vulkan/present/window_adapt_pass.h +++ b/src/video_core/renderer_vulkan/present/window_adapt_pass.h @@ -7,8 +7,10 @@ #pragma once #include +#include #include "common/math_util.h" +#include "video_core/renderer_vulkan/present/present_push_constants.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Layout { @@ -37,25 +39,32 @@ public: std::list& layers, std::span configs, const Layout::FramebufferLayout& layout, Frame* dst); + void DrawGenerated(const Device& device, Scheduler& scheduler, Layer& layer, size_t generation, + VkImage image, VkImageView view, const Layout::FramebufferLayout& layout, + Frame* dst); + VkDescriptorSetLayout GetDescriptorSetLayout(); - VkRenderPass GetRenderPass(); private: + void Record(Scheduler& scheduler, Frame* dst, std::vector push_constants, + std::vector descriptor_sets) const; + void CreateDescriptorSetLayout(const Device& device); void CreatePipelineLayout(const Device& device); void CreateVertexShader(const Device& device); - void CreateRenderPass(const Device& device, VkFormat frame_format); - void CreatePipelines(const Device& device); + void CreatePipelines(const Device& device, VkFormat frame_format); vk::DescriptorSetLayout descriptor_set_layout; vk::PipelineLayout pipeline_layout; vk::Sampler sampler; vk::ShaderModule vertex_shader; vk::ShaderModule fragment_shader; - vk::RenderPass render_pass; vk::Pipeline opaque_pipeline; vk::Pipeline premultiplied_pipeline; vk::Pipeline coverage_pipeline; + std::vector layer_pipelines; + std::vector layer_push_constants; + std::vector layer_descriptor_sets; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/renderer_vulkan.cpp b/src/video_core/renderer_vulkan/renderer_vulkan.cpp index 4fd4c0233b..b207997b76 100644 --- a/src/video_core/renderer_vulkan/renderer_vulkan.cpp +++ b/src/video_core/renderer_vulkan/renderer_vulkan.cpp @@ -52,23 +52,6 @@ constexpr VkExtent2D CaptureImageSize{ .height = VideoCore::Capture::LinearHeight, }; -#ifdef HAS_LSFG -[[nodiscard]] VkExtent2D GuestExtent(std::span framebuffers) { - if (framebuffers.empty()) { - return VkExtent2D{}; - } - - const auto& framebuffer = framebuffers.front(); - if (framebuffer.crop_rect.IsEmpty()) { - return VkExtent2D{.width = framebuffer.width, .height = framebuffer.height}; - } - return VkExtent2D{ - .width = static_cast(framebuffer.crop_rect.GetWidth()), - .height = static_cast(framebuffer.crop_rect.GetHeight()), - }; -} -#endif - constexpr VkExtent3D CaptureImageExtent{ .width = VideoCore::Capture::LinearWidth, .height = VideoCore::Capture::LinearHeight, @@ -202,8 +185,9 @@ RendererVulkan::~RendererVulkan() { } void RendererVulkan::Composite(std::span framebuffers) { + u32 pace_index = 0; SCOPE_EXIT { - render_window.OnFrameDisplayed(); + render_window.OnFrameDisplayed(pace_index + 1); }; RenderAppletCaptureLayer(framebuffers); @@ -217,27 +201,42 @@ void RendererVulkan::Composite(std::span framebu scheduler.RequestOutsideRenderPassOperationContext(); blit_swapchain.DrawToFrame(device, rasterizer, frame, framebuffers, - render_window.GetFramebufferLayout(), swapchain.GetImageCount(), + render_window.GetFramebufferLayout(), + present_manager.SwapchainImageCount(), swapchain.GetImageViewFormat()); + std::chrono::nanoseconds pace_step{}; #ifdef HAS_LSFG void(frame_gen.WantedGenerations(present_manager.MaxExtraFrames())); - frame_gen.Process(device, frame, swapchain.GetImageFormat(), GuestExtent(framebuffers)); + const FrameGenSource source = blit_swapchain.GenerationSource(); + frame_gen.Process(device, source.view, source.extent); + const Layout::FramebufferLayout layout = render_window.GetFramebufferLayout(); const size_t generated_frames = frame_gen.GeneratedFrameCount(); + const std::chrono::nanoseconds generated_step = frame_gen.PaceStep(); for (size_t generation = 0; generation < generated_frames; ++generation) { - Frame* generated = present_manager.GetRenderFrame(); - blit_swapchain.PrepareFrame(device, generated, render_window.GetFramebufferLayout()); - frame_gen.GenerateInto(device, generated, generation); + if (!blit_swapchain.IsGenerationFree(generation)) { + break; + } + Frame* const generated = present_manager.TryGetRenderFrame(); + if (generated == nullptr) { + break; + } + blit_swapchain.PrepareFrame(device, generated, layout); + const LsfgImage& output = frame_gen.Generate(device, generation); + blit_swapchain.DrawGenerated(device, generated, layout, generation, output.Handle(), + output.View()); scheduler.Flush(*generated->render_ready); - present_manager.Present(generated); + present_manager.Present(generated, static_cast(generation), generated_step); + pace_index = static_cast(generation + 1); + pace_step = generated_step; } #endif scheduler.Flush(*frame->render_ready); - present_manager.Present(frame); + present_manager.Present(frame, pace_index, pace_step); #ifdef HAS_LSFG scheduler.DispatchWork(); #endif @@ -273,7 +272,6 @@ vk::Buffer RendererVulkan::RenderToBuffer(std::span framebuffers, diff --git a/src/video_core/renderer_vulkan/vk_blit_screen.cpp b/src/video_core/renderer_vulkan/vk_blit_screen.cpp index 205382f81a..1654717a36 100644 --- a/src/video_core/renderer_vulkan/vk_blit_screen.cpp +++ b/src/video_core/renderer_vulkan/vk_blit_screen.cpp @@ -33,8 +33,8 @@ BlitScreen::BlitScreen(Tegra::MaxwellDeviceMemoryManager& device_memory_, const BlitScreen::~BlitScreen() = default; void BlitScreen::WaitIdle(const Device& device) { - present_manager.WaitPresent(); scheduler.Finish(); + present_manager.WaitPresent(); device.GetLogical().WaitIdle(); } @@ -92,14 +92,33 @@ void BlitScreen::PrepareFrame(const Device& device, Frame* frame, return; } - if (frame->width != layout.width || frame->height != layout.height) { - WaitIdle(device); - } else if (!present_manager.NeedsStorage(frame, true)) { + if (frame->width == layout.width && frame->height == layout.height) { return; } - present_manager.RecreateFrame(frame, layout.width, layout.height, swapchain_view_format, - window_adapt->GetRenderPass(), true); + WaitIdle(device); + present_manager.RecreateFrame(frame, layout.width, layout.height, swapchain_view_format); +} + +FrameGenSource BlitScreen::GenerationSource() const { + if (layers.empty()) { + return {}; + } + return FrameGenSource{ + .view = layers.front().GenerationSource(), + .extent = layers.front().GenerationExtent(), + }; +} + +bool BlitScreen::IsGenerationFree(size_t generation) const { + return !layers.empty() && layers.front().IsGenerationFree(generation); +} + +void BlitScreen::DrawGenerated(const Device& device, Frame* frame, + const Layout::FramebufferLayout& layout, size_t generation, + VkImage image, VkImageView view) { + window_adapt->DrawGenerated(device, scheduler, layers.front(), generation, image, view, layout, + frame); } void BlitScreen::DrawToFrame(const Device& device, RasterizerVulkan& rasterizer, Frame* frame, @@ -126,20 +145,16 @@ void BlitScreen::DrawToFrame(const Device& device, RasterizerVulkan& rasterizer, swapchain_view_format = current_swapchain_view_format; } - const bool storage_required = Settings::values.frame_gen.GetValue(); if (resource_update_required) { WaitIdle(device); SetWindowAdaptPass(device); if (presentation_recreate_required) { - present_manager.RecreateFrame(frame, layout.width, layout.height, swapchain_view_format, - window_adapt->GetRenderPass(), storage_required); + present_manager.RecreateFrame(frame, layout.width, layout.height, + swapchain_view_format); } image_index = 0; - } else if (present_manager.NeedsStorage(frame, storage_required)) { - present_manager.RecreateFrame(frame, layout.width, layout.height, swapchain_view_format, - window_adapt->GetRenderPass(), true); } const VkExtent2D window_size{ @@ -147,11 +162,14 @@ void BlitScreen::DrawToFrame(const Device& device, RasterizerVulkan& rasterizer, .height = layout.screen.GetHeight(), }; - if (layers.size() != framebuffers.size()) { + const size_t generations = Settings::FrameGenMaxGenerations(); + if (layers.size() != framebuffers.size() || generation_count != generations) { layers.clear(); + generation_count = generations; for (size_t i = 0; i < framebuffers.size(); ++i) { layers.emplace_back(device, memory_allocator, scheduler, device_memory, image_count, - window_size, window_adapt->GetDescriptorSetLayout(), filters); + generation_count, window_size, + window_adapt->GetDescriptorSetLayout(), filters); } } @@ -162,39 +180,4 @@ void BlitScreen::DrawToFrame(const Device& device, RasterizerVulkan& rasterizer, } } -vk::Framebuffer BlitScreen::CreateFramebuffer(const Device& device, const Layout::FramebufferLayout& layout, - VkImageView image_view, - VkFormat current_view_format) { - bool format_updated = swapchain_view_format != current_view_format; - swapchain_view_format = current_view_format; - - if (!window_adapt || scaling_filter != filters.get_scaling_filter() || format_updated) { - WaitIdle(device); - SetWindowAdaptPass(device); - image_index = 0; - } - - const VkExtent2D extent{ - .width = layout.width, - .height = layout.height, - }; - - return CreateFramebuffer(device, image_view, extent, window_adapt->GetRenderPass()); -} - -vk::Framebuffer BlitScreen::CreateFramebuffer(const Device& device, const VkImageView& image_view, VkExtent2D extent, - VkRenderPass render_pass) { - return device.GetLogical().CreateFramebuffer(VkFramebufferCreateInfo{ - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .renderPass = render_pass, - .attachmentCount = 1, - .pAttachments = &image_view, - .width = extent.width, - .height = extent.height, - .layers = 1, - }); -} - } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_blit_screen.h b/src/video_core/renderer_vulkan/vk_blit_screen.h index 56b429ae53..c6cac52321 100644 --- a/src/video_core/renderer_vulkan/vk_blit_screen.h +++ b/src/video_core/renderer_vulkan/vk_blit_screen.h @@ -48,6 +48,11 @@ struct FramebufferTextureInfo { u32 scaled_height{}; }; +struct FrameGenSource { + VkImageView view{}; + VkExtent2D extent{}; +}; + class BlitScreen { public: explicit BlitScreen(Tegra::MaxwellDeviceMemoryManager& device_memory, const Device& device, @@ -62,15 +67,16 @@ public: void PrepareFrame(const Device& device, Frame* frame, const Layout::FramebufferLayout& layout); - [[nodiscard]] vk::Framebuffer CreateFramebuffer(const Device& device, const Layout::FramebufferLayout& layout, - VkImageView image_view, - VkFormat current_view_format); + [[nodiscard]] FrameGenSource GenerationSource() const; + + [[nodiscard]] bool IsGenerationFree(size_t generation) const; + + void DrawGenerated(const Device& device, Frame* frame, const Layout::FramebufferLayout& layout, + size_t generation, VkImage image, VkImageView view); private: void WaitIdle(const Device& device); void SetWindowAdaptPass(const Device& device); - vk::Framebuffer CreateFramebuffer(const Device& device, const VkImageView& image_view, VkExtent2D extent, - VkRenderPass render_pass); Tegra::MaxwellDeviceMemoryManager& device_memory; MemoryAllocator& memory_allocator; @@ -79,6 +85,7 @@ private: const PresentFilters& filters; std::size_t image_count{}; std::size_t image_index{}; + std::size_t generation_count{}; VkFormat swapchain_view_format{}; Settings::ScalingFilter scaling_filter{}; diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp index 958f3b2a2e..49926d01ad 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.cpp @@ -24,8 +24,31 @@ namespace Vulkan { namespace { -VkBufferCopy MakeBufferCopy(const VideoCommon::BufferCopy& copy) { - return VkBufferCopy{ +constexpr u32 COMPACT_VERTEX_BINDINGS = 8; + +constexpr VkMemoryBarrier2 HOST_READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, + .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT, +}; + +template +struct VertexBindings { + std::array buffers; + std::array offsets; + std::array sizes; + std::array strides; + u32 first; + u32 count; +}; + +VkBufferCopy2 MakeBufferCopy(const VideoCommon::BufferCopy& copy) { + return VkBufferCopy2{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, .srcOffset = copy.src_offset, .dstOffset = copy.dst_offset, .size = copy.size, @@ -83,15 +106,20 @@ vk::Buffer CreateBuffer(const Device& device, const MemoryAllocator& memory_allo .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, }; - if (sparse_alignment > 1) { - return memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::DeviceLocal, sparse_alignment); + MemoryUsage usage = MemoryUsage::DeviceLocal; + if (device.IsUMA()) { + usage = MemoryUsage::Stream; } - return memory_allocator.CreateBuffer(buffer_ci, MemoryUsage::DeviceLocal); + if (sparse_alignment > 1) { + return memory_allocator.CreateBuffer(buffer_ci, usage, sparse_alignment); + } + return memory_allocator.CreateBuffer(buffer_ci, usage); } } // Anonymous namespace Buffer::Buffer(BufferCacheRuntime& runtime, VideoCommon::NullBufferParams null_params) - : VideoCommon::BufferBase(null_params), scheduler{&runtime.scheduler}, tracker{4096} { + : VideoCommon::BufferBase(null_params), scheduler{&runtime.scheduler}, tracker{4096}, + uploads{4096} { if (runtime.device.HasNullDescriptor()) { return; } @@ -109,7 +137,7 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_, scheduler{&runtime.scheduler}, buffer{CreateBuffer(*device, runtime.memory_allocator, SizeBytes(), runtime.SparseAlignmentFor(sparse_compatible_))}, - tracker{SizeBytes()} { + tracker{SizeBytes()}, uploads{SizeBytes()} { sparse_compatible = sparse_compatible_; if (runtime.device.HasDebuggingToolAttached()) { buffer.SetObjectNameEXT(fmt::format("Buffer {:#x}", CpuAddr()).c_str()); @@ -119,9 +147,38 @@ Buffer::Buffer(BufferCacheRuntime& runtime, DAddr cpu_addr_, u64 size_bytes_, } } +bool Buffer::IsRegionUsed(u64 offset, u64 size) const noexcept { + return tracker.IsUsed(offset, size, scheduler->GetMasterSemaphore().KnownGpuTick()); +} + +bool Buffer::IsRegionUploading(u64 offset, u64 size) const noexcept { + return uploads.IsUsed(offset, size, scheduler->GetMasterSemaphore().KnownGpuTick()); +} + void Buffer::MarkUsage(u64 offset, u64 size) noexcept { - tracker.Track(offset, size); - last_usage_tick = scheduler->CurrentTick(); + const u64 tick = scheduler->CurrentTick(); + if (tick == usage_tick && offset >= usage_begin && offset + size <= usage_end) { + return; + } + tracker.Track(offset, size, tick, scheduler->GetMasterSemaphore().KnownGpuTick()); + usage_tick = tick; + usage_begin = offset; + usage_end = offset + size; +} + +void Buffer::MarkUpload(std::span copies) noexcept { + last_upload_tick = scheduler->CurrentTick(); + const u64 gpu_tick = scheduler->GetMasterSemaphore().KnownGpuTick(); + for (const VideoCommon::BufferCopy& copy : copies) { + uploads.Track(copy.dst_offset, copy.size, last_upload_tick, gpu_tick); + } +} + +std::span Buffer::CoherentMapping() noexcept { + if (!buffer.IsHostCoherent()) { + return {}; + } + return buffer.Mapped(); } VkBufferView Buffer::View(u32 offset, u32 size, VideoCore::Surface::PixelFormat format) { @@ -133,6 +190,11 @@ VkBufferView Buffer::View(u32 offset, u32 size, VideoCore::Surface::PixelFormat offset = 0; size = 0; } + const u32 required = + static_cast(device->TexelBufferAlignment(VideoCore::Surface::BytesPerBlock(format))); + const u32 misalign = offset % required; + offset -= misalign; + size += misalign; const auto it{std::ranges::find_if(views, [offset, size, format](const BufferView& view) { return offset == view.offset && size == view.size && format == view.format; })}; @@ -217,16 +279,20 @@ public: scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([src_buffer = staging.buffer, src_offset = staging.offset, dst_buffer = *buffer, size_bytes](vk::CommandBuffer cmdbuf) { - const VkBufferCopy copy{ + const VkBufferCopy2 copy{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, .srcOffset = src_offset, .dstOffset = 0, .size = size_bytes, }; - const VkBufferMemoryBarrier write_barrier{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + const VkBufferMemoryBarrier2 write_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_INDEX_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT, + .dstAccessMask = VK_ACCESS_2_INDEX_READ_BIT, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .buffer = dst_buffer, @@ -234,8 +300,7 @@ public: .size = size_bytes, }; cmdbuf.CopyBuffer(src_buffer, dst_buffer, copy); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, write_barrier); + cmdbuf.PipelineBarrier(write_barrier); }); } else { buffer.Flush(); @@ -263,7 +328,6 @@ protected: StagingBufferPool& staging_pool; vk::Buffer buffer{}; - MemoryCommit memory_commit{}; VkIndexType index_type{}; u32 num_indices = 0; }; @@ -357,7 +421,23 @@ BufferCacheRuntime::BufferCacheRuntime(const Device& device_, MemoryAllocator& m staging_pool{staging_pool_}, guest_descriptor_queue{guest_descriptor_queue_}, quad_index_pass(device, scheduler, descriptor_pool, staging_pool, compute_pass_descriptor_queue), - multi_range_buffers(device_) { + multi_range_buffers(device_), + read_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = device_.GetBufferUserStages(), + .srcAccessMask = vk::ACCESS_BUFFER_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = vk::ACCESS_TRANSFER, + }, + write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = device_.GetBufferConsumerStages(), + .dstAccessMask = device_.GetBufferConsumerAccess(), + } { const VkDriverIdKHR driver_id = device.GetDriverID(); limit_dynamic_storage_buffers = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY; @@ -410,18 +490,26 @@ u32 BufferCacheRuntime::GetStorageBufferAlignment() const { return static_cast(device.GetStorageBufferAlignment()); } -void BufferCacheRuntime::TickFrame(Common::SlotVector& slot_buffers) noexcept { - for (auto it = slot_buffers.begin(); it != slot_buffers.end(); it++) { - if (scheduler.IsFree(it->LastUsageTick())) { - it->ResetUsageTracking(); - } - } +void BufferCacheRuntime::TickFrame(Common::SlotVector&) noexcept { + multi_range_buffers.DrainRetired(scheduler); } u64 BufferCacheRuntime::CurrentTick() { return scheduler.GetMasterSemaphore().CurrentTick(); } +u64 BufferCacheRuntime::RenderPassSerial() const noexcept { + return scheduler.ActiveRenderPassSerial(); +} + +u64 BufferCacheRuntime::WaitForIdleSerial() const noexcept { + return scheduler.WaitForIdleSerial(); +} + +void BufferCacheRuntime::MarkRenderPassWrites() noexcept { + scheduler.MarkRenderPassWrites(); +} + u64 BufferCacheRuntime::KnownGpuTick() { return scheduler.GetMasterSemaphore().KnownGpuTick(); } @@ -446,27 +534,43 @@ bool BufferCacheRuntime::CanReorderUpload(const Buffer& buffer, return can_use_upload_cmdbuf; } +std::span BufferCacheRuntime::DirectDownloadSpan(Buffer& buffer) { + const std::span mapping = buffer.CoherentMapping(); + if (mapping.empty()) { + return mapping; + } + if ((std::max)(buffer.getWriteTick(), buffer.LastUploadTick()) > host_read_tick) { + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record( + [](vk::CommandBuffer cmdbuf) { cmdbuf.PipelineBarrier(HOST_READ_BARRIER); }); + host_read_tick = scheduler.CurrentTick(); + } + scheduler.Wait(host_read_tick); + return mapping; +} + +std::span BufferCacheRuntime::DirectUploadSpan( + Buffer& buffer, std::span copies) { + const std::span mapping = buffer.CoherentMapping(); + const bool uploading = + std::ranges::any_of(copies, [&buffer](const VideoCommon::BufferCopy& copy) { + return buffer.IsRegionUploading(copy.dst_offset, copy.size); + }); + if (mapping.empty() || uploading || !CanReorderUpload(buffer, copies)) { + return {}; + } + return mapping; +} + void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, std::span copies, bool barrier, bool can_reorder_upload) { if (dst_buffer == VK_NULL_HANDLE || src_buffer == VK_NULL_HANDLE) { return; } - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - }; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, - }; // Measuring a popular game, this number never exceeds the specified size once data is warmed up - boost::container::small_vector vk_copies(copies.size()); + boost::container::small_vector vk_copies(copies.size()); std::ranges::transform(copies, vk_copies.begin(), MakeBufferCopy); if (src_buffer == staging_pool.StreamBuf() && can_reorder_upload) { scheduler.RecordWithUploadBuffer([src_buffer, dst_buffer, vk_copies]( @@ -477,44 +581,28 @@ void BufferCacheRuntime::CopyBuffer(VkBuffer dst_buffer, VkBuffer src_buffer, } scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([src_buffer, dst_buffer, vk_copies, barrier](vk::CommandBuffer cmdbuf) { + scheduler.Record([this, src_buffer, dst_buffer, vk_copies, barrier](vk::CommandBuffer cmdbuf) { if (barrier) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); + cmdbuf.PipelineBarrier(read_barrier); } cmdbuf.CopyBuffer(src_buffer, dst_buffer, VideoCommon::FixSmallVectorADL(vk_copies)); if (barrier) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, WRITE_BARRIER); + cmdbuf.PipelineBarrier(write_barrier); } }); } void BufferCacheRuntime::PreCopyBarrier() { - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - }; scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, READ_BARRIER); + scheduler.Record([this](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(read_barrier); }); } void BufferCacheRuntime::PostCopyBarrier() { - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, - }; scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, WRITE_BARRIER); + scheduler.Record([this](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(write_barrier); }); } @@ -522,26 +610,12 @@ void BufferCacheRuntime::ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t si if (dest_buffer == VK_NULL_HANDLE) { return; } - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - }; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, - }; scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([dest_buffer, offset, size, value](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, READ_BARRIER); + scheduler.Record([this, dest_buffer, offset, size, value](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.FillBuffer(dest_buffer, offset, size, value); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, WRITE_BARRIER); + cmdbuf.PipelineBarrier(write_barrier); }); } @@ -578,29 +652,40 @@ bool BufferCacheRuntime::BindMultiRangeStorageBuffer(u64 key, bool is_written) { void BufferCacheRuntime::BindIndexBuffer(PrimitiveTopology topology, IndexFormat index_format, u32 base_vertex, u32 num_indices, VkBuffer buffer, - u32 offset, [[maybe_unused]] u32 size) { + u32 offset, u32 size) { VkIndexType vk_index_type = MaxwellToVK::IndexFormat(index_format); VkDeviceSize vk_offset = offset; + VkDeviceSize vk_size = size; VkBuffer vk_buffer = buffer; if (topology == PrimitiveTopology::Quads || topology == PrimitiveTopology::QuadStrip) { vk_index_type = VK_INDEX_TYPE_UINT32; + vk_size = VK_WHOLE_SIZE; std::tie(vk_buffer, vk_offset) = quad_index_pass.Assemble(index_format, num_indices, base_vertex, buffer, offset, topology == PrimitiveTopology::QuadStrip); } else if (vk_index_type == VK_INDEX_TYPE_UINT8_EXT && !device.IsExtIndexTypeUint8Supported()) { vk_index_type = VK_INDEX_TYPE_UINT16; if (uint8_pass) { + vk_size = VK_WHOLE_SIZE; std::tie(vk_buffer, vk_offset) = uint8_pass->Assemble(num_indices, buffer, offset); } else if (device.GetDriverID() == VK_DRIVER_ID_QUALCOMM_PROPRIETARY) { ReserveNullBuffer(); vk_buffer = *null_buffer; vk_offset = 0; + vk_size = VK_WHOLE_SIZE; } } if (vk_buffer == VK_NULL_HANDLE) { // Vulkan doesn't support null index buffers. Replace it with our own null buffer. ReserveNullBuffer(); vk_buffer = *null_buffer; + vk_size = VK_WHOLE_SIZE; + } + if (device.IsKhrMaintenance5Supported()) { + scheduler.Record([vk_buffer, vk_offset, vk_size, vk_index_type](vk::CommandBuffer cmdbuf) { + cmdbuf.BindIndexBuffer2KHR(vk_buffer, vk_offset, vk_size, vk_index_type); + }); + return; } scheduler.Record([vk_buffer, vk_offset, vk_index_type](vk::CommandBuffer cmdbuf) { cmdbuf.BindIndexBuffer(vk_buffer, vk_offset, vk_index_type); @@ -625,16 +710,73 @@ void BufferCacheRuntime::BindQuadIndexBuffer(PrimitiveTopology topology, u32 fir } } +bool BufferCacheRuntime::BindQuadIndices(PrimitiveTopology topology, IndexFormat index_format, + u32 base_vertex, u32 num_indices, + std::span indices) { + const size_t index_size = size_t{1} << static_cast(index_format); + if (indices.size() < num_indices * index_size) { + return false; + } + u32 quads = num_indices / 4; + u32 stride = 4; + std::array swizzle{0, 1, 2, 0, 2, 3}; + if (topology == PrimitiveTopology::QuadStrip) { + quads = 0; + if (num_indices >= 2) { + quads = (num_indices - 2) / 2; + } + stride = 2; + swizzle = {0, 3, 1, 0, 2, 3}; + } + const size_t size = (std::max)(quads, 1U) * sizeof(swizzle); + const StagingBufferRef staging = staging_pool.Request(size, MemoryUsage::Upload); + const auto convert = [&](auto index_type) { + u8* output = staging.mapped_span.data(); + for (u32 quad = 0; quad < quads; ++quad) { + std::array triangles; + for (size_t vertex = 0; vertex < triangles.size(); ++vertex) { + decltype(index_type) index; + std::memcpy(&index, + indices.data() + (quad * stride + swizzle[vertex]) * sizeof(index), + sizeof(index)); + triangles[vertex] = index + base_vertex; + } + std::memcpy(output, triangles.data(), sizeof(triangles)); + output += sizeof(triangles); + } + }; + switch (index_format) { + case IndexFormat::UnsignedByte: + convert(u8{}); + break; + case IndexFormat::UnsignedShort: + convert(u16{}); + break; + case IndexFormat::UnsignedInt: + convert(u32{}); + break; + } + BindIndexBuffer(PrimitiveTopology::Triangles, IndexFormat::UnsignedInt, 0, 0, staging.buffer, + static_cast(staging.offset), static_cast(size)); + return true; +} + void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset, u32 size, u32 stride) { if (index >= device.GetMaxVertexInputBindings()) { return; } if (device.IsExtExtendedDynamicStateSupported()) { - scheduler.Record([index, buffer, offset, size, stride](vk::CommandBuffer cmdbuf) { + const bool dynamic_stride = !device.IsExtVertexInputDynamicStateSupported(); + scheduler.Record([index, buffer, offset, size, stride, + dynamic_stride](vk::CommandBuffer cmdbuf) { const VkDeviceSize vk_offset = buffer != VK_NULL_HANDLE ? offset : 0; const VkDeviceSize vk_size = buffer != VK_NULL_HANDLE ? size : VK_WHOLE_SIZE; const VkDeviceSize vk_stride = stride; - cmdbuf.BindVertexBuffers2EXT(index, 1, &buffer, &vk_offset, &vk_size, &vk_stride); + const VkDeviceSize* strides = nullptr; + if (dynamic_stride) { + strides = &vk_stride; + } + cmdbuf.BindVertexBuffers2EXT(index, 1, &buffer, &vk_offset, &vk_size, strides); }); } else { if (!device.HasNullDescriptor() && buffer == VK_NULL_HANDLE) { @@ -648,36 +790,56 @@ void BufferCacheRuntime::BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset } } -void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings& bindings) { - boost::container::static_vector buffer_handles(bindings.buffers.size()); - for (u32 i = 0; i < bindings.buffers.size(); ++i) { - auto handle = bindings.buffers[i]->Handle(); - if (handle == VK_NULL_HANDLE) { - bindings.offsets[i] = 0; - bindings.sizes[i] = VK_WHOLE_SIZE; +template +void BufferCacheRuntime::RecordVertexBuffers(const VideoCommon::HostBindings& bindings, + u32 count) { + VertexBindings vertex{}; + vertex.first = bindings.min_index; + vertex.count = count; + for (u32 i = 0; i < count; ++i) { + vertex.buffers[i] = bindings.buffers[i]->Handle(); + vertex.offsets[i] = bindings.offsets[i]; + vertex.sizes[i] = bindings.sizes[i]; + vertex.strides[i] = bindings.strides[i]; + if (vertex.buffers[i] == VK_NULL_HANDLE) { + vertex.offsets[i] = 0; + vertex.sizes[i] = VK_WHOLE_SIZE; if (!device.HasNullDescriptor()) { ReserveNullBuffer(); - handle = *null_buffer; + vertex.buffers[i] = *null_buffer; } } - buffer_handles[i] = handle; - } - const u32 device_max = device.GetMaxVertexInputBindings(); - const u32 min_binding = (std::min)(bindings.min_index, device_max); - const u32 max_binding = (std::min)(bindings.max_index, device_max); - const u32 binding_count = max_binding - min_binding; - if (binding_count == 0) { - return; } if (device.IsExtExtendedDynamicStateSupported()) { - scheduler.Record([bindings_ = std::move(bindings), buffer_handles_ = std::move(buffer_handles), binding_count](vk::CommandBuffer cmdbuf) { - cmdbuf.BindVertexBuffers2EXT(bindings_.min_index, binding_count, buffer_handles_.data(), bindings_.offsets.data(), bindings_.sizes.data(), bindings_.strides.data()); - }); - } else { - scheduler.Record([bindings_ = std::move(bindings), buffer_handles_ = std::move(buffer_handles), binding_count](vk::CommandBuffer cmdbuf) { - cmdbuf.BindVertexBuffers(bindings_.min_index, binding_count, buffer_handles_.data(), bindings_.offsets.data()); + const bool dynamic_stride = !device.IsExtVertexInputDynamicStateSupported(); + scheduler.Record([vertex, dynamic_stride](vk::CommandBuffer cmdbuf) { + const VkDeviceSize* strides = nullptr; + if (dynamic_stride) { + strides = vertex.strides.data(); + } + cmdbuf.BindVertexBuffers2EXT(vertex.first, vertex.count, vertex.buffers.data(), + vertex.offsets.data(), vertex.sizes.data(), strides); }); + return; } + scheduler.Record([vertex](vk::CommandBuffer cmdbuf) { + cmdbuf.BindVertexBuffers(vertex.first, vertex.count, vertex.buffers.data(), + vertex.offsets.data()); + }); +} + +void BufferCacheRuntime::BindVertexBuffers(VideoCommon::HostBindings& bindings) { + const u32 device_max = device.GetMaxVertexInputBindings(); + const u32 count = (std::min)(bindings.max_index, device_max) - + (std::min)(bindings.min_index, device_max); + if (count == 0) { + return; + } + if (count <= COMPACT_VERTEX_BINDINGS) { + RecordVertexBuffers(bindings, count); + return; + } + RecordVertexBuffers(bindings, count); } void BufferCacheRuntime::BindTransformFeedbackBuffer(u32 index, VkBuffer buffer, u32 offset, diff --git a/src/video_core/renderer_vulkan/vk_buffer_cache.h b/src/video_core/renderer_vulkan/vk_buffer_cache.h index fbd8c2e753..3209019bdf 100644 --- a/src/video_core/renderer_vulkan/vk_buffer_cache.h +++ b/src/video_core/renderer_vulkan/vk_buffer_cache.h @@ -55,19 +55,19 @@ public: return buffer.Location(); } - [[nodiscard]] bool IsRegionUsed(u64 offset, u64 size) const noexcept { - return tracker.IsUsed(offset, size); - } + [[nodiscard]] bool IsRegionUsed(u64 offset, u64 size) const noexcept; + + [[nodiscard]] bool IsRegionUploading(u64 offset, u64 size) const noexcept; void MarkUsage(u64 offset, u64 size) noexcept; - void ResetUsageTracking() noexcept { - tracker.Reset(); + void MarkUpload(std::span copies) noexcept; + + [[nodiscard]] u64 LastUploadTick() const noexcept { + return last_upload_tick; } - [[nodiscard]] u64 LastUsageTick() const noexcept { - return last_usage_tick; - } + [[nodiscard]] std::span CoherentMapping() noexcept; operator VkBuffer() const noexcept { return *buffer; @@ -86,8 +86,12 @@ private: vk::Buffer buffer; std::vector views; VideoCommon::UsageTracker tracker; + VideoCommon::UsageTracker uploads; VkDeviceAddress device_address{}; - u64 last_usage_tick{}; + u64 usage_tick{}; + u64 usage_begin{}; + u64 usage_end{}; + u64 last_upload_tick{}; bool is_null{}; bool sparse_compatible{}; }; @@ -108,10 +112,16 @@ public: ComputePassDescriptorQueue& compute_pass_descriptor_queue, DescriptorPool& descriptor_pool); - void TickFrame(Common::SlotVector& slot_buffers) noexcept; + void TickFrame(Common::SlotVector&) noexcept; u64 CurrentTick(); + u64 RenderPassSerial() const noexcept; + + u64 WaitForIdleSerial() const noexcept; + + void MarkRenderPassWrites() noexcept; + u64 KnownGpuTick(); void Wait(u64 buffer_tick); @@ -134,6 +144,11 @@ public: bool CanReorderUpload(const Buffer& buffer, std::span copies); + [[nodiscard]] std::span DirectDownloadSpan(Buffer& buffer); + + [[nodiscard]] std::span DirectUploadSpan(Buffer& buffer, + std::span copies); + void FreeDeferredStagingBuffer(StagingBufferRef& ref); void PreCopyBarrier(); @@ -142,6 +157,13 @@ public: std::span copies, bool barrier, bool can_reorder_upload = false); + void CopyBuffer(Buffer& dst_buffer, VkBuffer src_buffer, + std::span copies, bool barrier, + bool can_reorder_upload = false) { + dst_buffer.MarkUpload(copies); + CopyBuffer(dst_buffer.Handle(), src_buffer, copies, barrier, can_reorder_upload); + } + void PostCopyBarrier(); void ClearBuffer(VkBuffer dest_buffer, u32 offset, size_t size, u32 value); @@ -151,6 +173,9 @@ public: void BindQuadIndexBuffer(PrimitiveTopology topology, u32 first, u32 count); + bool BindQuadIndices(PrimitiveTopology topology, IndexFormat index_format, u32 base_vertex, + u32 num_indices, std::span indices); + void BindVertexBuffer(u32 index, VkBuffer buffer, u32 offset, u32 size, u32 stride); void BindVertexBuffers(VideoCommon::HostBindings& bindings); @@ -192,7 +217,7 @@ public: .memory_offset = location.offset, .offset = offset, .size = size, - .write_tick = buffer.getWriteTick(), + .content_serial = buffer.ContentSerial(), .memory_type = location.memory_type, }); multi_range_total += size; @@ -200,10 +225,6 @@ public: bool BindMultiRangeStorageBuffer(u64 key, bool is_written); - void InvalidateMultiRange(u64 key) { - multi_range_buffers.Invalidate(key); - } - void OnBufferDeleted(const Buffer& buffer) { multi_range_buffers.DropOwner(scheduler, buffer.Handle()); } @@ -244,6 +265,9 @@ private: VkFormat TexelBufferFormat(VideoCore::Surface::PixelFormat format) const; + template + void RecordVertexBuffers(const VideoCommon::HostBindings& bindings, u32 count); + void ReserveNullBuffer(); vk::Buffer CreateNullBuffer(); @@ -265,6 +289,10 @@ private: boost::container::small_vector multi_range_sources; VkDeviceSize multi_range_total{}; + VkMemoryBarrier2 read_barrier; + VkMemoryBarrier2 write_barrier; + u64 host_read_tick = 0; + bool limit_dynamic_storage_buffers = false; u32 max_dynamic_storage_buffers = (std::numeric_limits::max)(); }; diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.cpp b/src/video_core/renderer_vulkan/vk_compute_pass.cpp index ab58560170..f1b0248233 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pass.cpp @@ -12,17 +12,22 @@ #include "video_core/renderer_vulkan/vk_texture_cache.h" +#include + #include "common/assert.h" #include "common/common_types.h" #include "common/div_ceil.h" #include "common/vector_math.h" #include "video_core/host_shaders/astc_decoder_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_2d_comp_spv.h" +#include "video_core/host_shaders/block_linear_unswizzle_3d_comp_spv.h" +#include "video_core/host_shaders/pitch_unswizzle_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_comp_spv.h" #include "video_core/host_shaders/queries_prefix_scan_sum_nosubgroups_comp_spv.h" #include "video_core/host_shaders/resolve_conditional_render_comp_spv.h" +#include "video_core/host_shaders/vulkan_indirect_quads_comp_spv.h" #include "video_core/host_shaders/vulkan_quad_indexed_comp_spv.h" #include "video_core/host_shaders/vulkan_uint8_comp_spv.h" -#include "video_core/host_shaders/block_linear_unswizzle_3d_bcn_comp_spv.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" #include "video_core/surface.h" #include "video_core/renderer_vulkan/vk_descriptor_pool.h" @@ -185,6 +190,26 @@ struct AstcPushConstants { u32 block_height_mask; }; +struct BlockLinear3DImagePushConstants { + alignas(16) std::array origin; + alignas(16) std::array destination; + u32 bytes_per_block_log2; + u32 slice_size; + u32 block_size; + u32 x_shift; + u32 block_height; + u32 block_height_mask; + u32 block_depth; + u32 block_depth_mask; +}; + +struct PitchUnswizzlePushConstants { + std::array origin; + std::array destination; + u32 bytes_per_block_log2; + u32 pitch; +}; + struct QueriesPrefixScanPushConstants { u32 min_accumulation_base; u32 max_accumulation_base; @@ -268,7 +293,7 @@ ComputePass::ComputePass(const Device& device_, Scheduler& scheduler, Descriptor .layout = *layout, .basePipelineHandle = {}, .basePipelineIndex = 0, - }, device.StaticPipelineCache()); + }); } ComputePass::~ComputePass() = default; @@ -297,19 +322,20 @@ std::pair Uint8Pass::Assemble(u32 num_vertices, VkBuffer scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, descriptor_data, num_vertices](vk::CommandBuffer cmdbuf) { static constexpr u32 DISPATCH_SIZE = 1024; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT, + .dstAccessMask = VK_ACCESS_2_INDEX_READ_BIT, }; const VkDescriptorSet set = descriptor_allocator.Commit(); device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); cmdbuf.Dispatch(Common::DivCeil(num_vertices, DISPATCH_SIZE), 1, 1); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, WRITE_BARRIER); + cmdbuf.PipelineBarrier(WRITE_BARRIER); }); return {staging.buffer, staging.offset}; } @@ -342,7 +368,17 @@ std::pair QuadIndexedPass::Assemble( return 2; }(); const u32 input_size = num_vertices << index_shift; - const u32 num_tri_vertices = (is_strip ? (num_vertices - 2) / 2 : num_vertices / 4) * 6; + u32 quads = num_vertices / 4; + if (is_strip) { + quads = 0; + if (num_vertices >= 2) { + quads = (num_vertices - 2) / 2; + } + } + if (quads == 0) { + quads = 1; + } + const u32 num_tri_vertices = quads * 6; const std::size_t staging_size = num_tri_vertices * sizeof(u32); const auto staging = staging_buffer_pool.Request(staging_size, MemoryUsage::DeviceLocal); @@ -356,11 +392,13 @@ std::pair QuadIndexedPass::Assemble( scheduler.Record([this, descriptor_data, num_tri_vertices, base_vertex, index_shift, is_strip](vk::CommandBuffer cmdbuf) { static constexpr u32 DISPATCH_SIZE = 1024; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_INDEX_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT, + .dstAccessMask = VK_ACCESS_2_INDEX_READ_BIT, }; const std::array push_constants{base_vertex, index_shift, is_strip ? 1u : 0u}; const VkDescriptorSet set = descriptor_allocator.Commit(); @@ -370,8 +408,58 @@ std::pair QuadIndexedPass::Assemble( cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_constants), &push_constants); cmdbuf.Dispatch(Common::DivCeil(num_tri_vertices, DISPATCH_SIZE), 1, 1); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 0, WRITE_BARRIER); + cmdbuf.PipelineBarrier(WRITE_BARRIER); + }); + return {staging.buffer, staging.offset}; +} + +IndirectQuadsPass::IndirectQuadsPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, INPUT_OUTPUT_DESCRIPTOR_SET_BINDINGS, + INPUT_OUTPUT_DESCRIPTOR_UPDATE_TEMPLATE, INPUT_OUTPUT_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, VULKAN_INDIRECT_QUADS_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +IndirectQuadsPass::~IndirectQuadsPass() = default; + +std::pair IndirectQuadsPass::Assemble(u32 num_draws, u32 stride, + VkBuffer src_buffer, u32 src_offset) { + u32 src_stride = stride / static_cast(sizeof(u32)); + if (src_stride < COMMAND_WORDS) { + src_stride = COMMAND_WORDS; + } + const u32 input_size = num_draws * src_stride * static_cast(sizeof(u32)); + const std::size_t staging_size = std::size_t(num_draws) * COMMAND_WORDS * sizeof(u32); + const auto staging = staging_buffer_pool.Request(staging_size, MemoryUsage::DeviceLocal); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(src_buffer, src_offset, input_size); + compute_pass_descriptor_queue.AddBuffer(staging.buffer, staging.offset, staging_size); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([this, descriptor_data, num_draws, src_stride](vk::CommandBuffer cmdbuf) { + static constexpr u32 DISPATCH_SIZE = 32; + static constexpr VkMemoryBarrier2 WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT, + .dstAccessMask = VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT, + }; + const std::array push_constants{num_draws, src_stride}; + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push_constants), + &push_constants); + cmdbuf.Dispatch(Common::DivCeil(num_draws, DISPATCH_SIZE), 1, 1); + cmdbuf.PipelineBarrier(WRITE_BARRIER); }); return {staging.buffer, staging.offset}; } @@ -399,17 +487,21 @@ void ConditionalRenderingResolvePass::Resolve(VkBuffer dst_buffer, VkBuffer src_ scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, descriptor_data, compare_to_zero](vk::CommandBuffer cmdbuf) { - static constexpr VkMemoryBarrier read_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + const VkMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, + .srcStageMask = device.GetBufferUserStages(), + .srcAccessMask = vk::ACCESS_BUFFER_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = vk::ACCESS_SHADER_RESOURCES, }; - static constexpr VkMemoryBarrier write_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_CONDITIONAL_RENDERING_BIT_EXT, + .dstAccessMask = VK_ACCESS_2_CONDITIONAL_RENDERING_READ_BIT_EXT, }; const ConditionalRenderingResolvePushConstants uniforms{ .compare_to_zero = compare_to_zero ? 1U : 0U, @@ -417,15 +509,12 @@ void ConditionalRenderingResolvePass::Resolve(VkBuffer dst_buffer, VkBuffer src_ const VkDescriptorSet set = descriptor_allocator.Commit(); device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, read_barrier); + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); cmdbuf.Dispatch(1, 1, 1); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_CONDITIONAL_RENDERING_BIT_EXT, 0, - write_barrier); + cmdbuf.PipelineBarrier(write_barrier); }); } @@ -447,14 +536,22 @@ QueriesPrefixScanPass::QueriesPrefixScanPass( void QueriesPrefixScanPass::Run(VkBuffer accumulation_buffer, VkBuffer dst_buffer, VkBuffer src_buffer, size_t number_of_sums, size_t min_accumulation_limit, size_t max_accumulation_limit) { - constexpr VkAccessFlags BASE_DST_ACCESS = VK_ACCESS_SHADER_READ_BIT | - VK_ACCESS_TRANSFER_READ_BIT | - VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | - VK_ACCESS_INDIRECT_COMMAND_READ_BIT | - VK_ACCESS_INDEX_READ_BIT | - VK_ACCESS_UNIFORM_READ_BIT; - const VkAccessFlags conditional_access = - device.IsExtConditionalRendering() ? VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT : 0; + const VkMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = device.GetBufferUserStages(), + .srcAccessMask = vk::ACCESS_BUFFER_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = vk::ACCESS_SHADER_RESOURCES, + }; + const VkMemoryBarrier2 write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = device.GetBufferConsumerStages(), + .dstAccessMask = device.GetBufferConsumerAccess(), + }; size_t current_runs = number_of_sums; size_t offset = 0; while (current_runs != 0) { @@ -471,19 +568,8 @@ void QueriesPrefixScanPass::Run(VkBuffer accumulation_buffer, VkBuffer dst_buffe scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, descriptor_data, min_accumulation_limit, max_accumulation_limit, - runs_to_do, used_offset, conditional_access](vk::CommandBuffer cmdbuf) { - static constexpr VkMemoryBarrier read_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, - }; - const VkMemoryBarrier write_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = BASE_DST_ACCESS | conditional_access, - }; + runs_to_do, used_offset, read_barrier, + write_barrier](vk::CommandBuffer cmdbuf) { const QueriesPrefixScanPushConstants uniforms{ .min_accumulation_base = static_cast(min_accumulation_limit), .max_accumulation_base = static_cast(max_accumulation_limit), @@ -493,18 +579,97 @@ void QueriesPrefixScanPass::Run(VkBuffer accumulation_buffer, VkBuffer dst_buffe const VkDescriptorSet set = descriptor_allocator.Commit(); device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, read_barrier); + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, uniforms); cmdbuf.Dispatch(1, 1, 1); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, write_barrier); + cmdbuf.PipelineBarrier(write_barrier); }); } } +namespace { + +template +void RecordUnswizzle(Scheduler& scheduler, bool reorder, Func&& func) { + if (reorder) { + scheduler.RecordWithUploadBuffer( + [func = std::forward(func)](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + func(upload_cmdbuf); + }); + return; + } + scheduler.Record(std::forward(func)); +} + +void RecordUnswizzleBeginBarrier(Scheduler& scheduler, bool reorder, VkPipeline vk_pipeline, + VkImage vk_image, VkImageAspectFlags aspect_mask, + bool is_initialized) { + VkAccessFlags2 src_access = VK_ACCESS_2_NONE; + VkImageLayout old_layout = VK_IMAGE_LAYOUT_UNDEFINED; + VkPipelineStageFlags2 src_stage = VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT; + if (is_initialized) { + src_access = vk::ACCESS_IMAGE_WRITES; + old_layout = VK_IMAGE_LAYOUT_GENERAL; + src_stage = vk::PIPELINE_STAGE_IMAGE_USERS; + } + RecordUnswizzle(scheduler, reorder, [vk_pipeline, vk_image, aspect_mask, src_access, + old_layout, src_stage](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier2 image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = src_stage, + .srcAccessMask = src_access, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT, + .oldLayout = old_layout, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(image_barrier); + cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); + }); +} + +void RecordUnswizzleEndBarrier(Scheduler& scheduler, bool reorder, VkImage vk_image, + VkImageAspectFlags aspect_mask) { + RecordUnswizzle(scheduler, reorder, [vk_image, aspect_mask](vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier2 image_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = vk_image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + cmdbuf.PipelineBarrier(image_barrier); + }); +} + +} // Anonymous namespace + ASTCDecoderPass::ASTCDecoderPass(const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, StagingBufferPool& staging_buffer_pool_, @@ -520,43 +685,19 @@ ASTCDecoderPass::ASTCDecoderPass(const Device& device_, Scheduler& scheduler_, ASTCDecoderPass::~ASTCDecoderPass() = default; void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, - std::span swizzles) { + std::span swizzles, + bool reorder) { using namespace VideoCommon::Accelerated; const std::array block_dims{ VideoCore::Surface::DefaultBlockWidth(image.info.format), VideoCore::Surface::DefaultBlockHeight(image.info.format), }; - scheduler.RequestOutsideRenderPassOperationContext(); - const VkPipeline vk_pipeline = *pipeline; + if (!reorder) { + scheduler.RequestOutsideRenderPassOperationContext(); + } const VkImageAspectFlags aspect_mask = image.AspectMask(); const VkImage vk_image = image.Handle(); - const bool is_initialized = image.ExchangeInitialization(); - scheduler.Record([vk_pipeline, vk_image, aspect_mask, - is_initialized](vk::CommandBuffer cmdbuf) { - const VkImageMemoryBarrier image_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = static_cast(is_initialized ? VK_ACCESS_SHADER_WRITE_BIT - : VK_ACCESS_NONE), - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, - .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = vk_image, - .subresourceRange{ - .aspectMask = aspect_mask, - .baseMipLevel = 0, - .levelCount = VK_REMAINING_MIP_LEVELS, - .baseArrayLayer = 0, - .layerCount = VK_REMAINING_ARRAY_LAYERS, - }, - }; - cmdbuf.PipelineBarrier(is_initialized ? vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER - : VkPipelineStageFlags(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT), - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, image_barrier); - cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, vk_pipeline); - }); + RecordUnswizzleBeginBarrier(scheduler, reorder, *pipeline, vk_image, aspect_mask, !reorder); for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { const size_t input_offset = swizzle.buffer_offset + map.offset; const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 8U); @@ -574,8 +715,9 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, ASSERT(params.origin == (std::array{0, 0, 0})); ASSERT(params.destination == (std::array{0, 0, 0})); ASSERT(params.bytes_per_block_log2 == 4); - scheduler.Record([this, num_dispatches_x, num_dispatches_y, num_dispatches_z, block_dims, - params, descriptor_data](vk::CommandBuffer cmdbuf) { + RecordUnswizzle(scheduler, reorder, + [this, num_dispatches_x, num_dispatches_y, num_dispatches_z, block_dims, + params, descriptor_data](vk::CommandBuffer cmdbuf) { const AstcPushConstants uniforms{ .blocks_dims = block_dims, .layer_stride = params.layer_stride, @@ -591,285 +733,172 @@ void ASTCDecoderPass::Assemble(Image& image, const StagingBufferRef& map, cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, num_dispatches_z); }); } - scheduler.Record([vk_image, aspect_mask](vk::CommandBuffer cmdbuf) { - const VkImageMemoryBarrier image_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, - .oldLayout = VK_IMAGE_LAYOUT_GENERAL, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = vk_image, - .subresourceRange{ - .aspectMask = aspect_mask, - .baseMipLevel = 0, - .levelCount = VK_REMAINING_MIP_LEVELS, - .baseArrayLayer = 0, - .layerCount = VK_REMAINING_ARRAY_LAYERS, - }, - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, image_barrier); - }); - scheduler.Finish(); + RecordUnswizzleEndBarrier(scheduler, reorder, vk_image, aspect_mask); } -constexpr u32 BL3D_BINDING_INPUT_BUFFER = 0; -constexpr u32 BL3D_BINDING_OUTPUT_BUFFER = 1; - -struct alignas(16) BlockLinearUnswizzle3DPushConstants { - u32 blocks_dim[3]; // Offset 0 - u32 bytes_per_block_log2; // Offset 12 - - u32 origin[3]; // Offset 16 - u32 slice_size; // Offset 28 - - u32 block_size; // Offset 32 - u32 x_shift; // Offset 36 - u32 block_height; // Offset 40 - u32 block_height_mask; // Offset 44 - - u32 block_depth; // Offset 48 - u32 block_depth_mask; // Offset 52 - s32 _pad; // Offset 56 - - s32 destination[3]; // Offset 60 - s32 _pad_end; // Offset 72 -}; -static_assert(sizeof(BlockLinearUnswizzle3DPushConstants) <= 128); - -BlockLinearUnswizzle3DPass::BlockLinearUnswizzle3DPass( - const Device& device_, Scheduler& scheduler_, - DescriptorPool& descriptor_pool_, +BlockLinearUnswizzleImage2DPass::BlockLinearUnswizzleImage2DPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, StagingBufferPool& staging_buffer_pool_, ComputePassDescriptorQueue& compute_pass_descriptor_queue_) - : ComputePass(device_, scheduler_, descriptor_pool_, - std::array{{ - { - .binding = BL3D_BINDING_INPUT_BUFFER, - .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, // block-linear input - .descriptorCount = 1, - .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, - .pImmutableSamplers = nullptr, - }, - { - .binding = BL3D_BINDING_OUTPUT_BUFFER, - .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, - .descriptorCount = 1, - .stageFlags = VK_SHADER_STAGE_COMPUTE_BIT, - .pImmutableSamplers = nullptr, - }, - }}, - std::array{{ - { - .dstBinding = BL3D_BINDING_INPUT_BUFFER, - .dstArrayElement = 0, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, - .offset = BL3D_BINDING_INPUT_BUFFER * sizeof(DescriptorUpdateEntry), - .stride = sizeof(DescriptorUpdateEntry), - }, - { - .dstBinding = BL3D_BINDING_OUTPUT_BUFFER, - .dstArrayElement = 0, - .descriptorCount = 1, - .descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, - .offset = BL3D_BINDING_OUTPUT_BUFFER * sizeof(DescriptorUpdateEntry), - .stride = sizeof(DescriptorUpdateEntry), - } - }}, - DescriptorBankInfo{ - .uniform_buffers = 0, - .storage_buffers = 2, - .texture_buffers = 0, - .image_buffers = 0, - .textures = 0, - .images = 0, - .score = 2, - }, - COMPUTE_PUSH_CONSTANT_RANGE, - BLOCK_LINEAR_UNSWIZZLE_3D_BCN_COMP_SPV), - scheduler{scheduler_}, - staging_buffer_pool{staging_buffer_pool_}, + : ComputePass(device_, scheduler_, descriptor_pool_, ASTC_DESCRIPTOR_SET_BINDINGS, + ASTC_PASS_DESCRIPTOR_UPDATE_TEMPLATE_ENTRY, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE< + sizeof(VideoCommon::Accelerated::BlockLinearSwizzle2DParams)>, + BLOCK_LINEAR_UNSWIZZLE_2D_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} -BlockLinearUnswizzle3DPass::~BlockLinearUnswizzle3DPass() = default; +BlockLinearUnswizzleImage2DPass::~BlockLinearUnswizzleImage2DPass() = default; -// God have mercy on my soul -void BlockLinearUnswizzle3DPass::Unswizzle( - Image& image, - const StagingBufferRef& swizzled, - std::span swizzles, - u32 z_start, u32 z_count) -{ +void BlockLinearUnswizzleImage2DPass::Unswizzle( + Image& image, const StagingBufferRef& map, + std::span swizzles, bool reorder) { using namespace VideoCommon::Accelerated; - - const u32 MAX_BATCH_SLICES = (std::min)(z_count, image.info.size.depth); - - // Allocate or grow to cover this batch's slice count - image.AllocateComputeUnswizzleBuffer(MAX_BATCH_SLICES); - - ASSERT(swizzles.size() == 1); - const auto& sw = swizzles[0]; - const auto params = MakeBlockLinearSwizzle3DParams(sw, image.info); - - const u32 blocks_x = (image.info.size.width + 3) / 4; - const u32 blocks_y = (image.info.size.height + 3) / 4; - - scheduler.RequestOutsideRenderPassOperationContext(); - for (u32 z_offset = 0; z_offset < z_count; z_offset += MAX_BATCH_SLICES) { - const u32 current_chunk_slices = (std::min)(MAX_BATCH_SLICES, z_count - z_offset); - const u32 current_z_start = z_start + z_offset; - - UnswizzleChunk(image, swizzled, sw, params, blocks_x, blocks_y, - current_z_start, current_chunk_slices); + if (!reorder) { + scheduler.RequestOutsideRenderPassOperationContext(); } + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + RecordUnswizzleBeginBarrier(scheduler, reorder, *pipeline, vk_image, aspect_mask, !reorder); + for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { + const size_t input_offset = swizzle.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 32U); + const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); + const u32 num_dispatches_z = image.info.resources.layers; + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + const auto params = MakeBlockLinearSwizzle2DParams(swizzle, image.info); + RecordUnswizzle(scheduler, reorder, + [this, num_dispatches_x, num_dispatches_y, num_dispatches_z, params, + descriptor_data](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, params); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, num_dispatches_z); + }); + } + RecordUnswizzleEndBarrier(scheduler, reorder, vk_image, aspect_mask); } -void BlockLinearUnswizzle3DPass::UnswizzleChunk( - Image& image, - const StagingBufferRef& swizzled, - const VideoCommon::SwizzleParameters& sw, - const BlockLinearSwizzle3DParams& params, - u32 blocks_x, u32 blocks_y, - u32 z_start, u32 z_count) -{ - BlockLinearUnswizzle3DPushConstants pc{}; - pc.origin[0] = params.origin[0]; - pc.origin[1] = params.origin[1]; - pc.origin[2] = z_start; // Current chunk's Z start +BlockLinearUnswizzleImage3DPass::BlockLinearUnswizzleImage3DPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, ASTC_DESCRIPTOR_SET_BINDINGS, + ASTC_PASS_DESCRIPTOR_UPDATE_TEMPLATE_ENTRY, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + BLOCK_LINEAR_UNSWIZZLE_3D_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} - pc.destination[0] = params.destination[0]; - pc.destination[1] = params.destination[1]; - pc.destination[2] = 0; // Shader writes to start of output buffer +BlockLinearUnswizzleImage3DPass::~BlockLinearUnswizzleImage3DPass() = default; - pc.bytes_per_block_log2 = params.bytes_per_block_log2; - pc.slice_size = params.slice_size; - pc.block_size = params.block_size; - pc.x_shift = params.x_shift; - pc.block_height = params.block_height; - pc.block_height_mask = params.block_height_mask; - pc.block_depth = params.block_depth; - pc.block_depth_mask = params.block_depth_mask; +void BlockLinearUnswizzleImage3DPass::Unswizzle( + Image& image, const StagingBufferRef& map, + std::span swizzles, bool reorder) { + using namespace VideoCommon::Accelerated; + if (!reorder) { + scheduler.RequestOutsideRenderPassOperationContext(); + } + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + RecordUnswizzleBeginBarrier(scheduler, reorder, *pipeline, vk_image, aspect_mask, !reorder); + for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { + const size_t input_offset = swizzle.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 16U); + const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); + const u32 num_dispatches_z = Common::DivCeil(swizzle.num_tiles.depth, 2U); - pc.blocks_dim[0] = blocks_x; - pc.blocks_dim[1] = blocks_y; - pc.blocks_dim[2] = z_count; // Only process the count + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; - compute_pass_descriptor_queue.Acquire(scheduler, 3); - compute_pass_descriptor_queue.AddBuffer(swizzled.buffer, - sw.buffer_offset + swizzled.offset, - image.guest_size_bytes - sw.buffer_offset); - compute_pass_descriptor_queue.AddBuffer(*image.compute_unswizzle_buffer, 0, - image.compute_unswizzle_buffer_size); - - const void* descriptor_data = compute_pass_descriptor_queue.UpdateData(); - const VkDescriptorSet set = descriptor_allocator.Commit(); - - const u32 gx = Common::DivCeil(blocks_x, 8u); - const u32 gy = Common::DivCeil(blocks_y, 8u); - const u32 gz = Common::DivCeil(z_count, 4u); - - const u32 bytes_per_block = 1u << pc.bytes_per_block_log2; - const VkDeviceSize output_slice_size = - static_cast(blocks_x) * blocks_y * bytes_per_block; - const VkDeviceSize barrier_size = output_slice_size * z_count; - - const bool is_first_chunk = (z_start == 0); - - const VkBuffer out_buffer = *image.compute_unswizzle_buffer; - const VkImage dst_image = image.Handle(); - const VkImageAspectFlags aspect = image.AspectMask(); - const u32 image_width = image.info.size.width; - const u32 image_height = image.info.size.height; - - scheduler.Record([this, set, descriptor_data, pc, gx, gy, gz, z_start, z_count, - barrier_size, is_first_chunk, out_buffer, dst_image, aspect, - image_width, image_height - ](vk::CommandBuffer cmdbuf) { - - if (dst_image == VK_NULL_HANDLE || out_buffer == VK_NULL_HANDLE) { - return; - } - - device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); - cmdbuf.BindPipeline(VK_PIPELINE_BIND_POINT_COMPUTE, *pipeline); - cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); - cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pc), &pc); - cmdbuf.Dispatch(gx, gy, gz); - - // Single barrier for compute -> transfer (buffer ready, image transition) - const VkBufferMemoryBarrier buffer_barrier{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .buffer = out_buffer, - .offset = 0, - .size = barrier_size, + const auto p = MakeBlockLinearSwizzle3DParams(swizzle, image.info); + const BlockLinear3DImagePushConstants params{ + .origin = p.origin, + .destination = p.destination, + .bytes_per_block_log2 = p.bytes_per_block_log2, + .slice_size = p.slice_size, + .block_size = p.block_size, + .x_shift = p.x_shift, + .block_height = p.block_height, + .block_height_mask = p.block_height_mask, + .block_depth = p.block_depth, + .block_depth_mask = p.block_depth_mask, }; + RecordUnswizzle(scheduler, reorder, + [this, num_dispatches_x, num_dispatches_y, num_dispatches_z, params, + descriptor_data](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, params); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, num_dispatches_z); + }); + } + RecordUnswizzleEndBarrier(scheduler, reorder, vk_image, aspect_mask); +} - // Image layout transition - const VkImageMemoryBarrier pre_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = is_first_chunk ? VkAccessFlags{} : - static_cast(VK_ACCESS_TRANSFER_WRITE_BIT), - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .oldLayout = is_first_chunk ? VK_IMAGE_LAYOUT_UNDEFINED : - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = dst_image, - .subresourceRange = {aspect, 0, 1, 0, 1}, +PitchUnswizzlePass::PitchUnswizzlePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_) + : ComputePass(device_, scheduler_, descriptor_pool_, ASTC_DESCRIPTOR_SET_BINDINGS, + ASTC_PASS_DESCRIPTOR_UPDATE_TEMPLATE_ENTRY, ASTC_BANK_INFO, + COMPUTE_PUSH_CONSTANT_RANGE, + PITCH_UNSWIZZLE_COMP_SPV), + scheduler{scheduler_}, staging_buffer_pool{staging_buffer_pool_}, + compute_pass_descriptor_queue{compute_pass_descriptor_queue_} {} + +PitchUnswizzlePass::~PitchUnswizzlePass() = default; + +void PitchUnswizzlePass::Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles, + bool reorder) { + const u32 bytes_per_block = VideoCore::Surface::BytesPerBlock(image.info.format); + ASSERT(std::has_single_bit(bytes_per_block)); + if (!reorder) { + scheduler.RequestOutsideRenderPassOperationContext(); + } + const VkImageAspectFlags aspect_mask = image.AspectMask(); + const VkImage vk_image = image.Handle(); + RecordUnswizzleBeginBarrier(scheduler, reorder, *pipeline, vk_image, aspect_mask, !reorder); + for (const VideoCommon::SwizzleParameters& swizzle : swizzles) { + const size_t input_offset = swizzle.buffer_offset + map.offset; + const u32 num_dispatches_x = Common::DivCeil(swizzle.num_tiles.width, 32U); + const u32 num_dispatches_y = Common::DivCeil(swizzle.num_tiles.height, 8U); + + compute_pass_descriptor_queue.Acquire(scheduler, 2); + compute_pass_descriptor_queue.AddBuffer(map.buffer, input_offset, + image.guest_size_bytes - swizzle.buffer_offset); + compute_pass_descriptor_queue.AddImage(image.StorageImageView(swizzle.level)); + const void* const descriptor_data{compute_pass_descriptor_queue.UpdateData()}; + + const PitchUnswizzlePushConstants params{ + .origin = {0, 0}, + .destination = {0, 0}, + .bytes_per_block_log2 = static_cast(std::countr_zero(bytes_per_block)), + .pitch = image.info.pitch, }; - - // Single barrier handles both buffer and image - cmdbuf.PipelineBarrier( - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, - nullptr, buffer_barrier, pre_barrier - ); - - // Copy chunk to correct Z position in image - const VkBufferImageCopy copy{ - .bufferOffset = 0, // Read from start of staging buffer - .bufferRowLength = 0, - .bufferImageHeight = 0, - .imageSubresource = {aspect, 0, 0, 1}, - .imageOffset = {0, 0, static_cast(z_start)}, // Write to correct Z - .imageExtent = {image_width, image_height, z_count}, - }; - cmdbuf.CopyBufferToImage(out_buffer, dst_image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copy); - - // Post-copy transition - const VkImageMemoryBarrier post_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, - .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = dst_image, - .subresourceRange = {aspect, 0, 1, 0, 1}, - }; - - cmdbuf.PipelineBarrier( - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, - nullptr, nullptr, post_barrier - ); - }); + RecordUnswizzle(scheduler, reorder, + [this, num_dispatches_x, num_dispatches_y, params, + descriptor_data](vk::CommandBuffer cmdbuf) { + const VkDescriptorSet set = descriptor_allocator.Commit(); + device.GetLogical().UpdateDescriptorSet(set, *descriptor_template, descriptor_data); + cmdbuf.BindDescriptorSets(VK_PIPELINE_BIND_POINT_COMPUTE, *layout, 0, set, {}); + cmdbuf.PushConstants(*layout, VK_SHADER_STAGE_COMPUTE_BIT, params); + cmdbuf.Dispatch(num_dispatches_x, num_dispatches_y, 1); + }); + } + RecordUnswizzleEndBarrier(scheduler, reorder, vk_image, aspect_mask); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_compute_pass.h b/src/video_core/renderer_vulkan/vk_compute_pass.h index 3d30aa6bdc..7517b65aee 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pass.h +++ b/src/video_core/renderer_vulkan/vk_compute_pass.h @@ -17,7 +17,6 @@ #include "video_core/texture_cache/types.h" #include "video_core/vulkan_common/vulkan_memory_allocator.h" #include "video_core/vulkan_common/vulkan_wrapper.h" -#include "video_core/texture_cache/accelerated_swizzle.h" namespace VideoCommon { struct SwizzleParameters; @@ -25,8 +24,6 @@ struct SwizzleParameters; namespace Vulkan { -using VideoCommon::Accelerated::BlockLinearSwizzle3DParams; - class Device; class StagingBufferPool; class Scheduler; @@ -91,6 +88,25 @@ private: ComputePassDescriptorQueue& compute_pass_descriptor_queue; }; +class IndirectQuadsPass final : public ComputePass { +public: + explicit IndirectQuadsPass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~IndirectQuadsPass(); + + static constexpr u32 COMMAND_WORDS = 5; + + std::pair Assemble(u32 num_draws, u32 stride, VkBuffer src_buffer, + u32 src_offset); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + class ConditionalRenderingResolvePass final : public ComputePass { public: explicit ConditionalRenderingResolvePass( @@ -128,7 +144,7 @@ public: ~ASTCDecoderPass(); void Assemble(Image& image, const StagingBufferRef& map, - std::span swizzles); + std::span swizzles, bool reorder); private: Scheduler& scheduler; @@ -137,26 +153,50 @@ private: MemoryAllocator& memory_allocator; }; -class BlockLinearUnswizzle3DPass final : public ComputePass { +class PitchUnswizzlePass final : public ComputePass { public: - explicit BlockLinearUnswizzle3DPass(const Device& device_, Scheduler& scheduler_, - DescriptorPool& descriptor_pool_, - StagingBufferPool& staging_buffer_pool_, - ComputePassDescriptorQueue& compute_pass_descriptor_queue_); - ~BlockLinearUnswizzle3DPass(); + explicit PitchUnswizzlePass(const Device& device_, Scheduler& scheduler_, + DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~PitchUnswizzlePass(); - void Unswizzle(Image& image, - const StagingBufferRef& swizzled, - std::span swizzles, - u32 z_start, u32 z_count); + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles, bool reorder); - void UnswizzleChunk( - Image& image, - const StagingBufferRef& swizzled, - const VideoCommon::SwizzleParameters& sw, - const BlockLinearSwizzle3DParams& params, - u32 blocks_x, u32 blocks_y, - u32 z_start, u32 z_count); +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzleImage3DPass final : public ComputePass { +public: + explicit BlockLinearUnswizzleImage3DPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzleImage3DPass(); + + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles, bool reorder); + +private: + Scheduler& scheduler; + StagingBufferPool& staging_buffer_pool; + ComputePassDescriptorQueue& compute_pass_descriptor_queue; +}; + +class BlockLinearUnswizzleImage2DPass final : public ComputePass { +public: + explicit BlockLinearUnswizzleImage2DPass( + const Device& device_, Scheduler& scheduler_, DescriptorPool& descriptor_pool_, + StagingBufferPool& staging_buffer_pool_, + ComputePassDescriptorQueue& compute_pass_descriptor_queue_); + ~BlockLinearUnswizzleImage2DPass(); + + void Unswizzle(Image& image, const StagingBufferRef& map, + std::span swizzles, bool reorder); private: Scheduler& scheduler; diff --git a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp index 29356cfcec..840ff5897b 100644 --- a/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_compute_pipeline.cpp @@ -247,10 +247,6 @@ bool ComputePipeline::Configure(Tegra::Engines::KeplerCompute& kepler_compute, buffer_cache.UpdateComputeBuffers(); buffer_cache.BindHostComputeBuffers(); - if (buffer_cache.any_buffer_uploaded) { - buffer_cache.runtime.PostCopyBarrier(); - buffer_cache.any_buffer_uploaded = false; - } RescalingPushConstant rescaling; const VideoCommon::SamplerId* samplers_it{samplers.data()}; diff --git a/src/video_core/renderer_vulkan/vk_descriptor_buffer.h b/src/video_core/renderer_vulkan/vk_descriptor_buffer.h index 6f96bcc326..4cde5abcd4 100644 --- a/src/video_core/renderer_vulkan/vk_descriptor_buffer.h +++ b/src/video_core/renderer_vulkan/vk_descriptor_buffer.h @@ -17,9 +17,9 @@ class Device; class Scheduler; class DescriptorBufferRing final { - static constexpr size_t FRAMES_IN_FLIGHT = 8; - static constexpr VkDeviceSize TILER_FRAME_SIZE = 2 * 1024 * 1024; - static constexpr VkDeviceSize DESKTOP_FRAME_SIZE = 4 * 1024 * 1024; + static constexpr size_t FRAMES_IN_FLIGHT = 3; + static constexpr VkDeviceSize TILER_FRAME_SIZE = 5 * 1024 * 1024; + static constexpr VkDeviceSize DESKTOP_FRAME_SIZE = 8 * 1024 * 1024; public: explicit DescriptorBufferRing(const Device& device_, MemoryAllocator& memory_allocator); diff --git a/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp b/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp index b020d99b93..2f4af7c330 100644 --- a/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_descriptor_pool.cpp @@ -125,7 +125,7 @@ vk::DescriptorSets DescriptorAllocator::AllocateDescriptors(size_t count) { throw vk::Exception(VK_ERROR_OUT_OF_POOL_MEMORY); } -DescriptorPool::DescriptorPool(const Device& device_, Scheduler& scheduler) {} +DescriptorPool::DescriptorPool() = default; DescriptorPool::~DescriptorPool() = default; DescriptorAllocator DescriptorPool::Allocator(const Device& device, Scheduler& scheduler, VkDescriptorSetLayout layout, std::span infos) { diff --git a/src/video_core/renderer_vulkan/vk_descriptor_pool.h b/src/video_core/renderer_vulkan/vk_descriptor_pool.h index 4efeb60373..a524c2a043 100644 --- a/src/video_core/renderer_vulkan/vk_descriptor_pool.h +++ b/src/video_core/renderer_vulkan/vk_descriptor_pool.h @@ -65,7 +65,8 @@ private: class DescriptorPool { public: - explicit DescriptorPool(const Device& device, Scheduler& scheduler); + +DescriptorPool(); ~DescriptorPool(); DescriptorPool& operator=(const DescriptorPool&) = delete; diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp index 8b5b0bc9c5..d2401e42cd 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.cpp @@ -125,22 +125,49 @@ PixelFormat DecodeFormat(u8 encoded_format) { return PixelFormatFromRenderTargetFormat(format); } -RenderPassKey MakeRenderPassKey(const FixedPipelineState& state, const Device& device) { - RenderPassKey key{}; - std::ranges::transform(state.color_formats, key.color_formats.begin(), DecodeFormat); +RenderingFormats PipelineFormats(const FixedPipelineState& state, const Device& device) { + std::array color_formats; + std::ranges::transform(state.color_formats, color_formats.begin(), DecodeFormat); + PixelFormat depth_format = PixelFormat::Invalid; if (state.depth_enabled != 0) { - const auto depth_format{static_cast(state.depth_format.Value())}; - key.depth_format = PixelFormatFromDepthFormat(depth_format); - } else { - key.depth_format = PixelFormat::Invalid; + depth_format = PixelFormatFromDepthFormat( + static_cast(state.depth_format.Value())); } - key.samples = MaxwellToVK::MsaaMode(state.msaa_mode); - const bool has_color = std::ranges::any_of(key.color_formats, [](PixelFormat format) { - return format != PixelFormat::Invalid; - }); - key.resolve_color = - key.samples != VK_SAMPLE_COUNT_1_BIT && has_color && device.IsTiler(); - return key; + return MakeRenderingFormats(device, color_formats, depth_format); +} + +struct AttachmentAccess { + u32 touched; + u32 written; +}; + +AttachmentAccess MakeAttachmentAccess(const Maxwell& regs) { + AttachmentAccess access{}; + if (regs.rasterize_enable == 0) { + return access; + } + for (size_t index = 0; index < Maxwell::NumRenderTargets; ++index) { + size_t mask_index = index; + if (regs.color_mask_common) { + mask_index = 0; + } + const auto& mask = regs.color_mask[mask_index]; + if (mask.R || mask.G || mask.B || mask.A) { + access.touched |= 1u << index; + access.written |= 1u << index; + } + } + if (regs.depth_test_enable || regs.depth_bounds_enable) { + access.touched |= Scheduler::DEPTH_ATTACHMENT_BIT; + } + if (regs.depth_test_enable && regs.depth_write_enabled) { + access.written |= Scheduler::DEPTH_ATTACHMENT_BIT; + } + if (regs.stencil_enable) { + access.touched |= Scheduler::STENCIL_ATTACHMENT_BIT; + access.written |= Scheduler::STENCIL_ATTACHMENT_BIT; + } + return access; } size_t NumAttachments(const FixedPipelineState& state) { @@ -253,8 +280,8 @@ GraphicsPipeline::GraphicsPipeline( const Device& device_, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue_, DescriptorBufferRing& descriptor_buffer_ring_, Common::ThreadWorker* worker_thread, - PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache, - const GraphicsPipelineCacheKey& key_, std::array stages, + PipelineStatistics* pipeline_statistics, const GraphicsPipelineCacheKey& key_, + std::array stages, const std::array& infos) : key{key_}, device{device_}, texture_cache{texture_cache_}, buffer_cache{buffer_cache_}, pipeline_cache(pipeline_cache_), scheduler{scheduler_}, @@ -279,6 +306,11 @@ GraphicsPipeline::GraphicsPipeline( num_descriptor_entries += NumDescriptorEntries(*info); } fragment_has_color0_output = stage_infos[NUM_STAGES - 1].stores_frag_color[0]; + for (size_t index = 0; index < Maxwell::NumVertexAttributes; ++index) { + if (stage_infos[0].loads.Generic(index)) { + vertex_attribute_mask |= 1u << index; + } + } DescriptorLayoutBuilder builder{MakeBuilder(device, stage_infos)}; uses_push_descriptor = builder.CanUsePushDescriptor(); @@ -309,11 +341,10 @@ GraphicsPipeline::GraphicsPipeline( } } - auto func{[this, shader_notify, &render_pass_cache, pipeline_statistics] { - const VkRenderPass render_pass{render_pass_cache.Get(MakeRenderPassKey(key.state, device))}; + auto func{[this, shader_notify, pipeline_statistics] { Validate(); try { - MakePipeline(render_pass); + MakePipeline(PipelineFormats(key.state, device)); } catch (const vk::Exception& exception) { LOG_CRITICAL(Render_Vulkan, "Graphics pipeline build failed: {}", exception.what()); std::scoped_lock lock{build_mutex}; @@ -548,13 +579,17 @@ bool GraphicsPipeline::ConfigureImpl(bool is_indexed) { if constexpr (Spec::enabled_stages[4]) { prepare_stage(4); } - if (buffer_cache.any_buffer_uploaded) { - buffer_cache.runtime.PostCopyBarrier(); - buffer_cache.any_buffer_uploaded = false; - } texture_cache.UpdateRenderTargets(false); - texture_cache.CheckFeedbackLoop(std::span{views.data(), - views.size()}); + const std::span view_span{views.data(), views.size()}; + texture_cache.CheckFeedbackLoop(view_span); + const auto samples_render_pass_image = [&](const VideoCommon::ImageViewInOut& view) { + return view.id && + scheduler.IsRenderPassImage(texture_cache.GetImageView(view.id).ImageHandle()); + }; + if (scheduler.CanAliasRenderPass(texture_cache.GetFramebuffer()) && + std::ranges::any_of(view_span, samples_render_pass_image)) { + scheduler.RequestOutsideRenderPassOperationContext(); + } if (IsBuilt() && !pipeline) { return false; } @@ -595,7 +630,12 @@ bool GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, } } - scheduler.RequestRenderpass(texture_cache.GetFramebuffer()); + if (buffer_cache.TakeDrawHazard()) { + scheduler.RequestOutsideRenderPassOperationContext(); + } + const AttachmentAccess access = MakeAttachmentAccess(maxwell3d->regs); + scheduler.RequestRenderpass(texture_cache.GetFramebuffer(), access.touched, access.written); + buffer_cache.CommitDrawWrites(); if (!is_built.load(std::memory_order::relaxed)) { // Wait for the pipeline to be built scheduler.Record([this](vk::CommandBuffer) { @@ -680,7 +720,7 @@ bool GraphicsPipeline::ConfigureDraw(const RescalingPushConstant& rescaling, return true; } -void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { +void GraphicsPipeline::MakePipeline(const RenderingFormats& formats) { FixedPipelineState::DynamicState dynamic{}; if (!key.state.extended_dynamic_state) { dynamic = key.state.dynamic_state; @@ -705,7 +745,8 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { if (instanced) { vertex_binding_divisors.push_back({ .binding = static_cast(index), - .divisor = key.state.binding_divisors[index], + .divisor = (std::min)(key.state.binding_divisors[index], + device.GetMaxVertexAttribDivisor()), }); } } @@ -739,7 +780,7 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .vertexBindingDivisorCount = static_cast(vertex_binding_divisors.size()), .pVertexBindingDivisors = vertex_binding_divisors.data(), }; - if (!vertex_binding_divisors.empty()) { + if (!vertex_binding_divisors.empty() && device.IsExtVertexAttributeDivisorSupported()) { vertex_input_ci.pNext = &input_divisor_ci; } const bool has_tess_stages = spv_modules[1] || spv_modules[2]; @@ -908,8 +949,8 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .stencilTestEnable = dynamic.stencil_enable, .front = GetStencilFaceState(dynamic.front), .back = GetStencilFaceState(dynamic.back), - .minDepthBounds = f32(key.state.depth_bounds_min), - .maxDepthBounds = f32(key.state.depth_bounds_max), + .minDepthBounds = 0.0f, + .maxDepthBounds = 1.0f, }; if (dynamic.depth_bounds_enable && !device.IsDepthBoundsSupported()) { LOG_WARNING(Render_Vulkan, "Depth bounds is enabled but not supported"); @@ -1073,9 +1114,10 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { flags |= VK_PIPELINE_CREATE_DESCRIPTOR_BUFFER_BIT_EXT; } + const VkPipelineRenderingCreateInfo rendering_ci = formats.CreateInfo(); pipeline = device.GetLogical().CreateGraphicsPipeline({ .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, - .pNext = nullptr, + .pNext = &rendering_ci, .flags = flags, .stageCount = static_cast(shader_stages.size()), .pStages = shader_stages.data(), @@ -1089,7 +1131,6 @@ void GraphicsPipeline::MakePipeline(VkRenderPass render_pass) { .pColorBlendState = &color_blend_ci, .pDynamicState = &dynamic_state_ci, .layout = *pipeline_layout, - .renderPass = render_pass, .subpass = 0, .basePipelineHandle = nullptr, .basePipelineIndex = 0, diff --git a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h index b1ac5a3fcc..ba41dc2deb 100644 --- a/src/video_core/renderer_vulkan/vk_graphics_pipeline.h +++ b/src/video_core/renderer_vulkan/vk_graphics_pipeline.h @@ -67,7 +67,6 @@ namespace Vulkan { class Device; class PipelineStatistics; -class RenderPassCache; class RescalingPushConstant; class RenderAreaPushConstant; class Scheduler; @@ -82,11 +81,14 @@ public: const Device& device, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue, DescriptorBufferRing& descriptor_buffer_ring, Common::ThreadWorker* worker_thread, - PipelineStatistics* pipeline_statistics, RenderPassCache& render_pass_cache, - const GraphicsPipelineCacheKey& key, std::array stages, + PipelineStatistics* pipeline_statistics, const GraphicsPipelineCacheKey& key, + std::array stages, const std::array& infos); bool HasDynamicVertexInput() const noexcept { return key.state.dynamic_vertex_input; } + u32 VertexAttributeMask() const noexcept { + return vertex_attribute_mask; + } bool SupportsAlphaToCoverage() const noexcept { return fragment_has_color0_output; } @@ -140,7 +142,7 @@ private: bool ConfigureDraw(const RescalingPushConstant& rescaling, const RenderAreaPushConstant& render_are); - void MakePipeline(VkRenderPass render_pass); + void MakePipeline(const RenderingFormats& formats); void Validate(); @@ -169,6 +171,7 @@ private: size_t num_image_elements{}; u32 num_textures{}; bool fragment_has_color0_output{}; + u32 vertex_attribute_mask{}; vk::DescriptorSetLayout descriptor_set_layout; DescriptorAllocator descriptor_allocator; diff --git a/src/video_core/renderer_vulkan/vk_guest_memory.cpp b/src/video_core/renderer_vulkan/vk_guest_memory.cpp new file mode 100644 index 0000000000..9b2fd20694 --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_guest_memory.cpp @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#include +#include +#include + +#include "video_core/renderer_vulkan/vk_guest_memory.h" +#include "video_core/renderer_vulkan/vk_scheduler.h" +#include "video_core/vulkan_common/vulkan_memory_allocator.h" + +namespace Vulkan { +namespace { +constexpr size_t PROBE_WORDS = 1024; +constexpr size_t PROBE_SIZE = PROBE_WORDS * sizeof(u32); +constexpr u32 PROBE_CPU_STRIDE = 0x9E3779B9U; +constexpr u32 PROBE_GPU_VALUE = 0x5AC3E10FU; +} + +GuestMemory::GuestMemory([[maybe_unused]] const Device& device, + MemoryAllocator& memory_allocator, Scheduler& scheduler, + [[maybe_unused]] const Common::HostMemory& host_memory) { + if (!windows.empty() && !IsCoherent(memory_allocator, scheduler)) { + windows.clear(); + } +} + +std::optional GuestMemory::Find(const u8* pointer, size_t size) const { + if (windows.empty() || pointer < base) { + return std::nullopt; + } + const size_t offset = static_cast(pointer - base); + const size_t index = offset / window_size; + const size_t local_offset = offset % window_size; + if (index >= windows.size() || window_size - local_offset < size) { + return std::nullopt; + } + return Range{*windows[index].buffer, local_offset}; +} + +bool GuestMemory::IsCoherent(MemoryAllocator& memory_allocator, Scheduler& scheduler) const { + const VkDeviceSize probe_offset = window_size - PROBE_SIZE; + u8* const probe = base + (windows.size() - 1) * window_size + probe_offset; + std::array saved; + std::memcpy(saved.data(), probe, PROBE_SIZE); + std::array pattern; + for (size_t index = 0; index < PROBE_WORDS; ++index) { + pattern[index] = static_cast(index + 1) * PROBE_CPU_STRIDE; + } + std::memcpy(probe, pattern.data(), PROBE_SIZE); + + vk::Buffer readback = memory_allocator.CreateBuffer( + VkBufferCreateInfo{ + .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, + .pNext = nullptr, + .flags = 0, + .size = PROBE_SIZE, + .usage = VK_BUFFER_USAGE_TRANSFER_DST_BIT, + .sharingMode = VK_SHARING_MODE_EXCLUSIVE, + .queueFamilyIndexCount = 0, + .pQueueFamilyIndices = nullptr, + }, + MemoryUsage::Download); + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([src = *windows.back().buffer, dst = *readback, + probe_offset](vk::CommandBuffer cmdbuf) { + static constexpr VkMemoryBarrier2 READ_BEFORE_WRITE{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + }; + static constexpr VkMemoryBarrier2 WRITE_TO_HOST{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_HOST_BIT, + .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT, + }; + const VkBufferCopy2 copy{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, + .srcOffset = probe_offset, + .dstOffset = 0, + .size = PROBE_SIZE, + }; + cmdbuf.CopyBuffer(src, dst, copy); + cmdbuf.PipelineBarrier(READ_BEFORE_WRITE); + cmdbuf.FillBuffer(src, probe_offset, PROBE_SIZE, PROBE_GPU_VALUE); + cmdbuf.PipelineBarrier(WRITE_TO_HOST); + }); + scheduler.Finish(); + + const bool gpu_sees_cpu = + std::memcmp(readback.Mapped().data(), pattern.data(), PROBE_SIZE) == 0; + std::array written; + std::memcpy(written.data(), probe, PROBE_SIZE); + const bool cpu_sees_gpu = + std::ranges::all_of(written, [](u32 word) { return word == PROBE_GPU_VALUE; }); + std::memcpy(probe, saved.data(), PROBE_SIZE); + return gpu_sees_cpu && cpu_sees_gpu; +} + +} diff --git a/src/video_core/renderer_vulkan/vk_guest_memory.h b/src/video_core/renderer_vulkan/vk_guest_memory.h new file mode 100644 index 0000000000..ae206109ad --- /dev/null +++ b/src/video_core/renderer_vulkan/vk_guest_memory.h @@ -0,0 +1,51 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include + +#include "common/common_types.h" +#include "video_core/vulkan_common/vulkan_wrapper.h" + +namespace Common { +class HostMemory; +} + +namespace Vulkan { + +class Device; +class MemoryAllocator; +class Scheduler; + +class GuestMemory { +public: + struct Range { + VkBuffer buffer; + VkDeviceSize offset; + }; + + explicit GuestMemory(const Device& device, MemoryAllocator& memory_allocator, + Scheduler& scheduler, const Common::HostMemory& host_memory); + + [[nodiscard]] std::optional Find(const u8* pointer, size_t size) const; + + [[nodiscard]] bool Empty() const noexcept { + return windows.empty(); + } + +private: + struct Window { + vk::DeviceMemory memory; + vk::ExternalBuffer buffer; + }; + + [[nodiscard]] bool IsCoherent(MemoryAllocator& memory_allocator, Scheduler& scheduler) const; + + std::vector windows; + u8* base{}; + size_t window_size{}; +}; + +} diff --git a/src/video_core/renderer_vulkan/vk_master_semaphore.cpp b/src/video_core/renderer_vulkan/vk_master_semaphore.cpp index be5a1144ce..e52e079969 100644 --- a/src/video_core/renderer_vulkan/vk_master_semaphore.cpp +++ b/src/video_core/renderer_vulkan/vk_master_semaphore.cpp @@ -121,8 +121,9 @@ VkResult MasterSemaphore::SubmitQueue(vk::CommandBuffer& cmdbuf, vk::CommandBuff } } -static constexpr VkPipelineStageFlags wait_stage_mask = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; +static constexpr VkPipelineStageFlags2 wait_stage_mask = + VK_PIPELINE_STAGE_2_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT | + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT; VkResult MasterSemaphore::SubmitQueueTimeline(vk::CommandBuffer& cmdbuf, vk::CommandBuffer& upload_cmdbuf, @@ -130,130 +131,35 @@ VkResult MasterSemaphore::SubmitQueueTimeline(vk::CommandBuffer& cmdbuf, VkSemaphore wait_semaphore, u64 host_tick) { const VkSemaphore timeline_semaphore = *semaphore; - if (device.HasSynchronization2()) { - const std::array cmdbuffer_infos{{ - { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, - .pNext = nullptr, - .commandBuffer = *upload_cmdbuf, - .deviceMask = 0, - }, - { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, - .pNext = nullptr, - .commandBuffer = *cmdbuf, - .deviceMask = 0, - }, - }}; + const std::array cmdbuffer_infos{{ + { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .pNext = nullptr, + .commandBuffer = *upload_cmdbuf, + .deviceMask = 0, + }, + { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .pNext = nullptr, + .commandBuffer = *cmdbuf, + .deviceMask = 0, + }, + }}; - std::array signal_infos{{ - { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, - .pNext = nullptr, - .semaphore = timeline_semaphore, - .value = host_tick, - .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - .deviceIndex = 0, - }, - {}, - }}; - u32 num_signal_semaphores = 1; - if (signal_semaphore) { - signal_infos[1] = VkSemaphoreSubmitInfo{ - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, - .pNext = nullptr, - .semaphore = signal_semaphore, - .value = 0, - .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - .deviceIndex = 0, - }; - num_signal_semaphores = 2; - } - - const u32 num_wait_semaphores = wait_semaphore ? 1 : 0; - const VkSemaphoreSubmitInfo wait_info{ + std::array signal_infos{{ + { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .pNext = nullptr, - .semaphore = wait_semaphore, - .value = 0, - .stageMask = static_cast(wait_stage_mask), + .semaphore = timeline_semaphore, + .value = host_tick, + .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .deviceIndex = 0, - }; - - const VkSubmitInfo2 submit_info2{ - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, - .pNext = nullptr, - .flags = 0, - .waitSemaphoreInfoCount = num_wait_semaphores, - .pWaitSemaphoreInfos = num_wait_semaphores ? &wait_info : nullptr, - .commandBufferInfoCount = static_cast(cmdbuffer_infos.size()), - .pCommandBufferInfos = cmdbuffer_infos.data(), - .signalSemaphoreInfoCount = num_signal_semaphores, - .pSignalSemaphoreInfos = signal_infos.data(), - }; - return device.GetGraphicsQueue().Submit2(submit_info2); - } - - const u32 num_signal_semaphores = signal_semaphore ? 2 : 1; - const std::array signal_values{host_tick, u64(0)}; - const std::array signal_semaphores{timeline_semaphore, signal_semaphore}; - - const std::array cmdbuffers{*upload_cmdbuf, *cmdbuf}; - - const u32 num_wait_semaphores = wait_semaphore ? 1 : 0; - // Pointers must be null when the count is zero (best-practices) - const VkSemaphore* p_wait_sems = - (num_wait_semaphores > 0) ? &wait_semaphore : nullptr; - const VkPipelineStageFlags* p_wait_masks = - (num_wait_semaphores > 0) ? &wait_stage_mask : nullptr; - const VkSemaphore* p_signal_sems = - (num_signal_semaphores > 0) ? signal_semaphores.data() : nullptr; - const u64 wait_zero = 0; // dummy for binary wait - const VkTimelineSemaphoreSubmitInfo timeline_si{ - .sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO, - .pNext = nullptr, - .waitSemaphoreValueCount = num_wait_semaphores, - .pWaitSemaphoreValues = num_wait_semaphores ? &wait_zero : nullptr, - .signalSemaphoreValueCount = num_signal_semaphores, - .pSignalSemaphoreValues = signal_values.data(), - }; - const VkSubmitInfo submit_info{ - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, - .pNext = &timeline_si, - .waitSemaphoreCount = num_wait_semaphores, - .pWaitSemaphores = p_wait_sems, - .pWaitDstStageMask = p_wait_masks, - .commandBufferCount = static_cast(cmdbuffers.size()), - .pCommandBuffers = cmdbuffers.data(), - .signalSemaphoreCount = num_signal_semaphores, - .pSignalSemaphores = p_signal_sems, - }; - - return device.GetGraphicsQueue().Submit(submit_info); -} - -VkResult MasterSemaphore::SubmitQueueFence(vk::CommandBuffer& cmdbuf, - vk::CommandBuffer& upload_cmdbuf, - VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, - u64 host_tick) { - if (device.HasSynchronization2()) { - const std::array cmdbuffer_infos{{ - { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, - .pNext = nullptr, - .commandBuffer = *upload_cmdbuf, - .deviceMask = 0, - }, - { - .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, - .pNext = nullptr, - .commandBuffer = *cmdbuf, - .deviceMask = 0, - }, - }}; - - const u32 num_signal_semaphores = signal_semaphore ? 1 : 0; - const VkSemaphoreSubmitInfo signal_info{ + }, + {}, + }}; + u32 num_signal_semaphores = 1; + if (signal_semaphore) { + signal_infos[1] = VkSemaphoreSubmitInfo{ .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .pNext = nullptr, .semaphore = signal_semaphore, @@ -261,66 +167,86 @@ VkResult MasterSemaphore::SubmitQueueFence(vk::CommandBuffer& cmdbuf, .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, .deviceIndex = 0, }; - - const u32 num_wait_semaphores = wait_semaphore ? 1 : 0; - const VkSemaphoreSubmitInfo wait_info{ - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, - .pNext = nullptr, - .semaphore = wait_semaphore, - .value = 0, - .stageMask = static_cast(wait_stage_mask), - .deviceIndex = 0, - }; - - const VkSubmitInfo2 submit_info2{ - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, - .pNext = nullptr, - .flags = 0, - .waitSemaphoreInfoCount = num_wait_semaphores, - .pWaitSemaphoreInfos = num_wait_semaphores ? &wait_info : nullptr, - .commandBufferInfoCount = static_cast(cmdbuffer_infos.size()), - .pCommandBufferInfos = cmdbuffer_infos.data(), - .signalSemaphoreInfoCount = num_signal_semaphores, - .pSignalSemaphoreInfos = num_signal_semaphores ? &signal_info : nullptr, - }; - - auto fence = GetFreeFence(); - auto result = device.GetGraphicsQueue().Submit2(submit_info2, *fence); - - if (result == VK_SUCCESS) { - std::scoped_lock lock{wait_mutex}; - wait_queue.emplace(host_tick, std::move(fence)); - wait_cv.notify_one(); - } - - return result; + num_signal_semaphores = 2; } - const u32 num_signal_semaphores = signal_semaphore ? 1 : 0; const u32 num_wait_semaphores = wait_semaphore ? 1 : 0; - - const VkSemaphore* p_wait_sems = - (num_wait_semaphores > 0) ? &wait_semaphore : nullptr; - const VkPipelineStageFlags* p_wait_masks = - (num_wait_semaphores > 0) ? &wait_stage_mask : nullptr; - const VkSemaphore* p_signal_sems = - (num_signal_semaphores > 0) ? &signal_semaphore : nullptr; - const std::array cmdbuffers{*upload_cmdbuf, *cmdbuf}; - - const VkSubmitInfo submit_info{ - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + const VkSemaphoreSubmitInfo wait_info{ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .pNext = nullptr, - .waitSemaphoreCount = num_wait_semaphores, - .pWaitSemaphores = p_wait_sems, - .pWaitDstStageMask = p_wait_masks, - .commandBufferCount = static_cast(cmdbuffers.size()), - .pCommandBuffers = cmdbuffers.data(), - .signalSemaphoreCount = num_signal_semaphores, - .pSignalSemaphores = p_signal_sems, + .semaphore = wait_semaphore, + .value = 0, + .stageMask = wait_stage_mask, + .deviceIndex = 0, + }; + + const VkSubmitInfo2 submit_info2{ + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, + .pNext = nullptr, + .flags = 0, + .waitSemaphoreInfoCount = num_wait_semaphores, + .pWaitSemaphoreInfos = num_wait_semaphores ? &wait_info : nullptr, + .commandBufferInfoCount = static_cast(cmdbuffer_infos.size()), + .pCommandBufferInfos = cmdbuffer_infos.data(), + .signalSemaphoreInfoCount = num_signal_semaphores, + .pSignalSemaphoreInfos = signal_infos.data(), + }; + return device.GetGraphicsQueue().Submit(submit_info2); +} + +VkResult MasterSemaphore::SubmitQueueFence(vk::CommandBuffer& cmdbuf, + vk::CommandBuffer& upload_cmdbuf, + VkSemaphore signal_semaphore, VkSemaphore wait_semaphore, + u64 host_tick) { + const std::array cmdbuffer_infos{{ + { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .pNext = nullptr, + .commandBuffer = *upload_cmdbuf, + .deviceMask = 0, + }, + { + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .pNext = nullptr, + .commandBuffer = *cmdbuf, + .deviceMask = 0, + }, + }}; + + const u32 num_signal_semaphores = signal_semaphore ? 1 : 0; + const VkSemaphoreSubmitInfo signal_info{ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .semaphore = signal_semaphore, + .value = 0, + .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .deviceIndex = 0, + }; + + const u32 num_wait_semaphores = wait_semaphore ? 1 : 0; + const VkSemaphoreSubmitInfo wait_info{ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .semaphore = wait_semaphore, + .value = 0, + .stageMask = wait_stage_mask, + .deviceIndex = 0, + }; + + const VkSubmitInfo2 submit_info2{ + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, + .pNext = nullptr, + .flags = 0, + .waitSemaphoreInfoCount = num_wait_semaphores, + .pWaitSemaphoreInfos = num_wait_semaphores ? &wait_info : nullptr, + .commandBufferInfoCount = static_cast(cmdbuffer_infos.size()), + .pCommandBufferInfos = cmdbuffer_infos.data(), + .signalSemaphoreInfoCount = num_signal_semaphores, + .pSignalSemaphoreInfos = num_signal_semaphores ? &signal_info : nullptr, }; auto fence = GetFreeFence(); - auto result = device.GetGraphicsQueue().Submit(submit_info, *fence); + auto result = device.GetGraphicsQueue().Submit(submit_info2, *fence); if (result == VK_SUCCESS) { std::scoped_lock lock{wait_mutex}; @@ -353,7 +279,7 @@ void MasterSemaphore::WaitThread(std::stop_token token) { free_queue.push_front(std::move(fence)); gpu_tick.store(host_tick, std::memory_order_release); } - gpu_tick.notify_one(); + gpu_tick.notify_all(); } } diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp index 4d84390002..68dcab9250 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.cpp @@ -32,8 +32,6 @@ MultiRangeBufferCache::MultiRangeBufferCache(const Device& device) { VkDeviceSize MultiRangeBufferCache::QueryBlockSize(const Device& device, u32& memory_type_bits) const { - const VkDevice logical = *device.GetLogical(); - const auto& dld = device.GetDispatchLoader(); const VkBufferCreateInfo probe_ci{ .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, .pNext = nullptr, @@ -44,6 +42,14 @@ VkDeviceSize MultiRangeBufferCache::QueryBlockSize(const Device& device, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, }; + if (device.IsKhrMaintenance4Supported()) { + const VkMemoryRequirements reqs = + device.GetLogical().GetDeviceBufferMemoryRequirements(probe_ci); + memory_type_bits = reqs.memoryTypeBits; + return reqs.alignment; + } + const VkDevice logical = *device.GetLogical(); + const auto& dld = device.GetDispatchLoader(); VkBuffer probe{}; if (dld.vkCreateBuffer(logical, &probe_ci, nullptr, &probe) != VK_SUCCESS) { return 0; @@ -81,7 +87,7 @@ u64 MultiRangeBufferCache::HashSources(std::span sources u64 MultiRangeBufferCache::HashContent(std::span sources) const { u64 hash = 0xcbf29ce484222325ULL; for (const MultiRangeSource& source : sources) { - hash ^= source.write_tick; + hash ^= source.content_serial; hash *= 0x100000001b3ULL; } return hash; @@ -213,9 +219,6 @@ MultiRangeRef MultiRangeBufferCache::Get(const Device& device, Scheduler& schedu if (sources.empty() || total == 0) { return MultiRangeRef{}; } - if (!retired.empty()) { - DrainRetired(scheduler); - } const u64 geometry = HashSources(sources); const u64 content = HashContent(sources); const auto it = entries.find(key); @@ -329,10 +332,4 @@ void MultiRangeBufferCache::DropOwner(Scheduler& scheduler, VkBuffer owner) { } } -void MultiRangeBufferCache::Invalidate(u64 key) { - if (auto const it = entries.find(key); it != entries.end()) { - it->second.dirty = true; - } -} - } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h index f40d3aa1f8..d9c065df3f 100644 --- a/src/video_core/renderer_vulkan/vk_multi_range_buffer.h +++ b/src/video_core/renderer_vulkan/vk_multi_range_buffer.h @@ -27,7 +27,7 @@ struct MultiRangeSource { VkDeviceSize memory_offset{}; VkDeviceSize offset{}; VkDeviceSize size{}; - u64 write_tick{}; + u64 content_serial{}; u32 memory_type{}; }; @@ -55,10 +55,10 @@ public: void MarkGathered(u64 key); - void Invalidate(u64 key); - void DropOwner(Scheduler& scheduler, VkBuffer owner); + void DrainRetired(Scheduler& scheduler); + VkDeviceSize block_size{DEFAULT_BLOCK_SIZE}; bool use_sparse{}; @@ -94,8 +94,6 @@ private: void RetireEntry(Scheduler& scheduler, Entry& entry); - void DrainRetired(Scheduler& scheduler); - ::Common::unordered_map entries; boost::container::static_vector retired; u32 sparse_memory_type_bits{}; diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp index 642b9fa223..dd55b576e8 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.cpp @@ -62,7 +62,7 @@ using VideoCommon::FileEnvironment; using VideoCommon::GenericEnvironment; using VideoCommon::GraphicsEnvironment; -constexpr u32 CACHE_VERSION = 18; +constexpr u32 CACHE_VERSION = 20; constexpr size_t VULKAN_CACHE_FLUSH_PIPELINES = 128; constexpr size_t VULKAN_CACHE_FLUSH_MIN_SECONDS = 30; constexpr std::array VULKAN_CACHE_MAGIC_NUMBER{'y', 'u', 'z', 'u', 'v', 'k', 'c', 'h'}; @@ -343,12 +343,11 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, DescriptorPool& descriptor_pool_, GuestDescriptorQueue& guest_descriptor_queue_, DescriptorBufferRing& descriptor_buffer_ring_, - RenderPassCache& render_pass_cache_, BufferCache& buffer_cache_, - TextureCache& texture_cache_, VideoCore::ShaderNotify& shader_notify_) + BufferCache& buffer_cache_, TextureCache& texture_cache_, + VideoCore::ShaderNotify& shader_notify_) : VideoCommon::ShaderCache{device_memory_}, device{device_}, scheduler{scheduler_}, descriptor_pool{descriptor_pool_}, guest_descriptor_queue{guest_descriptor_queue_}, - descriptor_buffer_ring{descriptor_buffer_ring_}, - render_pass_cache{render_pass_cache_}, buffer_cache{buffer_cache_}, + descriptor_buffer_ring{descriptor_buffer_ring_}, buffer_cache{buffer_cache_}, texture_cache{texture_cache_}, shader_notify{shader_notify_}, use_asynchronous_shaders{Settings::values.use_asynchronous_shaders.GetValue()}, use_vulkan_pipeline_cache{Settings::values.use_vulkan_driver_pipeline_cache.GetValue()}, @@ -362,7 +361,7 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const auto subgroup_stage_bit{[subgroup_stages](VkShaderStageFlags flag, Shader::Stage stage) { return (subgroup_stages & flag) != 0 ? (1u << static_cast(stage)) : 0u; }}; - const u32 supported_subgroup_stages{ + u32 supported_subgroup_stages{ subgroup_stage_bit(VK_SHADER_STAGE_VERTEX_BIT, Shader::Stage::VertexA) | subgroup_stage_bit(VK_SHADER_STAGE_VERTEX_BIT, Shader::Stage::VertexB) | subgroup_stage_bit(VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT, @@ -372,6 +371,9 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, subgroup_stage_bit(VK_SHADER_STAGE_GEOMETRY_BIT, Shader::Stage::Geometry) | subgroup_stage_bit(VK_SHADER_STAGE_FRAGMENT_BIT, Shader::Stage::Fragment) | subgroup_stage_bit(VK_SHADER_STAGE_COMPUTE_BIT, Shader::Stage::Compute)}; + if (driver_id == VK_DRIVER_ID_MESA_TURNIP) { + supported_subgroup_stages &= ~(1u << static_cast(Shader::Stage::Geometry)); + } profile = Shader::Profile{ .supported_spirv = device.SupportedSpirvVersion(), .unified_descriptor_binding = true, @@ -409,6 +411,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, .support_shader_quad_control = device.IsKhrShaderQuadControlSupported(), .support_quad_shuffles = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_QUAD_BIT), .support_vote = device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_VOTE_BIT), + .support_shuffle_relative = + device.IsSubgroupFeatureSupported(VK_SUBGROUP_FEATURE_SHUFFLE_RELATIVE_BIT), .supported_subgroup_stages = supported_subgroup_stages, .support_viewport_index_layer_non_geometry = device.IsExtShaderViewportIndexLayerSupported(), @@ -444,13 +448,17 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, driver_id == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA, .has_broken_spirv_clamp = driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, - .has_broken_spirv_position_input = driver_id == false, + .has_broken_spirv_position_input = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY, .has_broken_unsigned_image_offsets = false, .has_broken_signed_operations = false, .has_broken_fp16_float_controls = driver_id == VK_DRIVER_ID_NVIDIA_PROPRIETARY, .has_broken_fp32_denorm_flush = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY, .ignore_nan_fp_comparisons = false, - .has_broken_spirv_subgroup_mask_vector_extract_dynamic = false, + .has_broken_spirv_subgroup_mask_vector_extract_dynamic = + driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY && + device.GetDriverVersion() < VK_MAKE_VERSION(512, 672, 0), + .has_broken_spirv_subgroup_shuffle = driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY, + .max_subgroup_size = device.GetMaxSubgroupSize(), .has_broken_robust = device.IsNvidia() && device.GetNvidiaArch() <= NvidiaArchitecture::Arch_Pascal, .min_ssbo_alignment = device.GetStorageBufferAlignment(), @@ -477,8 +485,10 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, driver_id == VK_DRIVER_ID_SAMSUNG_PROPRIETARY, .support_snorm_render_buffer = true, .support_viewport_index_layer = device.IsExtShaderViewportIndexLayerSupported(), + .support_viewport_mask = device.IsNvViewportArray2Supported(), .support_geometry_shader_passthrough = device.IsNvGeometryShaderPassthroughSupported(), .support_conditional_barrier = device.SupportsConditionalBarriers(), + .single_lane_geometry_subgroups = !profile.SupportsSubgroupStage(Shader::Stage::Geometry), }; host_info.ApplyDescriptorLimitPolicy(); @@ -541,6 +551,8 @@ PipelineCache::PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, device.SupportsProvokingVertexLastMode(); dynamic_features.has_provoking_vertex_tf_preserve = device.SupportsTransformFeedbackProvokingVertexPreservation(); + dynamic_features.has_dynamic_rendering_unused_attachments = + device.IsExtDynamicRenderingUnusedAttachmentsSupported(); } PipelineCache::~PipelineCache() { @@ -556,7 +568,7 @@ GraphicsPipeline* PipelineCache::CurrentGraphicsPipeline() { current_pipeline = nullptr; return nullptr; } - graphics_key.state.Refresh(*maxwell3d, dynamic_features); + graphics_key.state.Refresh(*maxwell3d, dynamic_features, fragment_color_outputs); if (current_pipeline) { GraphicsPipeline* const next{current_pipeline->Next(graphics_key)}; @@ -886,7 +898,7 @@ std::unique_ptr PipelineCache::CreateGraphicsPipeline( return std::make_unique( scheduler, buffer_cache, texture_cache, vulkan_pipeline_cache, &shader_notify, device, descriptor_pool, guest_descriptor_queue, descriptor_buffer_ring, thread_worker, statistics, - render_pass_cache, key, std::move(modules), infos); + key, std::move(modules), infos); } catch (const Shader::Exception& exception) { auto hash = key.Hash(); diff --git a/src/video_core/renderer_vulkan/vk_pipeline_cache.h b/src/video_core/renderer_vulkan/vk_pipeline_cache.h index 71dfb46e3d..bebac7cb15 100644 --- a/src/video_core/renderer_vulkan/vk_pipeline_cache.h +++ b/src/video_core/renderer_vulkan/vk_pipeline_cache.h @@ -85,7 +85,6 @@ class ComputePipeline; class DescriptorPool; class Device; class PipelineStatistics; -class RenderPassCache; class Scheduler; using VideoCommon::ShaderInfo; @@ -107,8 +106,7 @@ public: explicit PipelineCache(Tegra::MaxwellDeviceMemoryManager& device_memory_, const Device& device, Scheduler& scheduler, DescriptorPool& descriptor_pool, GuestDescriptorQueue& guest_descriptor_queue, - DescriptorBufferRing& descriptor_buffer_ring, - RenderPassCache& render_pass_cache, BufferCache& buffer_cache, + DescriptorBufferRing& descriptor_buffer_ring, BufferCache& buffer_cache, TextureCache& texture_cache, VideoCore::ShaderNotify& shader_notify_); ~PipelineCache(); @@ -153,7 +151,6 @@ private: DescriptorPool& descriptor_pool; GuestDescriptorQueue& guest_descriptor_queue; DescriptorBufferRing& descriptor_buffer_ring; - RenderPassCache& render_pass_cache; BufferCache& buffer_cache; TextureCache& texture_cache; VideoCore::ShaderNotify& shader_notify; diff --git a/src/video_core/renderer_vulkan/vk_present_manager.cpp b/src/video_core/renderer_vulkan/vk_present_manager.cpp index d173a3ba5f..52d497e446 100644 --- a/src/video_core/renderer_vulkan/vk_present_manager.cpp +++ b/src/video_core/renderer_vulkan/vk_present_manager.cpp @@ -7,9 +7,6 @@ #include "common/settings.h" #include "common/thread.h" #include "core/frontend/emu_window.h" -#ifdef HAS_LSFG -#include "video_core/renderer_vulkan/present/lsfg_common.h" -#endif #include "video_core/renderer_vulkan/vk_present_manager.h" #include "video_core/renderer_vulkan/vk_scheduler.h" #include "video_core/renderer_vulkan/vk_swapchain.h" @@ -23,18 +20,7 @@ namespace Vulkan { namespace { constexpr size_t MAX_FRAMES_IN_FLIGHT = 7; -#ifdef HAS_LSFG -static_assert(MAX_FRAMES_IN_FLIGHT <= LSFG_MAX_TARGETS); -#endif - -bool CanStoreToFrame(const vk::PhysicalDevice& physical_device, VkFormat format) { -#ifdef HAS_LSFG - const VkFormatProperties props{physical_device.GetFormatProperties(format)}; - return (props.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; -#else - return false; -#endif -} +constexpr u32 MAX_PRESENT_ATTEMPTS = 3; bool CanBlitToSwapchain(const vk::PhysicalDevice& physical_device, VkFormat format) { const VkFormatProperties props{physical_device.GetFormatProperties(format)}; @@ -50,9 +36,11 @@ bool CanBlitToSwapchain(const vk::PhysicalDevice& physical_device, VkFormat form }; } -[[nodiscard]] VkImageBlit MakeImageBlit(s32 frame_width, s32 frame_height, s32 swapchain_width, - s32 swapchain_height) { - return VkImageBlit{ +[[nodiscard]] VkImageBlit2 MakeImageBlit(s32 frame_width, s32 frame_height, s32 swapchain_width, + s32 swapchain_height) { + return VkImageBlit2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_BLIT_2, + .pNext = nullptr, .srcSubresource = MakeImageSubresourceLayers(), .srcOffsets = { @@ -84,9 +72,11 @@ bool CanBlitToSwapchain(const vk::PhysicalDevice& physical_device, VkFormat form }; } -[[nodiscard]] VkImageCopy MakeImageCopy(u32 frame_width, u32 frame_height, u32 swapchain_width, - u32 swapchain_height) { - return VkImageCopy{ +[[nodiscard]] VkImageCopy2 MakeImageCopy(u32 frame_width, u32 frame_height, u32 swapchain_width, + u32 swapchain_height) { + return VkImageCopy2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2, + .pNext = nullptr, .srcSubresource = MakeImageSubresourceLayers(), .srcOffset = { @@ -127,8 +117,8 @@ PresentManager::PresentManager(const vk::Instance& instance_, , swapchain{swapchain_} , surface{surface_} , blit_supported{CanBlitToSwapchain(device.GetPhysical(), swapchain.GetImageViewFormat())} - , storage_supported{CanStoreToFrame(device.GetPhysical(), swapchain.GetImageFormat())} - , use_present_thread{Settings::values.async_presentation.GetValue()} + , use_present_thread{Settings::values.async_presentation.GetValue() || + Settings::values.frame_gen.GetValue()} { SetImageCount(); @@ -157,7 +147,6 @@ PresentManager::PresentManager(const vk::Instance& instance_, .pNext = nullptr, .flags = VK_FENCE_CREATE_SIGNALED_BIT, }); - frame.storage_capable = storage_supported; free_queue.push_back(&frame); } @@ -185,7 +174,20 @@ Frame* PresentManager::GetRenderFrame() { return frame; } -void PresentManager::Present(Frame* frame) { +Frame* PresentManager::TryGetRenderFrame() { + std::scoped_lock lock{free_mutex}; + if (free_queue.size() < 2 || free_queue.front()->present_done.GetStatus() != VK_SUCCESS) { + return nullptr; + } + Frame* const frame = free_queue.front(); + free_queue.pop_front(); + frame->present_done.Reset(); + return frame; +} + +void PresentManager::Present(Frame* frame, u32 pace_index, std::chrono::nanoseconds pace_step) { + frame->pace_index = pace_index; + frame->pace_step = pace_step; if (use_present_thread) { scheduler.Record([this, frame](vk::CommandBuffer) { std::unique_lock lock{queue_mutex}; @@ -203,23 +205,13 @@ size_t PresentManager::MaxExtraFrames() const { return image_count - 1; } -bool PresentManager::NeedsStorage(const Frame* frame, bool required) const { - return required && frame->storage_capable && !frame->storage_view; -} - -void PresentManager::RecreateFrame(Frame* frame, u32 width, u32 height, VkFormat image_view_format, - VkRenderPass rd, bool storage) { +void PresentManager::RecreateFrame(Frame* frame, u32 width, u32 height, + VkFormat image_view_format) { auto& dld = device.GetLogical(); frame->width = width; frame->height = height; - const bool with_storage = storage && frame->storage_capable; - VkImageUsageFlags storage_usage = 0; - if (with_storage) { - storage_usage = VK_IMAGE_USAGE_STORAGE_BIT; - } - frame->image = memory_allocator.CreateImage({ .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, .pNext = nullptr, @@ -237,7 +229,7 @@ void PresentManager::RecreateFrame(Frame* frame, u32 width, u32 height, VkFormat .samples = VK_SAMPLE_COUNT_1_BIT, .tiling = VK_IMAGE_TILING_OPTIMAL, .usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT | - VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | storage_usage, + VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -267,46 +259,6 @@ void PresentManager::RecreateFrame(Frame* frame, u32 width, u32 height, VkFormat .layerCount = 1, }, }); - - frame->storage_view = vk::ImageView{}; - if (with_storage) { - frame->storage_view = dld.CreateImageView({ - .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .image = *frame->image, - .viewType = VK_IMAGE_VIEW_TYPE_2D, - .format = swapchain.GetImageFormat(), - .components = - { - .r = VK_COMPONENT_SWIZZLE_IDENTITY, - .g = VK_COMPONENT_SWIZZLE_IDENTITY, - .b = VK_COMPONENT_SWIZZLE_IDENTITY, - .a = VK_COMPONENT_SWIZZLE_IDENTITY, - }, - .subresourceRange = - { - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .baseMipLevel = 0, - .levelCount = 1, - .baseArrayLayer = 0, - .layerCount = 1, - }, - }); - } - - const VkImageView image_view{*frame->image_view}; - frame->framebuffer = dld.CreateFramebuffer({ - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .renderPass = rd, - .attachmentCount = 1, - .pAttachments = &image_view, - .width = width, - .height = height, - .layers = 1, - }); } void PresentManager::WaitPresent() { @@ -330,21 +282,33 @@ void PresentManager::PresentThread(std::stop_token token) { Common::SetCurrentThreadName("VulkanPresent"); Common::SetCurrentThreadPriority(Common::ThreadPriority::High); Common::SetCurrentThreadToPerformanceCores(); + std::chrono::steady_clock::time_point pace_anchor{}; while (!token.stop_requested()) { std::unique_lock lock{queue_mutex}; // Wait for presentation frames frame_cv.wait(lock, token, [this] { return !present_queue.empty(); }); + if (!present_queue.empty() && present_queue.front()->pace_index != 0) { + const Frame* const held = present_queue.front(); + const auto next_anchor = [this] { + return std::any_of(present_queue.begin() + 1, present_queue.end(), + [](const Frame* next) { return next->pace_index == 0; }); + }; + void(frame_cv.wait_until(lock, token, + pace_anchor + held->pace_step * held->pace_index, + next_anchor)); + } if (!token.stop_requested()) { // Take the frame and notify anyone waiting Frame* frame = present_queue.front(); present_queue.pop_front(); frame_cv.notify_one(); - // By exchanging the lock ownership we take the swapchain lock - // before the queue lock goes out of scope. This way the swapchain - // lock in WaitPresent is guaranteed to occur after here. void(std::exchange(lock, std::unique_lock{swapchain_mutex})); CopyToSwapchain(frame); + if (frame->pace_index == 0 && frame->pace_step.count() != 0) { + frame->present_done.Wait(static_cast(frame->pace_step.count())); + pace_anchor = std::chrono::steady_clock::now(); + } // Free the frame for reuse std::scoped_lock fl{free_mutex}; @@ -366,18 +330,45 @@ void PresentManager::SetImageCount() { #ifdef HAS_LSFG const size_t generations = Settings::FrameGenMaxGenerations(); const size_t queued_composites = Settings::values.frame_gen_queue_target.GetValue() + 1; - image_count = - std::clamp((generations + 1) * queued_composites, swapchain.GetImageCount(), - MAX_FRAMES_IN_FLIGHT); + const size_t baseline = swapchain.GetImageCount() + generations; + image_count = std::min((std::max)((generations + 1) * queued_composites, baseline), + MAX_FRAMES_IN_FLIGHT); #else image_count = std::min(swapchain.GetImageCount(), MAX_FRAMES_IN_FLIGHT); #endif + swapchain_image_count = swapchain.GetImageCount(); + swapchain_image_format = swapchain.GetImageFormat(); +} + +void PresentManager::DiscardFrame(Frame* frame) { + const VkSemaphoreSubmitInfo wait_info{ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .semaphore = *frame->render_ready, + .value = 0, + .stageMask = VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, + .deviceIndex = 0, + }; + const VkSubmitInfo2 submit_info{ + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, + .pNext = nullptr, + .flags = 0, + .waitSemaphoreInfoCount = 1U, + .pWaitSemaphoreInfos = &wait_info, + .commandBufferInfoCount = 0U, + .pCommandBufferInfos = nullptr, + .signalSemaphoreInfoCount = 0U, + .pSignalSemaphoreInfos = nullptr, + }; + + std::scoped_lock submit_lock{scheduler.submit_mutex}; + void(device.GetGraphicsQueue().Submit(submit_info, *frame->present_done)); } void PresentManager::CopyToSwapchain(Frame* frame) { bool requires_recreation = false; - while (true) { + for (u32 attempt = 0; attempt < MAX_PRESENT_ATTEMPTS; ++attempt) { try { // Recreate surface and swapchain if needed. if (requires_recreation) { @@ -397,6 +388,8 @@ void PresentManager::CopyToSwapchain(Frame* frame) { requires_recreation = true; } } + + DiscardFrame(frame); } void PresentManager::CopyToSwapchainImpl(Frame* frame) { @@ -425,11 +418,13 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { const VkImage image{swapchain.CurrentImage()}; const VkExtent2D extent = swapchain.GetExtent(); const std::array pre_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -443,11 +438,13 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -463,11 +460,13 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { }, }; const std::array post_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_NONE, + .dstAccessMask = VK_ACCESS_2_NONE, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -481,11 +480,13 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_READ_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -501,8 +502,7 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, {}, - {}, {}, pre_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, pre_barriers); if (blit_supported) { cmdbuf.BlitImage(*frame->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, image, @@ -515,30 +515,55 @@ void PresentManager::CopyToSwapchainImpl(Frame* frame) { MakeImageCopy(frame->width, frame->height, extent.width, extent.height)); } - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT, {}, - {}, {}, post_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, post_barriers); cmdbuf.End(); const VkSemaphore present_semaphore = swapchain.CurrentPresentSemaphore(); const VkSemaphore render_semaphore = swapchain.CurrentRenderSemaphore(); - const std::array wait_semaphores = {present_semaphore, *frame->render_ready}; - - static constexpr std::array wait_stage_masks{ - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, + const std::array wait_infos{{ + { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .semaphore = present_semaphore, + .value = 0, + .stageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .deviceIndex = 0, + }, + { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .semaphore = *frame->render_ready, + .value = 0, + .stageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .deviceIndex = 0, + }, + }}; + const VkCommandBufferSubmitInfo cmdbuf_info{ + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, + .pNext = nullptr, + .commandBuffer = *cmdbuf, + .deviceMask = 0, + }; + const VkSemaphoreSubmitInfo signal_info{ + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .pNext = nullptr, + .semaphore = render_semaphore, + .value = 0, + .stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + .deviceIndex = 0, }; - const VkSubmitInfo submit_info{ - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + const VkSubmitInfo2 submit_info{ + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, .pNext = nullptr, - .waitSemaphoreCount = 2U, - .pWaitSemaphores = wait_semaphores.data(), - .pWaitDstStageMask = wait_stage_masks.data(), - .commandBufferCount = 1, - .pCommandBuffers = cmdbuf.address(), - .signalSemaphoreCount = 1U, - .pSignalSemaphores = &render_semaphore, + .flags = 0, + .waitSemaphoreInfoCount = 2U, + .pWaitSemaphoreInfos = wait_infos.data(), + .commandBufferInfoCount = 1U, + .pCommandBufferInfos = &cmdbuf_info, + .signalSemaphoreInfoCount = 1U, + .pSignalSemaphoreInfos = &signal_info, }; // Submit the image copy/blit to the swapchain diff --git a/src/video_core/renderer_vulkan/vk_present_manager.h b/src/video_core/renderer_vulkan/vk_present_manager.h index 4741c55e61..7e1a47bb45 100644 --- a/src/video_core/renderer_vulkan/vk_present_manager.h +++ b/src/video_core/renderer_vulkan/vk_present_manager.h @@ -6,6 +6,8 @@ #pragma once +#include +#include #include #include #include @@ -31,12 +33,11 @@ struct Frame { u32 index; vk::Image image; vk::ImageView image_view; - vk::ImageView storage_view; - vk::Framebuffer framebuffer; vk::CommandBuffer cmdbuf; vk::Semaphore render_ready; vk::Fence present_done; - bool storage_capable{}; + std::chrono::nanoseconds pace_step{}; + u32 pace_index{}; }; class PresentManager { @@ -53,14 +54,13 @@ public: /// Returns the last used presentation frame Frame* GetRenderFrame(); + [[nodiscard]] Frame* TryGetRenderFrame(); + /// Pushes a frame for presentation - void Present(Frame* frame); + void Present(Frame* frame, u32 pace_index, std::chrono::nanoseconds pace_step); /// Recreates the present frame to match the provided parameters - void RecreateFrame(Frame* frame, u32 width, u32 height, VkFormat image_view_format, - VkRenderPass rd, bool storage); - - [[nodiscard]] bool NeedsStorage(const Frame* frame, bool required) const; + void RecreateFrame(Frame* frame, u32 width, u32 height, VkFormat image_view_format); /// Waits for the present thread to finish presenting all queued frames. void WaitPresent(); @@ -68,6 +68,9 @@ public: /// How many additional frames can be queued without stalling the render thread [[nodiscard]] size_t MaxExtraFrames() const; + [[nodiscard]] std::size_t SwapchainImageCount() const { return swapchain_image_count; } + [[nodiscard]] VkFormat SwapchainImageFormat() const { return swapchain_image_format; } + private: void PresentThread(std::stop_token token); @@ -77,6 +80,8 @@ private: void RecreateSwapchain(Frame* frame); + void DiscardFrame(Frame* frame); + void SetImageCount(); private: @@ -98,9 +103,10 @@ private: std::mutex free_mutex; std::jthread present_thread; bool blit_supported; - bool storage_supported; bool use_present_thread; - std::size_t image_count{}; + std::atomic image_count{}; + std::atomic swapchain_image_count{}; + std::atomic swapchain_image_format{}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_query_cache.cpp b/src/video_core/renderer_vulkan/vk_query_cache.cpp index 2b5aa24053..598bbd3633 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_query_cache.cpp @@ -37,6 +37,12 @@ using Tegra::Engines::Maxwell3D; using VideoCommon::QueryType; namespace { +bool LacksQueryWriteBack(const Device& device) { + const VkDriverId driver_id = device.GetDriverID(); + return driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || + driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP; +} + class SamplesQueryBank : public VideoCommon::BankBase { public: static constexpr size_t BANK_SIZE = 256; @@ -235,9 +241,7 @@ public: return; } PauseCounter(); - const auto driver_id = device.GetDriverID(); - if (driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || - driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { + if (LacksQueryWriteBack(device)) { pending_sync.clear(); sync_values_stash.clear(); return; @@ -260,11 +264,13 @@ public: scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([start, amount, base_offset, query_pool, buffer = *resolve_buffer](vk::CommandBuffer cmdbuf) { - const VkBufferMemoryBarrier copy_query_pool_barrier{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER, + const VkBufferMemoryBarrier2 copy_query_pool_barrier{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .buffer = buffer, @@ -276,8 +282,7 @@ public: query_pool, static_cast(start), static_cast(amount), buffer, static_cast(base_offset), SamplesQueryBank::QUERY_SIZE, VK_QUERY_RESULT_WAIT_BIT | VK_QUERY_RESULT_64_BIT); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, copy_query_pool_barrier); + cmdbuf.PipelineBarrier(copy_query_pool_barrier); }); offsets[bank_id] = {start, base_offset}; base_offset += amount * SamplesQueryBank::QUERY_SIZE; @@ -652,11 +657,13 @@ public: scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([this, dst_buffer = stagging_buffer.buffer, extra_offset, start, size](vk::CommandBuffer cmdbuf) { - std::array copy{VkBufferCopy{ + const VkBufferCopy2 copy{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, .srcOffset = start * QUERY_SIZE, .dstOffset = extra_offset, .size = size * QUERY_SIZE, - }}; + }; cmdbuf.CopyBuffer(*buffer, dst_buffer, copy); }); } @@ -862,16 +869,17 @@ public: offset_base += TFBQueryBank::QUERY_SIZE; bank.CloseReference(); } - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_HOST, + .dstAccessMask = VK_ACCESS_2_HOST_READ_BIT, }; scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - vk::PIPELINE_STAGE_HOST, 0, WRITE_BARRIER); + cmdbuf.PipelineBarrier(WRITE_BARRIER); }); std::scoped_lock lk(flush_guard); @@ -928,16 +936,7 @@ private: }); return; } - static constexpr VkMemoryBarrier COUNTER_RESUME_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT, - .dstAccessMask = VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT, - }; scheduler.Record([this, total = static_cast(buffers_count)](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT, - VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT, 0, - COUNTER_RESUME_BARRIER); cmdbuf.BeginTransformFeedbackEXT(0, total, counter_buffers.data(), offsets.data()); }); } @@ -961,6 +960,7 @@ private: }); } else { LOG_DEBUG(Render_Vulkan, "EndTransformFeedbackEXT called with counters (buffers_count={})", buffers_count); + scheduler.MarkRenderPassWrites(); scheduler.Record([this, total = static_cast(buffers_count)](vk::CommandBuffer cmdbuf) { cmdbuf.EndTransformFeedbackEXT(0, total, counter_buffers.data(), offsets.data()); @@ -1024,33 +1024,37 @@ private: const size_t slot = other; // workaround to compile bug. current_bank->AddReference(); - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFORM_FEEDBACK_BIT_EXT, + .srcAccessMask = VK_ACCESS_2_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, }; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 WRITE_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, }; scheduler.RequestOutsideRenderPassOperationContext(); scheduler.Record([dst_buffer = current_bank->GetBuffer(), src_buffer = counter_buffers[slot_index], src_offset = offsets[slot_index], slot](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); - std::array copy{VkBufferCopy{ + cmdbuf.PipelineBarrier(READ_BARRIER); + const VkBufferCopy2 copy{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, .srcOffset = src_offset, .dstOffset = slot * TFBQueryBank::QUERY_SIZE, .size = TFBQueryBank::QUERY_SIZE, - }}; + }; cmdbuf.CopyBuffer(src_buffer, dst_buffer, copy); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, WRITE_BARRIER); + cmdbuf.PipelineBarrier(WRITE_BARRIER); }); return {current_bank_id, slot}; } @@ -1343,7 +1347,7 @@ struct QueryCacheRuntimeImpl { std::vector> little_cache; std::vector> buffers_to_upload_to; std::vector redirect_cache; - std::vector> copies_setup; + std::vector> copies_setup; // Host conditional rendering data std::optional conditional_resolve_pass; @@ -1406,6 +1410,10 @@ void QueryCacheRuntime::ResumeHostConditionalRendering() { impl->is_hcr_running = true; } +bool QueryCacheRuntime::IsHostConditionalRenderingSet() const noexcept { + return impl->hcr_is_set; +} + void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::LookupData object, bool is_equal) { { @@ -1417,24 +1425,19 @@ void QueryCacheRuntime::HostConditionalRenderingCompareValueImpl(VideoCommon::Lo impl->hcr_buffer = buffer->Handle(); impl->hcr_offset = offset; } - if (impl->hcr_is_set) { - if (impl->hcr_setup.buffer == impl->hcr_buffer && - impl->hcr_setup.offset == impl->hcr_offset) { - return; - } + VkConditionalRenderingFlagsEXT flags = 0; + if (is_equal) { + flags = VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT; } - bool was_running = impl->is_hcr_running; - if (was_running) { - PauseHostConditionalRendering(); + if (impl->hcr_is_set && impl->hcr_setup.buffer == impl->hcr_buffer && + impl->hcr_setup.offset == impl->hcr_offset && impl->hcr_setup.flags == flags) { + return; } + PauseHostConditionalRendering(); impl->hcr_setup.buffer = impl->hcr_buffer; impl->hcr_setup.offset = impl->hcr_offset; - impl->hcr_setup.flags = is_equal ? VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT : 0; + impl->hcr_setup.flags = flags; impl->hcr_is_set = true; - impl->is_hcr_running = false; - if (was_running) { - ResumeHostConditionalRendering(); - } } void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, bool is_equal, @@ -1451,20 +1454,16 @@ void QueryCacheRuntime::HostConditionalRenderingCompareBCImpl(DAddr address, boo to_resolve = buffer->Handle(); to_resolve_offset = static_cast(offset); } - bool was_running = impl->is_hcr_running; - if (was_running) { - PauseHostConditionalRendering(); - } + PauseHostConditionalRendering(); impl->conditional_resolve_pass->Resolve(*impl->hcr_resolve_buffer, to_resolve, to_resolve_offset, compare_to_zero); impl->hcr_setup.buffer = *impl->hcr_resolve_buffer; impl->hcr_setup.offset = 0; - impl->hcr_setup.flags = is_equal ? 0 : VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT; - impl->hcr_is_set = true; - impl->is_hcr_running = false; - if (was_running) { - ResumeHostConditionalRendering(); + impl->hcr_setup.flags = 0; + if (!is_equal) { + impl->hcr_setup.flags = VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT; } + impl->hcr_is_set = true; } bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::LookupData object_1, @@ -1472,6 +1471,10 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValue(VideoCommon::Lookup if (!impl->device.IsExtConditionalRendering()) { return false; } + if (LacksQueryWriteBack(impl->device)) { + EndHostConditionalRendering(); + return true; + } HostConditionalRenderingCompareBCImpl(object_1.address, true, true); return true; } @@ -1521,7 +1524,8 @@ bool QueryCacheRuntime::HostConditionalRenderingCompareValues(VideoCommon::Looku auto driver_id = impl->device.GetDriverID(); const bool is_gpu_high = Settings::IsGPULevelHigh(); - if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || driver_id == VK_DRIVER_ID_QUALCOMM_PROPRIETARY || driver_id == VK_DRIVER_ID_ARM_PROPRIETARY || driver_id == VK_DRIVER_ID_MESA_TURNIP) { + if ((!is_gpu_high && driver_id == VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS) || + LacksQueryWriteBack(impl->device)) { EndHostConditionalRendering(); return true; } @@ -1573,30 +1577,24 @@ VideoCommon::StreamerInterface* QueryCacheRuntime::GetStreamerInterface(QueryTyp } void QueryCacheRuntime::Barriers(bool is_prebarrier) { - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + VkMemoryBarrier2 barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = impl->device.GetBufferConsumerStages(), + .dstAccessMask = impl->device.GetBufferConsumerAccess(), }; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, - }; - impl->scheduler.RequestOutsideRenderPassOperationContext(); if (is_prebarrier) { - impl->scheduler.Record([](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, READ_BARRIER); - }); - } else { - impl->scheduler.Record([](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER_HOST, 0, WRITE_BARRIER); - }); + barrier.srcStageMask = impl->device.GetBufferUserStages(); + barrier.srcAccessMask = vk::ACCESS_BUFFER_WRITES; + barrier.dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + barrier.dstAccessMask = vk::ACCESS_TRANSFER; } + impl->scheduler.RequestOutsideRenderPassOperationContext(); + impl->scheduler.Record([barrier](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(barrier); + }); } template @@ -1648,6 +1646,7 @@ void QueryCacheRuntime::SyncValues(std::span values, VkBuffer ba const auto post_op = VideoCommon::ObtainBufferOperation::DoNothing; const auto [buffer, offset] = impl->buffer_cache.ObtainCPUBuffer( pair.first, static_cast(pair.second - pair.first), sync_info, post_op); + buffer->MarkContentModified(); impl->buffers_to_upload_to.emplace_back(buffer->Handle(), offset); } }); @@ -1662,7 +1661,9 @@ void QueryCacheRuntime::SyncValues(std::span values, VkBuffer ba size_t accumulated_size = 0; for (size_t i = 0; i < values.size(); i++) { size_t which_copy = impl->redirect_cache[i]; - impl->copies_setup[which_copy].emplace_back(VkBufferCopy{ + impl->copies_setup[which_copy].emplace_back(VkBufferCopy2{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, .srcOffset = current_offset + accumulated_size, .dstOffset = impl->buffers_to_upload_to[which_copy].second + values[i].address - impl->little_cache[which_copy].first, @@ -1676,7 +1677,9 @@ void QueryCacheRuntime::SyncValues(std::span values, VkBuffer ba } else { for (size_t i = 0; i < values.size(); i++) { size_t which_copy = impl->redirect_cache[i]; - impl->copies_setup[which_copy].emplace_back(VkBufferCopy{ + impl->copies_setup[which_copy].emplace_back(VkBufferCopy2{ + .sType = VK_STRUCTURE_TYPE_BUFFER_COPY_2, + .pNext = nullptr, .srcOffset = values[i].offset, .dstOffset = impl->buffers_to_upload_to[which_copy].second + values[i].address - impl->little_cache[which_copy].first, diff --git a/src/video_core/renderer_vulkan/vk_query_cache.h b/src/video_core/renderer_vulkan/vk_query_cache.h index bbb5234e11..f89038f8b7 100644 --- a/src/video_core/renderer_vulkan/vk_query_cache.h +++ b/src/video_core/renderer_vulkan/vk_query_cache.h @@ -48,6 +48,8 @@ public: void ResumeHostConditionalRendering(); + [[nodiscard]] bool IsHostConditionalRenderingSet() const noexcept; + bool HostConditionalRenderingCompareValue(VideoCommon::LookupData object_1, bool qc_dirty); bool HostConditionalRenderingCompareValues(VideoCommon::LookupData object_1, diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.cpp b/src/video_core/renderer_vulkan/vk_rasterizer.cpp index f43bea1220..a7bf7aecfd 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.cpp +++ b/src/video_core/renderer_vulkan/vk_rasterizer.cpp @@ -166,7 +166,11 @@ DrawParams MakeDrawParams(const Tegra::Engines::Maxwell3D::DrawManager::State& d params.base_vertex = 0; params.is_indexed = true; } else if (draw_state.topology == Maxwell::PrimitiveTopology::QuadStrip) { - params.num_vertices = (params.num_vertices - 2) / 2 * 6; + u32 strips = 0; + if (params.num_vertices >= 2) { + strips = (params.num_vertices - 2) / 2; + } + params.num_vertices = strips * 6; params.base_vertex = 0; params.is_indexed = true; } @@ -202,15 +206,15 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra StateTracker& state_tracker_, Scheduler& scheduler_) : gpu{gpu_}, device_memory{device_memory_}, device{device_}, memory_allocator{memory_allocator_}, state_tracker{state_tracker_}, scheduler{scheduler_}, - staging_pool(device, memory_allocator, scheduler), descriptor_pool(device, scheduler), + staging_pool(device, memory_allocator, scheduler), guest_descriptor_queue(device, UpdateDescriptorQueue::GUEST_FRAME_PAYLOAD_SIZE, device.IsExtDescriptorBufferSupported()), compute_pass_descriptor_queue(device, UpdateDescriptorQueue::COMPUTE_FRAME_PAYLOAD_SIZE), descriptor_buffer_ring(device, memory_allocator), - blit_image(device, scheduler, state_tracker, descriptor_pool), render_pass_cache(device), - texture_cache_runtime{ - device, scheduler, memory_allocator, staging_pool, - blit_image, render_pass_cache, descriptor_pool, compute_pass_descriptor_queue}, + blit_image(device, scheduler, state_tracker, descriptor_pool), + texture_cache_runtime{device, scheduler, memory_allocator, + staging_pool, blit_image, descriptor_pool, + compute_pass_descriptor_queue}, texture_cache(texture_cache_runtime, device_memory), buffer_cache_runtime(device, memory_allocator, scheduler, staging_pool, guest_descriptor_queue, compute_pass_descriptor_queue, descriptor_pool), @@ -219,12 +223,13 @@ RasterizerVulkan::RasterizerVulkan(Core::Frontend::EmuWindow& emu_window_, Tegra staging_pool, compute_pass_descriptor_queue, descriptor_pool, texture_cache), query_cache(gpu, *this, device_memory, query_cache_runtime), pipeline_cache(device_memory, device, scheduler, descriptor_pool, guest_descriptor_queue, - descriptor_buffer_ring, render_pass_cache, buffer_cache, texture_cache, - gpu.ShaderNotify()), + descriptor_buffer_ring, buffer_cache, texture_cache, gpu.ShaderNotify()), accelerate_dma(buffer_cache, texture_cache, scheduler), - fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler), - wfi_event(device.GetLogical().CreateEvent()) { + fence_manager(*this, gpu, texture_cache, buffer_cache, query_cache, device, scheduler) { scheduler.SetQueryCache(query_cache); + if (Settings::values.use_unified_memory.GetValue()) { + texture_cache_runtime.ImportGuestMemory(device_memory.GetHostMemory()); + } } RasterizerVulkan::~RasterizerVulkan() { @@ -233,7 +238,7 @@ RasterizerVulkan::~RasterizerVulkan() { } template -void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { +void RasterizerVulkan::PrepareDraw(bool is_indexed, bool skip_empty, Func&& draw_func) { SCOPE_EXIT { gpu.TickWork(); @@ -241,6 +246,17 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { FlushWork(); gpu_memory->FlushCaching(); + if (skip_empty) { + const auto& draw_state = maxwell3d->draw_manager.draw_state; + u32 count = draw_state.vertex_buffer.count; + if (is_indexed) { + count = draw_state.index_buffer.count; + } + if (count == 0) { + return; + } + } + GraphicsPipeline* const pipeline{pipeline_cache.CurrentGraphicsPipeline()}; if (!pipeline) { return; @@ -251,7 +267,7 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { if (!pipeline->Configure(is_indexed)) return; - UpdateDynamicStates(); + UpdateDynamicStates(*pipeline); query_cache.NotifySegment(true); HandleTransformFeedback(); @@ -260,7 +276,7 @@ void RasterizerVulkan::PrepareDraw(bool is_indexed, Func&& draw_func) { } void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { - PrepareDraw(is_indexed, [this, is_indexed, instance_count] { + PrepareDraw(is_indexed, true, [this, is_indexed, instance_count] { const auto& draw_state = maxwell3d->draw_manager.draw_state; const u32 num_instances{instance_count}; const DrawParams draw_params{MakeDrawParams(draw_state, num_instances, is_indexed)}; @@ -295,10 +311,13 @@ void RasterizerVulkan::Draw(bool is_indexed, u32 instance_count) { void RasterizerVulkan::DrawIndirect() { const auto& params = maxwell3d->draw_manager.indirect_state; buffer_cache.SetDrawIndirect(¶ms); - PrepareDraw(params.is_indexed, [this, ¶ms] { + PrepareDraw(params.is_indexed, false, [this, ¶ms] { const auto indirect_buffer = buffer_cache.GetDrawIndirectBuffer(); const auto& buffer = indirect_buffer.first; const auto& offset = indirect_buffer.second; + VkBuffer command_buffer = buffer->Handle(); + VkDeviceSize command_offset = offset; + u32 command_stride = static_cast(params.stride); if (params.is_byte_count) { scheduler.Record([buffer_obj = buffer->Handle(), offset, stride = params.stride](vk::CommandBuffer cmdbuf) { @@ -308,32 +327,38 @@ void RasterizerVulkan::DrawIndirect() { return; } if (params.include_count) { + if (!device.IsDrawIndirectCountSupported()) { + return; + } const auto count = buffer_cache.GetDrawIndirectCount(); const auto& draw_buffer = count.first; const auto& offset_base = count.second; - scheduler.Record([draw_buffer_obj = draw_buffer->Handle(), - buffer_obj = buffer->Handle(), offset_base, offset, + scheduler.Record([draw_buffer_obj = draw_buffer->Handle(), command_buffer, + offset_base, command_offset, command_stride, params](vk::CommandBuffer cmdbuf) { if (params.is_indexed) { - cmdbuf.DrawIndexedIndirectCount( - buffer_obj, offset, draw_buffer_obj, offset_base, - static_cast(params.max_draw_counts), static_cast(params.stride)); + cmdbuf.DrawIndexedIndirectCount(command_buffer, command_offset, draw_buffer_obj, + offset_base, + static_cast(params.max_draw_counts), + command_stride); } else { - cmdbuf.DrawIndirectCount(buffer_obj, offset, draw_buffer_obj, offset_base, + cmdbuf.DrawIndirectCount(command_buffer, command_offset, draw_buffer_obj, + offset_base, static_cast(params.max_draw_counts), - static_cast(params.stride)); + command_stride); } }); return; } - scheduler.Record([buffer_obj = buffer->Handle(), offset, params](vk::CommandBuffer cmdbuf) { + scheduler.Record([command_buffer, command_offset, command_stride, + params](vk::CommandBuffer cmdbuf) { if (params.is_indexed) { - cmdbuf.DrawIndexedIndirect(buffer_obj, offset, + cmdbuf.DrawIndexedIndirect(command_buffer, command_offset, static_cast(params.max_draw_counts), - static_cast(params.stride)); + command_stride); } else { - cmdbuf.DrawIndirect(buffer_obj, offset, static_cast(params.max_draw_counts), - static_cast(params.stride)); + cmdbuf.DrawIndirect(command_buffer, command_offset, + static_cast(params.max_draw_counts), command_stride); } }); @@ -361,14 +386,14 @@ void RasterizerVulkan::DrawTexture() { texture_cache.SynchronizeDescriptors(false); texture_cache.UpdateRenderTargets(false); - UpdateDynamicStates(); - - query_cache.NotifySegment(true); - query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); const auto& draw_texture_state = maxwell3d->draw_manager.draw_texture_state; const auto& sampler = texture_cache.GetSampler(draw_texture_state.src_sampler, false); const auto& texture = texture_cache.GetImageView(draw_texture_state.src_texture); const auto* framebuffer = texture_cache.GetFramebuffer(); + scheduler.RequestRenderpass(framebuffer, 0, 0); + query_cache.NotifySegment(true); + query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, + maxwell3d->regs.zpass_pixel_count_enable); const bool src_rescaling = texture_cache.IsRescaling() && texture.IsRescaled(); const bool dst_rescaling = texture_cache.IsRescaling() && framebuffer->IsRescaled(); @@ -393,8 +418,9 @@ void RasterizerVulkan::DrawTexture() { .y = ScaleSrc(draw_texture_state.src_y1)}}; Extent3D src_size = {static_cast(ScaleSrc(texture.size.width)), static_cast(ScaleSrc(texture.size.height)), texture.size.depth}; - blit_image.BlitColor(framebuffer, texture.RenderTarget(), texture.ImageHandle(), - sampler->Handle(), dst_region, src_region, src_size); + blit_image.BlitColor(framebuffer, texture.Handle(Shader::TextureType::Color2D), + texture.ImageHandle(), sampler->Handle(), dst_region, src_region, + src_size); } void RasterizerVulkan::Clear(u32 layer_count) { @@ -415,15 +441,10 @@ void RasterizerVulkan::Clear(u32 layer_count) { const Framebuffer* const framebuffer = texture_cache.GetFramebuffer(); const VkExtent2D render_area = framebuffer->RenderArea(); - constexpr bool ENABLE_DEFERRED_CLEAR = true; const bool color_full_channels = regs.clear_surface.R && regs.clear_surface.G && regs.clear_surface.B && regs.clear_surface.A; const bool stencil_partial = use_stencil && framebuffer->HasAspectStencilBit() && regs.stencil_front_mask != 0xFF && regs.stencil_front_mask != 0; - const bool ds_used = use_depth || use_stencil; - const bool ds_deferrable = - !ds_used || ((!framebuffer->HasAspectDepthBit() || use_depth) && - (!framebuffer->HasAspectStencilBit() || use_stencil) && !stencil_partial); u32 up_scale = 1; u32 down_shift = 0; if (texture_cache.IsRescaling()) { @@ -489,16 +510,16 @@ void RasterizerVulkan::Clear(u32 layer_count) { clear_rect.rect.offset.x == 0 && clear_rect.rect.offset.y == 0 && clear_rect.rect.extent.width >= render_area.width && clear_rect.rect.extent.height >= render_area.height; - const bool can_defer_clear = ENABLE_DEFERRED_CLEAR && (!regs.clear_control.use_scissor || clear_covers_render_area) && - regs.clear_surface.layer == 0 && - !scheduler.IsRenderPassActive() && - (!use_color || color_full_channels) && ds_deferrable; - if (!can_defer_clear) { + const bool clear_on_load = (!regs.clear_control.use_scissor || clear_covers_render_area) && + regs.clear_surface.layer == 0 && + layer_count >= framebuffer->Attachments().layers && + !query_cache_runtime.IsHostConditionalRenderingSet(); + const auto request_renderpass = [&] { scheduler.RequestRenderpass(framebuffer); - } - - query_cache.NotifySegment(true); - query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, maxwell3d->regs.zpass_pixel_count_enable); + query_cache.NotifySegment(true); + query_cache.CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, + maxwell3d->regs.zpass_pixel_count_enable); + }; UpdateViewportsState(regs); const u32 color_attachment = regs.clear_surface.RT; @@ -518,26 +539,27 @@ void RasterizerVulkan::Clear(u32 layer_count) { clear_value.color.int32[i] = s32(f32(s64(int_size - 1) << 1) * (regs.clear_color[i] - 0.5f)); } - if (color_full_channels) { - if (can_defer_clear) { - scheduler.DeferColorClear(framebuffer, color_attachment, clear_value); - } else { - scheduler.Record([color_attachment, clear_value, clear_rect](vk::CommandBuffer cmdbuf) { - const VkClearAttachment attachment{ - .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, - .colorAttachment = color_attachment, - .clearValue = clear_value, - }; - cmdbuf.ClearAttachments(attachment, clear_rect); - }); - } - } else { + if (!color_full_channels) { u8 color_mask = u8(regs.clear_surface.R | regs.clear_surface.G << 1 | regs.clear_surface.B << 2 | regs.clear_surface.A << 3); Region2D dst_region = { Offset2D{.x = clear_rect.rect.offset.x, .y = clear_rect.rect.offset.y}, Offset2D{.x = clear_rect.rect.offset.x + s32(clear_rect.rect.extent.width), .y = clear_rect.rect.offset.y + s32(clear_rect.rect.extent.height)}}; - blit_image.ClearColor(framebuffer, color_mask, regs.clear_color, dst_region); + request_renderpass(); + blit_image.ClearColor(framebuffer, color_attachment, color_mask, regs.clear_color, + dst_region); + } else if (!clear_on_load || + !scheduler.OverrideLoadOps(framebuffer, 1u << color_attachment, + VK_ATTACHMENT_LOAD_OP_CLEAR, clear_value)) { + request_renderpass(); + scheduler.Record([color_attachment, clear_value, clear_rect](vk::CommandBuffer cmdbuf) { + const VkClearAttachment attachment{ + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .colorAttachment = color_attachment, + .clearValue = clear_value, + }; + cmdbuf.ClearAttachments(attachment, clear_rect); + }); } } @@ -545,41 +567,46 @@ void RasterizerVulkan::Clear(u32 layer_count) { return; } VkImageAspectFlags aspect_flags = 0; + u32 ds_attachments = 0; if (use_depth && framebuffer->HasAspectDepthBit()) { aspect_flags |= VK_IMAGE_ASPECT_DEPTH_BIT; + ds_attachments |= Scheduler::DEPTH_ATTACHMENT_BIT; } if (use_stencil && framebuffer->HasAspectStencilBit()) { aspect_flags |= VK_IMAGE_ASPECT_STENCIL_BIT; + ds_attachments |= Scheduler::STENCIL_ATTACHMENT_BIT; } if (aspect_flags == 0) { return; } - if (use_stencil && framebuffer->HasAspectStencilBit() && regs.stencil_front_mask != 0xFF && - regs.stencil_front_mask != 0) { + if (stencil_partial) { Region2D dst_region = { Offset2D{.x = clear_rect.rect.offset.x, .y = clear_rect.rect.offset.y}, Offset2D{.x = clear_rect.rect.offset.x + s32(clear_rect.rect.extent.width), .y = clear_rect.rect.offset.y + s32(clear_rect.rect.extent.height)}}; + request_renderpass(); blit_image.ClearDepthStencil(framebuffer, use_depth, regs.clear_depth, u8(regs.stencil_front_mask), regs.clear_stencil, regs.stencil_front_func_mask, dst_region); - } else if (can_defer_clear) { - VkClearValue ds_value{}; - ds_value.depthStencil.depth = regs.clear_depth; - ds_value.depthStencil.stencil = regs.clear_stencil; - scheduler.DeferDepthStencilClear(framebuffer, ds_value); - } else { - scheduler.Record([clear_depth = regs.clear_depth, clear_stencil = regs.clear_stencil, - clear_rect, aspect_flags](vk::CommandBuffer cmdbuf) { - VkClearAttachment attachment; - attachment.aspectMask = aspect_flags; - attachment.colorAttachment = 0; - attachment.clearValue.depthStencil.depth = clear_depth; - attachment.clearValue.depthStencil.stencil = clear_stencil; - cmdbuf.ClearAttachments(attachment, clear_rect); - }); + return; } + VkClearValue ds_value{}; + ds_value.depthStencil.depth = regs.clear_depth; + ds_value.depthStencil.stencil = regs.clear_stencil; + if (clear_on_load && scheduler.OverrideLoadOps(framebuffer, ds_attachments, + VK_ATTACHMENT_LOAD_OP_CLEAR, ds_value)) { + return; + } + request_renderpass(); + scheduler.Record([ds_value, clear_rect, aspect_flags](vk::CommandBuffer cmdbuf) { + const VkClearAttachment attachment{ + .aspectMask = aspect_flags, + .colorAttachment = 0, + .clearValue = ds_value, + }; + cmdbuf.ClearAttachments(attachment, clear_rect); + }); } void RasterizerVulkan::DispatchCompute() { @@ -596,6 +623,14 @@ void RasterizerVulkan::DispatchCompute() { return; } + const VkMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = device.GetBufferUserStages(), + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .dstAccessMask = vk::ACCESS_SHADER_RESOURCES, + }; const auto& qmd{kepler_compute->launch_description}; auto indirect_address = kepler_compute->GetIndirectComputeAddress(); if (indirect_address) { @@ -604,12 +639,16 @@ void RasterizerVulkan::DispatchCompute() { const auto post_op = VideoCommon::ObtainBufferOperation::DiscardWrite; const auto [buffer, offset] = buffer_cache.ObtainBuffer(*indirect_address, 12, sync_info, post_op); - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([pipeline, indirect_buffer = buffer->Handle(), - indirect_offset = offset](vk::CommandBuffer cmdbuf) { + scheduler.RequestComputeDispatchContext(); + VkMemoryBarrier2 indirect_read_barrier = read_barrier; + indirect_read_barrier.dstStageMask |= VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT; + indirect_read_barrier.dstAccessMask |= VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT; + scheduler.Record([pipeline, indirect_buffer = buffer->Handle(), indirect_offset = offset, + indirect_read_barrier](vk::CommandBuffer cmdbuf) { if (!pipeline->IsBound()) { return; } + cmdbuf.PipelineBarrier(indirect_read_barrier); cmdbuf.DispatchIndirect(indirect_buffer, indirect_offset); }); return; @@ -619,19 +658,12 @@ void RasterizerVulkan::DispatchCompute() { if (dim[0] > max_dim[0] || dim[1] > max_dim[1] || dim[2] > max_dim[2]) { return; } - scheduler.RequestOutsideRenderPassOperationContext(); - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT, - }; - scheduler.Record([](vk::CommandBuffer cmdbuf) { cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, - 0, READ_BARRIER); }); - scheduler.Record([pipeline, dim](vk::CommandBuffer cmdbuf) { + scheduler.RequestComputeDispatchContext(); + scheduler.Record([pipeline, dim, read_barrier](vk::CommandBuffer cmdbuf) { if (!pipeline->IsBound()) { return; } + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.Dispatch(dim[0], dim[1], dim[2]); }); @@ -830,8 +862,8 @@ void RasterizerVulkan::SyncOperation(std::function&& func) { fence_manager.SyncOperation(std::move(func)); } -void RasterizerVulkan::SignalSyncPoint(u32 value) { - fence_manager.SignalSyncPoint(value); +void RasterizerVulkan::SignalSyncPoint(u32 value, u32 count) { + fence_manager.SignalSyncPoint(value, count); } void RasterizerVulkan::SignalReference() { @@ -851,25 +883,8 @@ void RasterizerVulkan::FlushAndInvalidateRegion(DAddr addr, u64 size, } void RasterizerVulkan::WaitForIdle() { - // Everything but wait pixel operations. This intentionally includes FRAGMENT_SHADER_BIT because - // fragment shaders can still write storage buffers. - VkPipelineStageFlags flags = - VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | - VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT | - VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | - VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT; - if (device.IsExtTransformFeedbackSupported()) { - flags |= VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT; - } - query_cache.NotifyWFI(); - - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([event = *wfi_event, flags](vk::CommandBuffer cmdbuf) { - cmdbuf.SetEvent(event, flags); - cmdbuf.WaitEvents(event, flags, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, {}, {}, {}); - }); + scheduler.NotifyWaitForIdle(); fence_manager.SignalOrdering(); } @@ -883,7 +898,9 @@ void RasterizerVulkan::TiledCacheBarrier() { } void RasterizerVulkan::FlushCommands() { - if (draw_counter == 0) { + if (draw_tick != scheduler.CurrentTick() || + (scheduler.IsRenderPassActive() && + !maxwell3d->dirty.flags[VideoCommon::Dirty::RenderTargets])) { return; } draw_counter = 0; @@ -982,23 +999,21 @@ void RasterizerVulkan::LoadDiskResources(u64 title_id, std::stop_token stop_load void RasterizerVulkan::FlushWork() { #ifdef __ANDROID__ - static constexpr u32 DRAWS_TO_DISPATCH = 512; - static constexpr u32 CHECK_MASK = 3; + static constexpr u32 DRAWS_TO_DISPATCH = 1024; #else static constexpr u32 DRAWS_TO_DISPATCH = 4096; - static constexpr u32 CHECK_MASK = 7; #endif // __ANDROID__ + static constexpr u32 CHECK_MASK = 7; - static_assert(DRAWS_TO_DISPATCH % (CHECK_MASK + 1) == 0); - if ((++draw_counter & CHECK_MASK) != CHECK_MASK) { - return; - } - if (draw_counter < DRAWS_TO_DISPATCH) { + if (++draw_counter >= DRAWS_TO_DISPATCH && + (!scheduler.IsRenderPassActive() || + maxwell3d->dirty.flags[VideoCommon::Dirty::RenderTargets])) { + scheduler.Flush(); + draw_counter = 0; + } else if ((draw_counter & CHECK_MASK) == CHECK_MASK && !scheduler.IsRenderPassActive()) { scheduler.DispatchWork(); - return; } - scheduler.Flush(); - draw_counter = 0; + draw_tick = scheduler.CurrentTick(); } AccelerateDMA::AccelerateDMA(BufferCache& buffer_cache_, TextureCache& texture_cache_, @@ -1060,7 +1075,7 @@ bool AccelerateDMA::BufferToImage(const Tegra::DMA::ImageCopy& copy_info, return DmaBufferImageCopy(copy_info, buffer_operand, image_operand); } -void RasterizerVulkan::UpdateDynamicStates() { +void RasterizerVulkan::UpdateDynamicStates(const GraphicsPipeline& pipeline) { auto& regs = maxwell3d->regs; auto& flags = maxwell3d->dirty.flags; const auto topology = maxwell3d->draw_manager.draw_state.topology; @@ -1108,27 +1123,13 @@ void RasterizerVulkan::UpdateDynamicStates() { } if (device.IsExtExtendedDynamicState3EnablesSupported()) { - using namespace Tegra::Engines; - // AMD Workaround: LogicOp incompatible with float render targets - if (device.GetDriverID() == VkDriverIdKHR::VK_DRIVER_ID_AMD_OPEN_SOURCE || - device.GetDriverID() == VkDriverIdKHR::VK_DRIVER_ID_AMD_PROPRIETARY) { - const auto has_float = std::any_of( - regs.vertex_attrib_format.begin(), regs.vertex_attrib_format.end(), - [](const auto& attrib) { - return attrib.type == Maxwell3D::Regs::VertexAttribute::Type::Float; - } - ); - if (regs.logic_op.enable) { - regs.logic_op.enable = static_cast(!has_float); - } - } UpdateLogicOpEnable(regs); UpdateDepthClampEnable(regs); UpdateLineRasterizationMode(regs); UpdateLineStippleEnable(regs); UpdateConservativeRasterizationMode(regs); - UpdateAlphaToCoverageEnable(regs); - UpdateAlphaToOneEnable(regs); + UpdateAlphaToCoverageEnable(regs, pipeline); + UpdateAlphaToOneEnable(regs, pipeline); } if (device.IsExtExtendedDynamicState3BlendingSupported()) { @@ -1137,10 +1138,8 @@ void RasterizerVulkan::UpdateDynamicStates() { UpdateColorWriteEnable(regs); } - if (device.IsExtVertexInputDynamicStateSupported()) { - if (auto* gp = pipeline_cache.CurrentGraphicsPipeline(); gp && gp->HasDynamicVertexInput()) { - UpdateVertexInput(regs); - } + if (pipeline.HasDynamicVertexInput()) { + UpdateVertexInput(regs, pipeline.VertexAttributeMask()); } } @@ -1283,10 +1282,18 @@ void RasterizerVulkan::UpdateScissorsState(Tegra::Engines::Maxwell3D::Regs& regs } void RasterizerVulkan::UpdateDepthBias(Tegra::Engines::Maxwell3D::Regs& regs) { - if (!state_tracker.TouchDepthBias()) { + const bool zeta_changed = + std::exchange(depth_bias_zeta_format, regs.zeta.format) != regs.zeta.format; + const bool bias_changed = state_tracker.TouchDepthBias(); + if (!bias_changed && !zeta_changed) { return; } float units = regs.depth_bias / 2.0f; + const bool is_float = regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT || + regs.zeta.format == Tegra::DepthFormat::Z32_FLOAT_X24S8_UINT; + if (is_float && !device.IsExtDepthBiasControlSupported()) { + units *= 256.0f; + } const bool is_d24 = regs.zeta.format == Tegra::DepthFormat::Z24_UNORM_S8_UINT || regs.zeta.format == Tegra::DepthFormat::X8Z24_UNORM || regs.zeta.format == Tegra::DepthFormat::S8Z24_UNORM || @@ -1666,36 +1673,33 @@ void RasterizerVulkan::UpdateDepthClampEnable(Tegra::Engines::Maxwell3D::Regs& r [is_enabled](vk::CommandBuffer cmdbuf) { cmdbuf.SetDepthClampEnableEXT(is_enabled); }); } -void RasterizerVulkan::UpdateAlphaToCoverageEnable(Tegra::Engines::Maxwell3D::Regs& regs) { - if (!state_tracker.TouchAlphaToCoverageEnable()) { - return; - } +void RasterizerVulkan::UpdateAlphaToCoverageEnable(Tegra::Engines::Maxwell3D::Regs& regs, + const GraphicsPipeline& pipeline) { if (!device.SupportsDynamicState3AlphaToCoverageEnable()) { return; } - GraphicsPipeline* const pipeline = pipeline_cache.CurrentGraphicsPipeline(); - const bool enable = pipeline != nullptr && pipeline->SupportsAlphaToCoverage() && - regs.anti_alias_alpha_control.alpha_to_coverage != 0; + const bool enable = + pipeline.SupportsAlphaToCoverage() && regs.anti_alias_alpha_control.alpha_to_coverage != 0; + if (!state_tracker.TouchAlphaToCoverageEnable() && enable == alpha_to_coverage_enabled) { + return; + } + alpha_to_coverage_enabled = enable; scheduler.Record([enable](vk::CommandBuffer cmdbuf) { cmdbuf.SetAlphaToCoverageEnableEXT(enable ? VK_TRUE : VK_FALSE); }); } -void RasterizerVulkan::UpdateAlphaToOneEnable(Tegra::Engines::Maxwell3D::Regs& regs) { - if (!state_tracker.TouchAlphaToOneEnable()) { - return; - } +void RasterizerVulkan::UpdateAlphaToOneEnable(Tegra::Engines::Maxwell3D::Regs& regs, + const GraphicsPipeline& pipeline) { if (!device.SupportsDynamicState3AlphaToOneEnable()) { - static std::once_flag warn_alpha_to_one; - std::call_once(warn_alpha_to_one, [] { - LOG_WARNING(Render_Vulkan, - "Alpha-to-one is not supported on this device; forcing it disabled"); - }); return; } - GraphicsPipeline* const pipeline = pipeline_cache.CurrentGraphicsPipeline(); - const bool enable = pipeline != nullptr && pipeline->SupportsAlphaToOne() && - regs.anti_alias_alpha_control.alpha_to_one != 0; + const bool enable = + pipeline.SupportsAlphaToOne() && regs.anti_alias_alpha_control.alpha_to_one != 0; + if (!state_tracker.TouchAlphaToOneEnable() && enable == alpha_to_one_enabled) { + return; + } + alpha_to_one_enabled = enable; scheduler.Record([enable](vk::CommandBuffer cmdbuf) { cmdbuf.SetAlphaToOneEnableEXT(enable ? VK_TRUE : VK_FALSE); }); @@ -1769,7 +1773,17 @@ void RasterizerVulkan::UpdateLogicOp(Tegra::Engines::Maxwell3D::Regs& regs) { } void RasterizerVulkan::UpdateBlending(Tegra::Engines::Maxwell3D::Regs& regs) { - if (!state_tracker.TouchBlending()) { + u8 integer_targets = 0; + for (size_t index = 0; index < Maxwell::NumRenderTargets; index++) { + const auto format = regs.rt[index].format; + if (format != Tegra::RenderTargetFormat::NONE && + IsPixelFormatInteger(VideoCore::Surface::PixelFormatFromRenderTargetFormat(format))) { + integer_targets |= static_cast(1u << index); + } + } + const bool targets_changed = + std::exchange(blend_integer_targets, integer_targets) != integer_targets; + if (!state_tracker.TouchBlending() && !targets_changed) { return; } @@ -1796,17 +1810,12 @@ void RasterizerVulkan::UpdateBlending(Tegra::Engines::Maxwell3D::Regs& regs) { }); } - if (state_tracker.TouchBlendEnable()) { + if (state_tracker.TouchBlendEnable() || targets_changed) { std::array setup_enables{}; for (size_t index = 0; index < Maxwell::NumRenderTargets; index++) { - bool is_integer = false; - if (regs.rt[index].format != Tegra::RenderTargetFormat::NONE) { - const auto format = - VideoCore::Surface::PixelFormatFromRenderTargetFormat(regs.rt[index].format); - is_integer = IsPixelFormatInteger(format); - } + const bool is_integer = ((integer_targets >> index) & 1) != 0; setup_enables[index] = - (!is_integer && regs.blend.enable[index] != 0) ? VK_TRUE : VK_FALSE; + static_cast(!is_integer && regs.blend.enable[index] != 0); } scheduler.Record([setup_enables](vk::CommandBuffer cmdbuf) { cmdbuf.SetColorBlendEnableEXT(0, setup_enables); @@ -1879,18 +1888,9 @@ void RasterizerVulkan::UpdateStencilTestEnable(Tegra::Engines::Maxwell3D::Regs& }); } -void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs) { +void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs, + u32 attribute_mask) { auto& dirty{maxwell3d->dirty.flags}; - const bool vertex_input_dirty = dirty[Dirty::VertexInput]; - const bool vertex_buffers_dirty = dirty[VideoCommon::Dirty::VertexBuffers]; - if (!vertex_input_dirty && !vertex_buffers_dirty) { - return; - } - dirty[Dirty::VertexInput] = false; - - boost::container::static_vector bindings; - boost::container::static_vector attributes; - const u32 max_attributes = static_cast(std::min(Maxwell::NumVertexAttributes, device.GetMaxVertexInputAttributes())); @@ -1898,11 +1898,25 @@ void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs) static_cast(std::min(Maxwell::NumVertexArrays, device.GetMaxVertexInputBindings())); + bool changed = dirty[Dirty::VertexInput] || vertex_input_attributes != attribute_mask; + dirty[Dirty::VertexInput] = false; + vertex_input_attributes = attribute_mask; + for (u32 binding = 0; binding < max_bindings; ++binding) { + const u32 stride = regs.vertex_streams[binding].stride; + changed |= vertex_input_strides[binding] != stride; + vertex_input_strides[binding] = stride; + } + if (!changed) { + return; + } + + boost::container::static_vector bindings; + boost::container::static_vector attributes; for (u32 index = 0; index < max_attributes; ++index) { const Maxwell::VertexAttribute attribute{regs.vertex_attrib_format[index]}; const u32 binding{attribute.buffer}; - if (attribute.constant || binding >= max_bindings) { + if (((attribute_mask >> index) & 1) == 0 || attribute.constant || binding >= max_bindings) { continue; } attributes.push_back({ @@ -1917,14 +1931,20 @@ void RasterizerVulkan::UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs) for (u32 binding = 0; binding < max_bindings; ++binding) { const auto& input_binding{regs.vertex_streams[binding]}; - const bool is_instanced{regs.vertex_stream_instances.IsInstancingEnabled(binding)}; + VkVertexInputRate input_rate = VK_VERTEX_INPUT_RATE_VERTEX; + u32 divisor = 1; + if (regs.vertex_stream_instances.IsInstancingEnabled(binding) && + input_binding.frequency != 0) { + input_rate = VK_VERTEX_INPUT_RATE_INSTANCE; + divisor = (std::min)(input_binding.frequency, device.GetMaxVertexAttribDivisor()); + } bindings.push_back({ .sType = VK_STRUCTURE_TYPE_VERTEX_INPUT_BINDING_DESCRIPTION_2_EXT, .pNext = nullptr, .binding = binding, - .stride = input_binding.stride, - .inputRate = is_instanced ? VK_VERTEX_INPUT_RATE_INSTANCE : VK_VERTEX_INPUT_RATE_VERTEX, - .divisor = is_instanced ? input_binding.frequency : 1, + .stride = vertex_input_strides[binding], + .inputRate = input_rate, + .divisor = divisor, }); } diff --git a/src/video_core/renderer_vulkan/vk_rasterizer.h b/src/video_core/renderer_vulkan/vk_rasterizer.h index 7470aa4f14..1d54694f2c 100644 --- a/src/video_core/renderer_vulkan/vk_rasterizer.h +++ b/src/video_core/renderer_vulkan/vk_rasterizer.h @@ -22,7 +22,6 @@ #include "video_core/renderer_vulkan/vk_fence_manager.h" #include "video_core/renderer_vulkan/vk_pipeline_cache.h" #include "video_core/renderer_vulkan/vk_query_cache.h" -#include "video_core/renderer_vulkan/vk_render_pass_cache.h" #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h" #include "video_core/renderer_vulkan/vk_texture_cache.h" #include "video_core/renderer_vulkan/vk_update_descriptor.h" @@ -112,7 +111,7 @@ public: void ModifyGPUMemory(size_t as_id, GPUVAddr addr, u64 size) override; void SignalFence(std::function&& func) override; void SyncOperation(std::function&& func) override; - void SignalSyncPoint(u32 value) override; + void SignalSyncPoint(u32 value, u32 count) override; void SignalReference() override; void ReleaseFences(bool force = true) override; void FlushAndInvalidateRegion( @@ -155,11 +154,11 @@ private: static constexpr VkDeviceSize DEFAULT_BUFFER_SIZE = 4 * sizeof(float); template - void PrepareDraw(bool is_indexed, Func&&); + void PrepareDraw(bool is_indexed, bool skip_empty, Func&&); void FlushWork(); - void UpdateDynamicStates(); + void UpdateDynamicStates(const GraphicsPipeline& pipeline); void HandleTransformFeedback(); @@ -180,13 +179,14 @@ private: void UpdateRasterizerDiscardEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateConservativeRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLineStippleEnable(Tegra::Engines::Maxwell3D::Regs& regs); - void UpdateLineStipple(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLineRasterizationMode(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateDepthBiasEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateLogicOpEnable(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateDepthClampEnable(Tegra::Engines::Maxwell3D::Regs& regs); - void UpdateAlphaToCoverageEnable(Tegra::Engines::Maxwell3D::Regs& regs); - void UpdateAlphaToOneEnable(Tegra::Engines::Maxwell3D::Regs& regs); + void UpdateAlphaToCoverageEnable(Tegra::Engines::Maxwell3D::Regs& regs, + const GraphicsPipeline& pipeline); + void UpdateAlphaToOneEnable(Tegra::Engines::Maxwell3D::Regs& regs, + const GraphicsPipeline& pipeline); void UpdateFrontFace(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateStencilOp(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateStencilTestEnable(Tegra::Engines::Maxwell3D::Regs& regs); @@ -194,7 +194,7 @@ private: void UpdateBlending(Tegra::Engines::Maxwell3D::Regs& regs); void UpdateColorWriteEnable(Tegra::Engines::Maxwell3D::Regs& regs); - void UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs); + void UpdateVertexInput(Tegra::Engines::Maxwell3D::Regs& regs, u32 attribute_mask); Tegra::GPU& gpu; Tegra::MaxwellDeviceMemoryManager& device_memory; @@ -210,7 +210,6 @@ private: ComputePassDescriptorQueue compute_pass_descriptor_queue; DescriptorBufferRing descriptor_buffer_ring; BlitImageHelper blit_image; - RenderPassCache render_pass_cache; TextureCacheRuntime texture_cache_runtime; TextureCache texture_cache; @@ -222,13 +221,19 @@ private: AccelerateDMA accelerate_dma; FenceManager fence_manager; - vk::Event wfi_event; boost::container::static_vector image_view_indices; std::array image_view_ids; boost::container::static_vector sampler_handles; u32 draw_counter = 0; + u64 draw_tick = 0; + Tegra::DepthFormat depth_bias_zeta_format{}; + std::array vertex_input_strides{}; + u32 vertex_input_attributes{}; + u8 blend_integer_targets{}; + bool alpha_to_coverage_enabled{}; + bool alpha_to_one_enabled{}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_render_pass_cache.cpp b/src/video_core/renderer_vulkan/vk_render_pass_cache.cpp index 4d8fa15d16..b6be5317e0 100644 --- a/src/video_core/renderer_vulkan/vk_render_pass_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_render_pass_cache.cpp @@ -4,9 +4,7 @@ // SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include "common/container/unordered_map.h" - -#include +#include #include "video_core/renderer_vulkan/maxwell_to_vk.h" #include "video_core/renderer_vulkan/vk_render_pass_cache.h" @@ -19,107 +17,90 @@ namespace { using VideoCore::Surface::PixelFormat; using VideoCore::Surface::SurfaceType; - constexpr SurfaceType GetSurfaceType(PixelFormat format) { - switch (format) { - // Depth formats - case PixelFormat::D16_UNORM: - case PixelFormat::D32_FLOAT: - case PixelFormat::X8_D24_UNORM: - return SurfaceType::Depth; +struct AttachmentAspects { + bool depth; + bool stencil; +}; - // Stencil formats - case PixelFormat::S8_UINT: - return SurfaceType::Stencil; +constexpr SurfaceType GetSurfaceType(PixelFormat format) { + switch (format) { + case PixelFormat::D16_UNORM: + case PixelFormat::D32_FLOAT: + case PixelFormat::X8_D24_UNORM: + return SurfaceType::Depth; + case PixelFormat::S8_UINT: + return SurfaceType::Stencil; + case PixelFormat::D24_UNORM_S8_UINT: + case PixelFormat::S8_UINT_D24_UNORM: + case PixelFormat::D32_FLOAT_S8_UINT: + return SurfaceType::DepthStencil; + default: + return SurfaceType::ColorTexture; + } +} - // Depth+Stencil formats - case PixelFormat::D24_UNORM_S8_UINT: - case PixelFormat::S8_UINT_D24_UNORM: - case PixelFormat::D32_FLOAT_S8_UINT: - return SurfaceType::DepthStencil; +constexpr AttachmentAspects GetAttachmentAspects(PixelFormat format) { + const SurfaceType surface_type = GetSurfaceType(format); + return AttachmentAspects{ + .depth = surface_type == SurfaceType::Depth || surface_type == SurfaceType::DepthStencil, + .stencil = + surface_type == SurfaceType::Stencil || surface_type == SurfaceType::DepthStencil, + }; +} - // Everything else is a color texture - default: - return SurfaceType::ColorTexture; - } - } +VkFormat AttachmentFormat(const Device& device, PixelFormat format) { + return MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, true, format).format; +} - VkAttachmentDescription AttachmentDescription(const Device& device, PixelFormat format, - VkSampleCountFlagBits samples, - VkAttachmentLoadOp load_op, - VkAttachmentStoreOp store_op) { - using MaxwellToVK::SurfaceFormat; +VkRenderingAttachmentInfo MakeAttachment(VkImageView view) { + return { + .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, + .pNext = nullptr, + .imageView = view, + .imageLayout = VK_IMAGE_LAYOUT_GENERAL, + .resolveMode = VK_RESOLVE_MODE_NONE, + .resolveImageView = VK_NULL_HANDLE, + .resolveImageLayout = VK_IMAGE_LAYOUT_GENERAL, + .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, + .storeOp = VK_ATTACHMENT_STORE_OP_STORE, + .clearValue = {}, + }; +} +} // Anonymous namespace - const SurfaceType surface_type = GetSurfaceType(format); - const bool has_stencil = surface_type == SurfaceType::DepthStencil || - surface_type == SurfaceType::Stencil; +ResolveModes PickResolveModes(const Device& device, PixelFormat format) { + constexpr VkResolveModeFlagBits mode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT; - return { - .flags = {}, - .format = SurfaceFormat(device, FormatType::Optimal, true, format).format, - .samples = samples, - .loadOp = load_op, - .storeOp = store_op, - .stencilLoadOp = has_stencil ? load_op : VK_ATTACHMENT_LOAD_OP_DONT_CARE, - .stencilStoreOp = has_stencil ? store_op : VK_ATTACHMENT_STORE_OP_DONT_CARE, - .initialLayout = VK_IMAGE_LAYOUT_GENERAL, - .finalLayout = VK_IMAGE_LAYOUT_GENERAL, - }; - } + const AttachmentAspects aspects = GetAttachmentAspects(format); + const bool depth_mode_supported = (device.GetDepthResolveModes() & mode) != 0; + const bool stencil_mode_supported = (device.GetStencilResolveModes() & mode) != 0; - struct ResolveAspects { - bool depth; - bool stencil; - }; - - struct ResolveModes { - VkResolveModeFlagBits depth; - VkResolveModeFlagBits stencil; - }; - - constexpr ResolveAspects GetResolveAspects(PixelFormat format) { - const SurfaceType surface_type = GetSurfaceType(format); - return ResolveAspects{ - .depth = surface_type == SurfaceType::Depth || - surface_type == SurfaceType::DepthStencil, - .stencil = surface_type == SurfaceType::Stencil || - surface_type == SurfaceType::DepthStencil, - }; - } - - ResolveModes PickResolveModes(const Device& device, PixelFormat format) { - constexpr VkResolveModeFlagBits mode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT; - - const ResolveAspects aspects = GetResolveAspects(format); - const bool depth_mode_supported = (device.GetDepthResolveModes() & mode) != 0; - const bool stencil_mode_supported = (device.GetStencilResolveModes() & mode) != 0; - - ResolveModes modes{ - .depth = VK_RESOLVE_MODE_NONE, - .stencil = VK_RESOLVE_MODE_NONE, - }; - if (aspects.depth && depth_mode_supported) { - modes.depth = mode; - } - if (aspects.stencil && stencil_mode_supported) { - modes.stencil = mode; - } - if (modes.depth == modes.stencil || device.SupportsIndependentResolveNone()) { - return modes; - } - if (modes.depth != VK_RESOLVE_MODE_NONE && stencil_mode_supported) { - modes.stencil = mode; - } else if (modes.stencil != VK_RESOLVE_MODE_NONE && depth_mode_supported) { - modes.depth = mode; - } - return modes; - } - } // Anonymous namespace + ResolveModes modes{ + .depth = VK_RESOLVE_MODE_NONE, + .stencil = VK_RESOLVE_MODE_NONE, + }; + if (aspects.depth && depth_mode_supported) { + modes.depth = mode; + } + if (aspects.stencil && stencil_mode_supported) { + modes.stencil = mode; + } + if (modes.depth == modes.stencil || device.SupportsIndependentResolveNone()) { + return modes; + } + if (modes.depth != VK_RESOLVE_MODE_NONE && stencil_mode_supported) { + modes.stencil = mode; + } else if (modes.stencil != VK_RESOLVE_MODE_NONE && depth_mode_supported) { + modes.depth = mode; + } + return modes; +} bool SupportsDepthStencilResolve(const Device& device, PixelFormat depth_format) { if (depth_format == PixelFormat::Invalid || !device.IsKhrDepthStencilResolveSupported()) { return false; } - const ResolveAspects aspects = GetResolveAspects(depth_format); + const AttachmentAspects aspects = GetAttachmentAspects(depth_format); if (!aspects.depth && !aspects.stencil) { return false; } @@ -131,214 +112,65 @@ bool SupportsDepthStencilResolve(const Device& device, PixelFormat depth_format) return modes.depth == modes.stencil || device.SupportsIndependentResolveNone(); } -RenderPassCache::RenderPassCache(const Device& device_) : device{&device_} {} +RenderingFormats MakeRenderingFormats(const Device& device, std::span colors, + PixelFormat depth) { + RenderingFormats formats{}; + for (size_t index = 0; index < colors.size(); ++index) { + if (colors[index] == PixelFormat::Invalid) { + continue; + } + formats.colors[index] = AttachmentFormat(device, colors[index]); + formats.num_colors = static_cast(index + 1); + } + if (depth == PixelFormat::Invalid) { + return formats; + } + const VkFormat depth_format = AttachmentFormat(device, depth); + const AttachmentAspects aspects = GetAttachmentAspects(depth); + if (aspects.depth) { + formats.depth = depth_format; + } + if (aspects.stencil) { + formats.stencil = depth_format; + } + return formats; +} -VkRenderPass RenderPassCache::Get(const RenderPassKey& key) { - std::scoped_lock lock{mutex}; - const auto [pair, is_new] = cache.try_emplace(key); - if (!is_new) { - return *pair->second; - } - static constexpr size_t MAX_ATTACHMENTS = - 2 * std::tuple_size_v + 2; - boost::container::static_vector descriptions; - std::array references{}; - u32 num_attachments{}; - u32 num_colors{}; - for (size_t index = 0; index < key.color_formats.size(); ++index) { - const PixelFormat format{key.color_formats[index]}; - const bool is_valid{format != PixelFormat::Invalid}; - references[index] = VkAttachmentReference{ - .attachment = is_valid ? num_colors : VK_ATTACHMENT_UNUSED, - .layout = VK_IMAGE_LAYOUT_GENERAL, - }; - if (is_valid) { - const VkAttachmentLoadOp load_op = (key.color_clear_mask & (1u << index)) != 0 - ? VK_ATTACHMENT_LOAD_OP_CLEAR - : VK_ATTACHMENT_LOAD_OP_LOAD; - const VkAttachmentStoreOp store_op = (key.color_discard_mask & (1u << index)) != 0 - ? VK_ATTACHMENT_STORE_OP_DONT_CARE - : VK_ATTACHMENT_STORE_OP_STORE; - descriptions.push_back( - AttachmentDescription(*device, format, key.samples, load_op, store_op)); - num_attachments = static_cast(index + 1); - ++num_colors; - } - } - const bool has_depth{key.depth_format != PixelFormat::Invalid}; - VkAttachmentReference depth_reference{}; - if (key.depth_format != PixelFormat::Invalid) { - depth_reference = VkAttachmentReference{ - .attachment = num_colors, - .layout = VK_IMAGE_LAYOUT_GENERAL, - }; - const VkAttachmentLoadOp depth_load_op = key.depth_stencil_clear - ? VK_ATTACHMENT_LOAD_OP_CLEAR - : VK_ATTACHMENT_LOAD_OP_LOAD; - const VkAttachmentStoreOp depth_store_op = key.depth_stencil_discard - ? VK_ATTACHMENT_STORE_OP_DONT_CARE - : VK_ATTACHMENT_STORE_OP_STORE; - descriptions.push_back(AttachmentDescription(*device, key.depth_format, key.samples, - depth_load_op, depth_store_op)); - } - std::array resolve_references{}; - const bool do_resolve_color = - key.resolve_color && key.samples != VK_SAMPLE_COUNT_1_BIT && num_colors > 0; - if (do_resolve_color) { - for (size_t index = 0; index < key.color_formats.size(); ++index) { - const PixelFormat format{key.color_formats[index]}; - const bool is_valid{format != PixelFormat::Invalid}; - resolve_references[index] = VkAttachmentReference{ - .attachment = is_valid ? static_cast(descriptions.size()) : VK_ATTACHMENT_UNUSED, - .layout = VK_IMAGE_LAYOUT_GENERAL, - }; - if (is_valid) { - VkAttachmentDescription resolve_desc = - AttachmentDescription(*device, format, VK_SAMPLE_COUNT_1_BIT, - VK_ATTACHMENT_LOAD_OP_DONT_CARE, - VK_ATTACHMENT_STORE_OP_STORE); - resolve_desc.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - descriptions.push_back(resolve_desc); - } - } - } - const bool do_resolve_depth_stencil = key.resolve_depth_stencil && has_depth && - key.samples != VK_SAMPLE_COUNT_1_BIT && - SupportsDepthStencilResolve(*device, key.depth_format); - VkAttachmentReference depth_resolve_reference{}; - if (do_resolve_depth_stencil) { - depth_resolve_reference = VkAttachmentReference{ - .attachment = static_cast(descriptions.size()), - .layout = VK_IMAGE_LAYOUT_GENERAL, - }; - VkAttachmentDescription resolve_desc = - AttachmentDescription(*device, key.depth_format, VK_SAMPLE_COUNT_1_BIT, - VK_ATTACHMENT_LOAD_OP_DONT_CARE, VK_ATTACHMENT_STORE_OP_STORE); - resolve_desc.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - descriptions.push_back(resolve_desc); - } - const VkSubpassDescription subpass{ - .flags = 0, - .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, - .inputAttachmentCount = 0, - .pInputAttachments = nullptr, - .colorAttachmentCount = num_attachments, - .pColorAttachments = references.data(), - .pResolveAttachments = do_resolve_color ? resolve_references.data() : nullptr, - .pDepthStencilAttachment = has_depth ? &depth_reference : nullptr, - .preserveAttachmentCount = 0, - .pPreserveAttachments = nullptr, +RenderingAttachments MakeRenderingAttachments(const RenderingFormats& formats, + std::span colors, + VkImageView depth, const VkRect2D& render_area, + u32 layers) { + RenderingAttachments attachments{ + .render_area = render_area, + .num_colors = formats.num_colors, + .layers = layers, }; - const VkSubpassDependency dependency{ - .srcSubpass = 0, // Current subpass - .dstSubpass = 0, // Same subpass (self-dependency) - .srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT, - .dstStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, - .srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT, - .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT - }; - - if (device->IsKhrCreateRenderPass2Supported()) { - boost::container::static_vector descriptions2; - for (const VkAttachmentDescription& description : descriptions) { - descriptions2.push_back(VkAttachmentDescription2{ - .sType = VK_STRUCTURE_TYPE_ATTACHMENT_DESCRIPTION_2, - .pNext = nullptr, - .flags = description.flags, - .format = description.format, - .samples = description.samples, - .loadOp = description.loadOp, - .storeOp = description.storeOp, - .stencilLoadOp = description.stencilLoadOp, - .stencilStoreOp = description.stencilStoreOp, - .initialLayout = description.initialLayout, - .finalLayout = description.finalLayout, - }); - } - const auto promote = [](const VkAttachmentReference& reference) { - return VkAttachmentReference2{ - .sType = VK_STRUCTURE_TYPE_ATTACHMENT_REFERENCE_2, - .pNext = nullptr, - .attachment = reference.attachment, - .layout = reference.layout, - .aspectMask = 0, - }; - }; - std::array references2{}; - std::array resolve_references2{}; - for (size_t index = 0; index < references.size(); ++index) { - references2[index] = promote(references[index]); - resolve_references2[index] = promote(resolve_references[index]); - } - const VkAttachmentReference2 depth_reference2 = promote(depth_reference); - const VkAttachmentReference2 depth_resolve_reference2 = promote(depth_resolve_reference); - const ResolveModes resolve_modes = PickResolveModes(*device, key.depth_format); - const VkSubpassDescriptionDepthStencilResolve depth_stencil_resolve{ - .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_DEPTH_STENCIL_RESOLVE, - .pNext = nullptr, - .depthResolveMode = resolve_modes.depth, - .stencilResolveMode = resolve_modes.stencil, - .pDepthStencilResolveAttachment = &depth_resolve_reference2, - }; - const VkSubpassDescription2 subpass2{ - .sType = VK_STRUCTURE_TYPE_SUBPASS_DESCRIPTION_2, - .pNext = do_resolve_depth_stencil ? &depth_stencil_resolve : nullptr, - .flags = 0, - .pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS, - .viewMask = 0, - .inputAttachmentCount = 0, - .pInputAttachments = nullptr, - .colorAttachmentCount = num_attachments, - .pColorAttachments = references2.data(), - .pResolveAttachments = do_resolve_color ? resolve_references2.data() : nullptr, - .pDepthStencilAttachment = has_depth ? &depth_reference2 : nullptr, - .preserveAttachmentCount = 0, - .pPreserveAttachments = nullptr, - }; - const VkSubpassDependency2 dependency2{ - .sType = VK_STRUCTURE_TYPE_SUBPASS_DEPENDENCY_2, - .pNext = nullptr, - .srcSubpass = dependency.srcSubpass, - .dstSubpass = dependency.dstSubpass, - .srcStageMask = dependency.srcStageMask, - .dstStageMask = dependency.dstStageMask, - .srcAccessMask = dependency.srcAccessMask, - .dstAccessMask = dependency.dstAccessMask, - .dependencyFlags = dependency.dependencyFlags, - .viewOffset = 0, - }; - pair->second = device->GetLogical().CreateRenderPass2({ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO_2, - .pNext = nullptr, - .flags = 0, - .attachmentCount = static_cast(descriptions2.size()), - .pAttachments = descriptions2.empty() ? nullptr : descriptions2.data(), - .subpassCount = 1, - .pSubpasses = &subpass2, - .dependencyCount = 1, - .pDependencies = &dependency2, - .correlatedViewMaskCount = 0, - .pCorrelatedViewMasks = nullptr, - }); - return *pair->second; + attachments.colors.fill(MakeAttachment(VK_NULL_HANDLE)); + std::ranges::transform(colors, attachments.colors.begin(), MakeAttachment); + attachments.depth = MakeAttachment(VK_NULL_HANDLE); + attachments.stencil = attachments.depth; + if (formats.depth != VK_FORMAT_UNDEFINED) { + attachments.depth.imageView = depth; } + if (formats.stencil != VK_FORMAT_UNDEFINED) { + attachments.stencil.imageView = depth; + } + return attachments; +} - pair->second = device->GetLogical().CreateRenderPass({ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO, +void BeginRendering(vk::CommandBuffer cmdbuf, const RenderingAttachments& attachments) { + cmdbuf.BeginRendering(VkRenderingInfo{ + .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, .pNext = nullptr, .flags = 0, - .attachmentCount = static_cast(descriptions.size()), - .pAttachments = descriptions.empty() ? nullptr : descriptions.data(), - .subpassCount = 1, - .pSubpasses = &subpass, - .dependencyCount = 1, - .pDependencies = &dependency, + .renderArea = attachments.render_area, + .layerCount = attachments.layers, + .viewMask = 0, + .colorAttachmentCount = attachments.num_colors, + .pColorAttachments = attachments.colors.data(), + .pDepthAttachment = &attachments.depth, + .pStencilAttachment = &attachments.stencil, }); - return *pair->second; } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_render_pass_cache.h b/src/video_core/renderer_vulkan/vk_render_pass_cache.h index 0edad4639e..a9c5b4e6a5 100644 --- a/src/video_core/renderer_vulkan/vk_render_pass_cache.h +++ b/src/video_core/renderer_vulkan/vk_render_pass_cache.h @@ -6,77 +6,67 @@ #pragma once -#include -#include "common/container/unordered_map.h" +#include +#include -#include "common/container_hash.h" #include "video_core/surface.h" #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { -struct RenderPassKey { - bool operator==(const RenderPassKey&) const noexcept = default; - - std::array color_formats; - VideoCore::Surface::PixelFormat depth_format; - VkSampleCountFlagBits samples; - bool resolve_color; - bool resolve_depth_stencil; - u32 color_clear_mask; - bool depth_stencil_clear; - u32 color_discard_mask; - bool depth_stencil_discard; -}; - -} // namespace Vulkan - -namespace std { -template <> -struct hash { - static_assert(std::tuple_size_v <= 8); - static_assert(static_cast(VideoCore::Surface::PixelFormat::Invalid) <= 0xFF); - static_assert(static_cast(VideoCore::Surface::PixelFormat::Max) <= 0xFF); - static_assert(VK_SAMPLE_COUNT_64_BIT <= 0xFF); - - [[nodiscard]] size_t operator()(const Vulkan::RenderPassKey& key) const noexcept { - u64 formats = 0; - for (size_t index = 0; index < key.color_formats.size(); ++index) { - formats |= static_cast(key.color_formats[index]) << (index * 8); - } - const u64 state = static_cast(key.depth_format) | - (static_cast(key.samples) << 8) | - (static_cast(key.color_clear_mask) << 16) | - (static_cast(key.color_discard_mask) << 24) | - (static_cast(key.resolve_color) << 32) | - (static_cast(key.depth_stencil_clear) << 33) | - (static_cast(key.resolve_depth_stencil) << 34) | - (static_cast(key.depth_stencil_discard) << 35); - size_t seed = 0; - Common::HashCombine(seed, formats); - Common::HashCombine(seed, state); - return seed; - } -}; -} // namespace std - -namespace Vulkan { - class Device; +struct RenderingFormats { + constexpr auto operator<=>(const RenderingFormats&) const noexcept = default; + + [[nodiscard]] VkPipelineRenderingCreateInfo CreateInfo() const noexcept { + return { + .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, + .pNext = nullptr, + .viewMask = 0, + .colorAttachmentCount = num_colors, + .pColorAttachmentFormats = colors.data(), + .depthAttachmentFormat = depth, + .stencilAttachmentFormat = stencil, + }; + } + + std::array colors{}; + u32 num_colors{}; + VkFormat depth{}; + VkFormat stencil{}; +}; + +struct RenderingAttachments { + std::array colors; + VkRenderingAttachmentInfo depth; + VkRenderingAttachmentInfo stencil; + VkRect2D render_area; + u32 num_colors; + u32 layers; +}; + +struct ResolveModes { + VkResolveModeFlagBits depth; + VkResolveModeFlagBits stencil; +}; + [[nodiscard]] bool SupportsDepthStencilResolve(const Device& device, VideoCore::Surface::PixelFormat depth_format); -class RenderPassCache { -public: - explicit RenderPassCache(const Device& device_); +[[nodiscard]] ResolveModes PickResolveModes(const Device& device, + VideoCore::Surface::PixelFormat format); - VkRenderPass Get(const RenderPassKey& key); +[[nodiscard]] RenderingFormats MakeRenderingFormats( + const Device& device, std::span colors, + VideoCore::Surface::PixelFormat depth); -private: - const Device* device{}; - ::Common::unordered_map cache; - std::mutex mutex; -}; +[[nodiscard]] RenderingAttachments MakeRenderingAttachments(const RenderingFormats& formats, + std::span colors, + VkImageView depth, + const VkRect2D& render_area, + u32 layers); + +void BeginRendering(vk::CommandBuffer cmdbuf, const RenderingAttachments& attachments); } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_scheduler.cpp b/src/video_core/renderer_vulkan/vk_scheduler.cpp index 4a7a7d5e78..1851f0b628 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.cpp +++ b/src/video_core/renderer_vulkan/vk_scheduler.cpp @@ -4,8 +4,10 @@ // SPDX-FileCopyrightText: Copyright 2019 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later +#include #include #include +#include #include #include @@ -13,6 +15,7 @@ #include "video_core/renderer_vulkan/vk_query_cache.h" +#include "common/make_unique_for_overwrite.h" #include "common/settings.h" #include "common/thread.h" #include "video_core/gpu_logging/gpu_logging.h" @@ -26,6 +29,35 @@ #include "video_core/vulkan_common/vulkan_wrapper.h" namespace Vulkan { +namespace { +constexpr size_t DEPTH_VIEW = 8; + +struct BeginRenderingCommand { + void operator()(vk::CommandBuffer cmdbuf, vk::CommandBuffer) const { + BeginRendering(cmdbuf, attachments); + } + + RenderingAttachments attachments; +}; + +VkImageView DepthStencilView(const RenderingAttachments& attachments) { + if (attachments.depth.imageView != VK_NULL_HANDLE) { + return attachments.depth.imageView; + } + return attachments.stencil.imageView; +} + +VkAccessFlags2 AttachmentWriteAccess(VkImageAspectFlags aspect_mask) { + if ((aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) != 0) { + return VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT; + } + if ((aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0) { + return VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + } + return VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; +} +} // Anonymous namespace void Scheduler::CommandChunk::ExecuteAll(vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { @@ -46,6 +78,25 @@ Scheduler::Scheduler(const Device& device_, StateTracker& state_tracker_) : device{device_}, state_tracker{state_tracker_}, master_semaphore{std::make_unique(device)}, command_pool{std::make_unique(*master_semaphore, device)} { + compute_write_barrier = VkMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + .srcAccessMask = VK_ACCESS_2_SHADER_WRITE_BIT, + .dstStageMask = device.GetBufferConsumerStages() | vk::PIPELINE_STAGE_ATTACHMENTS, + .dstAccessMask = device.GetBufferConsumerAccess() | vk::ACCESS_ATTACHMENTS, + }; + renderpass_write_barrier = compute_write_barrier; + renderpass_write_barrier.srcStageMask = vk::PIPELINE_STAGE_GRAPHICS_SHADERS; + upload_write_barrier = compute_write_barrier; + upload_write_barrier.srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT; + upload_write_barrier.srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT; + if (device.IsExtTransformFeedbackSupported()) { + renderpass_write_barrier.srcStageMask |= VK_PIPELINE_STAGE_2_TRANSFORM_FEEDBACK_BIT_EXT; + renderpass_write_barrier.srcAccessMask |= + VK_ACCESS_2_TRANSFORM_FEEDBACK_WRITE_BIT_EXT | + VK_ACCESS_2_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT; + } AcquireNewChunk(); AllocateWorkerCommandBuffer(); @@ -56,9 +107,7 @@ Scheduler::~Scheduler() = default; u64 Scheduler::Flush(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { // When flushing, we only send data to the worker thread; no waiting is necessary. - const u64 signal_value = SubmitExecution(signal_semaphore, wait_semaphore); - AllocateNewContext(); - return signal_value; + return SubmitExecution(signal_semaphore, wait_semaphore); } void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { @@ -66,7 +115,6 @@ void Scheduler::Finish(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) const u64 presubmit_tick = CurrentTick(); SubmitExecution(signal_semaphore, wait_semaphore); Wait(presubmit_tick); - AllocateNewContext(); } void Scheduler::WaitWorker() { @@ -89,138 +137,219 @@ void Scheduler::DispatchWork() { work_queue.push(std::move(chunk)); } event_cv.notify_all(); + recorded_attachments = nullptr; + ended_attachments = nullptr; AcquireNewChunk(); } } -void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, - const VkClearValue* clear_values, u32 clear_value_count) { - const VkFramebuffer framebuffer_handle = framebuffer->Handle(); - const VkExtent2D render_area = framebuffer->RenderArea(); - state.renderpass = renderpass; - state.framebuffer = framebuffer_handle; - state.render_area = render_area; +void Scheduler::BeginRenderPassImpl(const Framebuffer* framebuffer) { + const RenderingAttachments& attachments = framebuffer->Attachments(); + PublishComputeWrites(); + state.framebuffer_id = framebuffer->Id(); + ++renderpass_serial; + renderpass_pristine = true; + alias_framebuffer_id = 0; + std::ranges::transform(attachments.colors, renderpass_views.begin(), + &VkRenderingAttachmentInfo::imageView); + renderpass_views[DEPTH_VIEW] = DepthStencilView(attachments); + renderpass_extent = attachments.render_area.extent; + renderpass_layers = attachments.layers; + attachments_touched = 0; + attachments_written = 0; if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { + const VkExtent2D render_area = attachments.render_area.extent; const std::string render_pass_info = fmt::format("renderArea={}x{}, numImages={}", render_area.width, render_area.height, framebuffer->NumImages()); GPU::Logging::GPULogger::GetInstance().LogRenderPassBegin(render_pass_info); } - std::array values{}; - for (u32 i = 0; i < clear_value_count && i < values.size(); ++i) { - values[i] = clear_values[i]; + BeginRenderingCommand command{attachments}; + BeginRenderingCommand* recorded = chunk->Record(command); + if (recorded == nullptr) { + DispatchWork(); + recorded = chunk->Record(command); } - Record([renderpass, framebuffer_handle, render_area, values, clear_value_count]( - vk::CommandBuffer cmdbuf) { - const VkRenderPassBeginInfo renderpass_bi{ - .sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO, - .pNext = nullptr, - .renderPass = renderpass, - .framebuffer = framebuffer_handle, - .renderArea = - { - .offset = {.x = 0, .y = 0}, - .extent = render_area, - }, - .clearValueCount = clear_value_count, - .pClearValues = clear_value_count != 0 ? values.data() : nullptr, - }; - cmdbuf.BeginRenderPass(renderpass_bi, VK_SUBPASS_CONTENTS_INLINE); - }); + recorded_attachments = &recorded->attachments; num_renderpass_images = framebuffer->NumImages(); renderpass_images = framebuffer->Images(); renderpass_image_ranges = framebuffer->ImageRanges(); framebuffer->MarkResolveShadowsUpToDate(); } -void Scheduler::RealizeDeferredClear() { - if (deferred_clear.framebuffer == nullptr) { +void Scheduler::BindFramebuffer(const Framebuffer* framebuffer) { + if (framebuffer->Id() == state.framebuffer_id) { return; } - const DeferredClear dc = deferred_clear; - deferred_clear = {}; + if (CanAliasRenderPass(framebuffer)) { + alias_framebuffer_id = framebuffer->Id(); + return; + } + EndRenderPass(); + BeginRenderPassImpl(framebuffer); +} - std::array clear_values{}; - u32 count = 0; - const RenderPassKey& base = dc.framebuffer->RenderPassKeyBase(); - for (u32 slot = 0; slot < 8; ++slot) { - if (base.color_formats[slot] == VideoCore::Surface::PixelFormat::Invalid) { - continue; +bool Scheduler::CanAliasRenderPass(const Framebuffer* framebuffer) const { + const u64 id = framebuffer->Id(); + if (id == alias_framebuffer_id) { + return true; + } + if (id == state.framebuffer_id || state.framebuffer_id == 0 || + !device.IsExtDynamicRenderingUnusedAttachmentsSupported()) { + return false; + } + const RenderingAttachments& attachments = framebuffer->Attachments(); + if (attachments.layers != renderpass_layers || + attachments.render_area.extent.width != renderpass_extent.width || + attachments.render_area.extent.height != renderpass_extent.height) { + return false; + } + for (u32 slot = 0; slot < attachments.num_colors; ++slot) { + if (attachments.colors[slot].imageView != renderpass_views[slot]) { + return false; } - clear_values[count++] = dc.color_values[slot]; } - if (base.depth_format != VideoCore::Surface::PixelFormat::Invalid) { - clear_values[count++] = dc.depth_stencil_value; - } - const u32 color_discard_mask = - dc.framebuffer->DiscardsMsaaColor() ? dc.color_clear_mask : 0u; - const bool depth_stencil_discard = - dc.depth_stencil && dc.framebuffer->DiscardsMsaaDepthStencil(); - const VkRenderPass renderpass = dc.framebuffer->RenderPassVariant( - dc.color_clear_mask, dc.depth_stencil, color_discard_mask, depth_stencil_discard); - EndRenderPass(); - BeginRenderPassImpl(dc.framebuffer, renderpass, clear_values.data(), count); + const VkImageView depth_stencil = DepthStencilView(attachments); + return depth_stencil == VK_NULL_HANDLE || depth_stencil == renderpass_views[DEPTH_VIEW]; } -bool Scheduler::DeferColorClear(const Framebuffer* framebuffer, u32 rt_slot, - const VkClearValue& value) { - if (IsRenderPassActive()) { +bool Scheduler::IsRenderPassImage(VkImage image) const { + const auto images = std::span(renderpass_images).first(num_renderpass_images); + return std::ranges::find(images, image) != images.end(); +} + +void Scheduler::RequestRenderpass(const Framebuffer* framebuffer, u32 touched, u32 written) { + BindFramebuffer(framebuffer); + renderpass_pristine = false; + attachments_touched |= touched; + attachments_written |= written; +} + +bool Scheduler::OverrideLoadOps(const Framebuffer* framebuffer, u32 attachments, + VkAttachmentLoadOp load_op, const VkClearValue& value) { + BindFramebuffer(framebuffer); + if (!renderpass_pristine || recorded_attachments == nullptr) { return false; } - if (deferred_clear.framebuffer != nullptr && deferred_clear.framebuffer != framebuffer) { - RealizeDeferredClear(); - EndRenderPass(); + const auto set_load_op = [&](VkRenderingAttachmentInfo& attachment) { + attachment.loadOp = load_op; + attachment.clearValue = value; + }; + for (u32 slot = 0; slot < recorded_attachments->num_colors; ++slot) { + if ((attachments & (1u << slot)) != 0) { + set_load_op(recorded_attachments->colors[slot]); + } } - deferred_clear.framebuffer = framebuffer; - deferred_clear.color_clear_mask |= 1u << rt_slot; - deferred_clear.color_values[rt_slot] = value; + if ((attachments & DEPTH_ATTACHMENT_BIT) != 0) { + set_load_op(recorded_attachments->depth); + } + if ((attachments & STENCIL_ATTACHMENT_BIT) != 0) { + set_load_op(recorded_attachments->stencil); + } + attachments_touched |= attachments; + attachments_written |= attachments; return true; } -bool Scheduler::DeferDepthStencilClear(const Framebuffer* framebuffer, const VkClearValue& value) { - if (IsRenderPassActive()) { - return false; - } - if (deferred_clear.framebuffer != nullptr && deferred_clear.framebuffer != framebuffer) { - RealizeDeferredClear(); - EndRenderPass(); - } - deferred_clear.framebuffer = framebuffer; - deferred_clear.depth_stencil = true; - deferred_clear.depth_stencil_value = value; - return true; -} - -void Scheduler::FlushDeferredClear() { - if (deferred_clear.framebuffer == nullptr) { +void Scheduler::DiscardResolvedAttachments(VkImageView resolve_view) { + if (ended_attachments == nullptr) { return; } - RealizeDeferredClear(); - EndRenderPass(); -} - -void Scheduler::RequestRenderpass(const Framebuffer* framebuffer) { - if (deferred_clear.framebuffer == framebuffer) { - RealizeDeferredClear(); - return; - } - const VkRenderPass renderpass = framebuffer->RenderPass(); - const VkFramebuffer framebuffer_handle = framebuffer->Handle(); - const VkExtent2D render_area = framebuffer->RenderArea(); - if (renderpass == state.renderpass && framebuffer_handle == state.framebuffer && - render_area.width == state.render_area.width && - render_area.height == state.render_area.height) { - return; - } - // Ends any active pass and realizes a deferred clear - EndRenderPass(); - BeginRenderPassImpl(framebuffer, renderpass, nullptr, 0); + const auto discard = [resolve_view](VkRenderingAttachmentInfo& attachment) { + if (attachment.resolveImageView == resolve_view) { + attachment.storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + } + }; + std::ranges::for_each(std::span(ended_attachments->colors).first(ended_attachments->num_colors), + discard); + discard(ended_attachments->depth); + discard(ended_attachments->stencil); } void Scheduler::RequestOutsideRenderPassOperationContext() { EndRenderPass(); + PublishComputeWrites(); +} + +void Scheduler::RequestComputeDispatchContext() { + EndRenderPass(); + compute_writes = true; +} + +void Scheduler::RelaxAttachmentOps(RenderingAttachments& attachments) const { + const bool load_op_none = device.IsLoadOpNoneSupported(); + const bool drop_unused = device.IsExtDynamicRenderingUnusedAttachmentsSupported(); + const auto relax = [&](VkRenderingAttachmentInfo& attachment, u32 bit) { + if (attachment.imageView == VK_NULL_HANDLE) { + return true; + } + if (attachment.resolveMode != VK_RESOLVE_MODE_NONE || + attachment.loadOp == VK_ATTACHMENT_LOAD_OP_CLEAR || + attachment.storeOp != VK_ATTACHMENT_STORE_OP_STORE || + (attachments_written & bit) != 0) { + return false; + } + attachment.storeOp = VK_ATTACHMENT_STORE_OP_NONE; + return (attachments_touched & bit) == 0; + }; + for (u32 slot = 0; slot < attachments.num_colors; ++slot) { + VkRenderingAttachmentInfo& color = attachments.colors[slot]; + if (!relax(color, 1u << slot)) { + continue; + } + if (drop_unused) { + color.imageView = VK_NULL_HANDLE; + } else if (load_op_none) { + color.loadOp = VK_ATTACHMENT_LOAD_OP_NONE; + } + } + const bool depth_unused = relax(attachments.depth, DEPTH_ATTACHMENT_BIT); + const bool stencil_unused = relax(attachments.stencil, STENCIL_ATTACHMENT_BIT); + if (drop_unused && depth_unused && stencil_unused) { + attachments.depth.imageView = VK_NULL_HANDLE; + attachments.stencil.imageView = VK_NULL_HANDLE; + } +} + +u32 Scheduler::RefineImageAccesses(const RenderingAttachments& attachments, + std::array& accesses) const { + const auto stores = [](const VkRenderingAttachmentInfo& attachment) { + return attachment.imageView != VK_NULL_HANDLE && + attachment.storeOp != VK_ATTACHMENT_STORE_OP_NONE; + }; + u32 unused_images = 0; + u32 image = 0; + const auto refine = [&](bool used, bool stored) { + if (!used) { + unused_images |= 1u << image; + } + if (!stored) { + accesses[image] = 0; + } + ++image; + }; + for (u32 slot = 0; slot < attachments.num_colors; ++slot) { + if (renderpass_views[slot] != VK_NULL_HANDLE) { + const VkRenderingAttachmentInfo& color = attachments.colors[slot]; + refine(color.imageView != VK_NULL_HANDLE, stores(color)); + } + } + if (renderpass_views[DEPTH_VIEW] != VK_NULL_HANDLE) { + refine(DepthStencilView(attachments) != VK_NULL_HANDLE, + stores(attachments.depth) || stores(attachments.stencil)); + } + return unused_images; +} + +void Scheduler::PublishComputeWrites() { + if (!std::exchange(compute_writes, false)) { + return; + } + Record([barrier = &compute_write_barrier](vk::CommandBuffer cmdbuf) { + cmdbuf.PipelineBarrier(*barrier); + }); } bool Scheduler::UpdateGraphicsPipeline(GraphicsPipeline* pipeline) { @@ -340,19 +469,14 @@ void Scheduler::AllocateWorkerCommandBuffer() { } u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore) { + master_semaphore->Refresh(); EndPendingOperations(); InvalidateState(); const u64 signal_value = master_semaphore->NextTick(); RecordWithUploadBuffer([signal_semaphore, wait_semaphore, signal_value, this](vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf) { - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, - }; - upload_cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, WRITE_BARRIER); + upload_cmdbuf.PipelineBarrier(upload_write_barrier); upload_cmdbuf.End(); cmdbuf.End(); @@ -368,7 +492,7 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { GPU::Logging::GPULogger::GetInstance().LogVulkanCall( - "vkQueueSubmit", "", VK_SUCCESS); + "vkQueueSubmit2", "", VK_SUCCESS); } break; case VK_ERROR_DEVICE_LOST: @@ -384,10 +508,6 @@ u64 Scheduler::SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_se return signal_value; } -void Scheduler::AllocateNewContext() { - // Enable counters once again. These are disabled when a command buffer is finished. -} - void Scheduler::InvalidateState() { state.graphics_pipeline = nullptr; state.rescaling_defined = false; @@ -400,92 +520,75 @@ void Scheduler::EndPendingOperations() { EndRenderPass(); } -void Scheduler::EndRenderPass() - { - RealizeDeferredClear(); - if (!state.renderpass) { - return; - } +void Scheduler::EndRenderPass() { + if (state.framebuffer_id == 0) { + return; + } + std::array accesses{}; + for (u32 index = 0; index < num_renderpass_images; ++index) { + accesses[index] = AttachmentWriteAccess(renderpass_image_ranges[index].aspectMask); + } + u32 unused_images = 0; + if (recorded_attachments != nullptr) { + RelaxAttachmentOps(*recorded_attachments); + unused_images = RefineImageAccesses(*recorded_attachments, accesses); + } + ended_attachments = std::exchange(recorded_attachments, nullptr); - query_cache->CounterClose(VideoCommon::QueryType::StreamingByteCount); + query_cache->CounterClose(VideoCommon::QueryType::StreamingByteCount); - // Log render pass end - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_vulkan_calls.GetValue()) { - GPU::Logging::GPULogger::GetInstance().LogRenderPassEnd(); - } - - query_cache->CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, false); - query_cache->NotifySegment(false); - - Record([num_images = num_renderpass_images, - images = renderpass_images, - ranges = renderpass_image_ranges, - has_transform_feedback = device.IsExtTransformFeedbackSupported()]( - vk::CommandBuffer cmdbuf) { - std::array barriers; - for (size_t i = 0; i < num_images; ++i) { - const VkImageSubresourceRange& range = ranges[i]; - const bool is_color = (range.aspectMask & VK_IMAGE_ASPECT_COLOR_BIT) != 0; - const bool is_depth_stencil = (range.aspectMask - & (VK_IMAGE_ASPECT_DEPTH_BIT - | VK_IMAGE_ASPECT_STENCIL_BIT)) !=0; - - VkAccessFlags src_access = 0; - - if (is_color) - src_access |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - else if (is_depth_stencil) - src_access |= VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - else - src_access |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - - barriers[i] = VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = src_access, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT - | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT - | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT - | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, - .oldLayout = VK_IMAGE_LAYOUT_GENERAL, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = images[i], - .subresourceRange = range, - }; - } - cmdbuf.EndRenderPass(); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, nullptr, nullptr, vk::Span(barriers.data(), num_images)); - if (has_transform_feedback) { - static constexpr VkMemoryBarrier XFB_OUTPUT_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, - .dstAccessMask = VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT, - }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT, - VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, XFB_OUTPUT_BARRIER); - } - }); - - state.renderpass = VkRenderPass{}; - num_renderpass_images = 0; + // Log render pass end + if (GPU::Logging::IsActive() && Settings::values.gpu_log_vulkan_calls.GetValue()) { + GPU::Logging::GPULogger::GetInstance().LogRenderPassEnd(); } + query_cache->CounterEnable(VideoCommon::QueryType::ZPassPixelCount64, false); + query_cache->NotifySegment(false); + + Record([num_images = num_renderpass_images, images = renderpass_images, + ranges = renderpass_image_ranges, accesses, unused_images, + write_barrier = &renderpass_write_barrier, + num_memory_barriers = static_cast(std::exchange(renderpass_writes, false))]( + vk::CommandBuffer cmdbuf) { + std::array barriers; + size_t num_barriers = 0; + for (size_t i = 0; i < num_images; ++i) { + if (((unused_images >> i) & 1) != 0) { + continue; + } + barriers[num_barriers++] = VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = vk::PIPELINE_STAGE_ATTACHMENTS, + .srcAccessMask = accesses[i], + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = images[i], + .subresourceRange = ranges[i], + }; + } + cmdbuf.EndRendering(); + if (num_barriers + num_memory_barriers != 0) { + cmdbuf.PipelineBarrier(0, vk::Span(write_barrier, num_memory_barriers), {}, + vk::Span(barriers.data(), num_barriers)); + } + }); + + state.framebuffer_id = 0; + alias_framebuffer_id = 0; + num_renderpass_images = 0; +} void Scheduler::AcquireNewChunk() { std::scoped_lock rl{reserve_mutex}; if (chunk_reserve.empty()) { // If we don't have anything reserved, we need to make a new chunk. - chunk = std::make_unique(); + chunk = Common::make_unique_for_overwrite(); } else { // Otherwise, we can just take from the reserve. chunk = std::move(chunk_reserve.back()); diff --git a/src/video_core/renderer_vulkan/vk_scheduler.h b/src/video_core/renderer_vulkan/vk_scheduler.h index d4bc8d0ec5..a0bdeedccc 100644 --- a/src/video_core/renderer_vulkan/vk_scheduler.h +++ b/src/video_core/renderer_vulkan/vk_scheduler.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -35,11 +36,16 @@ class GraphicsPipeline; class StateTracker; struct QueryCacheParams; +struct RenderingAttachments; /// The scheduler abstracts command buffer and fence management with an interface that's able to do /// OpenGL-like operations on Vulkan command buffers. class Scheduler { public: + static constexpr u32 DEPTH_ATTACHMENT_BIT = 1u << 8; + static constexpr u32 STENCIL_ATTACHMENT_BIT = 1u << 9; + static constexpr u32 ALL_ATTACHMENTS = 0x3FF; + explicit Scheduler(const Device& device, StateTracker& state_tracker); ~Scheduler(); @@ -57,24 +63,50 @@ public: void DispatchWork(); /// Requests to begin a renderpass. - void RequestRenderpass(const Framebuffer* framebuffer); + void RequestRenderpass(const Framebuffer* framebuffer, u32 touched = ALL_ATTACHMENTS, + u32 written = ALL_ATTACHMENTS); - /// Defers a full-attachment color clear so it becomes the next render pass. - bool DeferColorClear(const Framebuffer* framebuffer, u32 rt_slot, const VkClearValue& value); + bool OverrideLoadOps(const Framebuffer* framebuffer, u32 attachments, + VkAttachmentLoadOp load_op, const VkClearValue& value); - /// Defers a full depth/stencil clear so it becomes the next render pass. - bool DeferDepthStencilClear(const Framebuffer* framebuffer, const VkClearValue& value); + void DiscardResolvedAttachments(VkImageView resolve_view); - /// Realizes any pending deferred clear before its framebuffer can be moved or freed. - void FlushDeferredClear(); + bool CanAliasRenderPass(const Framebuffer* framebuffer) const; + + bool IsRenderPassImage(VkImage image) const; /// Requests the current execution context to be able to execute operations only allowed outside /// of a renderpass. void RequestOutsideRenderPassOperationContext(); + void RequestComputeDispatchContext(); + /// Returns true when a render pass is currently active in the scheduler state. bool IsRenderPassActive() const { - return state.renderpass != VK_NULL_HANDLE; + return state.framebuffer_id != 0; + } + + u64 ActiveRenderPassSerial() const noexcept { + if (state.framebuffer_id != 0) { + return renderpass_serial; + } + return (std::numeric_limits::max)(); + } + + u64 WaitForIdleSerial() const noexcept { + return wfi_serial; + } + + void NotifyWaitForIdle() noexcept { + ++wfi_serial; + } + + void MarkRenderPassWrites() noexcept { + renderpass_writes = true; + } + + bool HasDepthWrites() const noexcept { + return (attachments_written & DEPTH_ATTACHMENT_BIT) != 0; } /// Update the pipeline to the current execution context. @@ -129,39 +161,12 @@ public: return master_semaphore->IsFree(tick); } - /// Waits for the given GPU tick, optionally pacing frames. - void Wait(u64 tick, double target_fps = 0.0) { - if (tick > 0) { - if (tick >= master_semaphore->CurrentTick()) { - Flush(); - } - master_semaphore->Wait(tick); - } - if (Settings::values.use_speed_limit.GetValue() && target_fps > 0.0) { - auto now = std::chrono::steady_clock::now(); - if (last_target_fps != target_fps) { - frame_interval = std::chrono::duration_cast(std::chrono::duration(1.0 / target_fps)); - max_frame_count = static_cast(0.1 * target_fps); - last_target_fps = target_fps; - frame_counter = 0; - start_time = now; - } - frame_counter++; - auto target_time = start_time + frame_interval * frame_counter; - if (target_time >= now) { - constexpr auto spin_tail = std::chrono::milliseconds(1); - auto sleep_time = target_time - now; - if (sleep_time > spin_tail * 2) { - std::this_thread::sleep_for(sleep_time - spin_tail); - } - while (std::chrono::steady_clock::now() < target_time) { - std::this_thread::yield(); - } - } else if (frame_counter > max_frame_count) { - frame_counter = 0; - start_time = now; - } + /// Waits for the given tick to trigger on the GPU. + void Wait(u64 tick) { + if (tick >= master_semaphore->CurrentTick()) { + Flush(); } + master_semaphore->Wait(tick); } /// Returns the master timeline semaphore. @@ -204,6 +209,10 @@ private: command(cmdbuf, upload_cmdbuf); } + T& Get() noexcept { + return command; + } + private: T command; }; @@ -213,16 +222,18 @@ private: void ExecuteAll(vk::CommandBuffer cmdbuf, vk::CommandBuffer upload_cmdbuf); template - bool Record(T& command) { + T* Record(T& command) { using FuncType = TypedCommand; static_assert(sizeof(FuncType) < sizeof(data), "Lambda is too large"); command_offset = Common::AlignUp(command_offset, alignof(FuncType)); if (command_offset > sizeof(data) - sizeof(FuncType)) { - return false; + return nullptr; } Command* const current_last = last; - last = new (data.data() + command_offset) FuncType(std::move(command)); + FuncType* const recorded = + new (data.data() + command_offset) FuncType(std::move(command)); + last = recorded; if (current_last) { current_last->SetNext(last); @@ -230,7 +241,7 @@ private: first = last; } command_offset += sizeof(FuncType); - return true; + return &recorded->Get(); } void MarkSubmit() { @@ -251,13 +262,11 @@ private: size_t command_offset = 0; bool submit = false; - alignas(std::max_align_t) std::array data{}; + alignas(std::max_align_t) std::array data; }; struct State { - VkRenderPass renderpass{}; - VkFramebuffer framebuffer{}; - VkExtent2D render_area = {0, 0}; + u64 framebuffer_id = 0; GraphicsPipeline* graphics_pipeline = nullptr; bool is_rescaling = false; bool rescaling_defined = false; @@ -266,20 +275,9 @@ private: bool descriptor_buffer_bound = false; }; - struct DeferredClear { - const Framebuffer* framebuffer = nullptr; - u32 color_clear_mask = 0; - std::array color_values{}; - bool depth_stencil = false; - VkClearValue depth_stencil_value{}; - }; + void BeginRenderPassImpl(const Framebuffer* framebuffer); - /// Begins a render pass for the given framebuffer, optionally with clear values. - void BeginRenderPassImpl(const Framebuffer* framebuffer, VkRenderPass renderpass, - const VkClearValue* clear_values, u32 clear_value_count); - - /// If a deferred clear is pending. - void RealizeDeferredClear(); + void BindFramebuffer(const Framebuffer* framebuffer); void WorkerThread(std::stop_token stop_token); @@ -287,12 +285,17 @@ private: u64 SubmitExecution(VkSemaphore signal_semaphore, VkSemaphore wait_semaphore); - void AllocateNewContext(); - void EndPendingOperations(); void EndRenderPass(); + void PublishComputeWrites(); + + void RelaxAttachmentOps(RenderingAttachments& attachments) const; + + u32 RefineImageAccesses(const RenderingAttachments& attachments, + std::array& accesses) const; + void AcquireNewChunk(); const Device& device; @@ -306,16 +309,30 @@ private: vk::CommandBuffer current_cmdbuf; vk::CommandBuffer current_upload_cmdbuf; - DeferredClear deferred_clear; - std::unique_ptr chunk; std::function on_submit; State state; + u64 renderpass_serial = 0; + u64 alias_framebuffer_id = 0; + u64 wfi_serial = 0; + bool renderpass_writes = false; + bool renderpass_pristine = false; + bool compute_writes = false; + u32 attachments_touched = 0; + u32 attachments_written = 0; + RenderingAttachments* recorded_attachments = nullptr; + RenderingAttachments* ended_attachments = nullptr; + VkMemoryBarrier2 renderpass_write_barrier{}; + VkMemoryBarrier2 compute_write_barrier{}; + VkMemoryBarrier2 upload_write_barrier{}; u32 num_renderpass_images = 0; std::array renderpass_images{}; std::array renderpass_image_ranges{}; + std::array renderpass_views{}; + VkExtent2D renderpass_extent{}; + u32 renderpass_layers = 0; std::queue> work_queue; std::vector> chunk_reserve; @@ -324,12 +341,6 @@ private: std::mutex queue_mutex; std::condition_variable_any event_cv; std::jthread worker_thread; - - std::chrono::steady_clock::duration frame_interval{}; - std::chrono::steady_clock::time_point start_time{}; - double last_target_fps{}; - u64 max_frame_count{}; - u64 frame_counter{}; }; } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp index 7b6571d33d..e2bc1ad3a4 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.cpp @@ -28,16 +28,7 @@ using namespace Common::Literals; // Maximum potential alignment of a Vulkan buffer constexpr VkDeviceSize MAX_ALIGNMENT = 256; -// Stream buffer size in bytes -// *NIX drivers are more sensitive to increased buffers for streaming. -// Windows ones however, can intake bigger buffers and generally do not OOM. -// - GTX 960 on Windows will not OOM with 256mib -// - GT 1030 on ^NIX will OOM with 256mib -#if defined(__FreeBSD__) constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 128_MiB; -#else -constexpr VkDeviceSize MAX_STREAM_BUFFER_SIZE = 256_MiB; -#endif size_t GetStreamBufferSize(const Device& device) { if (!device.HasDebuggingToolAttached()) { @@ -120,7 +111,14 @@ void StagingBufferPool::FreeDeferred(StagingBufferRef& ref) { } void StagingBufferPool::TickFrame() { - current_delete_level = (current_delete_level + 1) % NUM_LEVELS; + for (size_t step = 0; step < NUM_LEVELS; ++step) { + current_delete_level = (current_delete_level + 1) % NUM_LEVELS; + if (!device_local_cache[current_delete_level].entries.empty() || + !upload_cache[current_delete_level].entries.empty() || + !download_cache[current_delete_level].entries.empty()) { + break; + } + } ReleaseCache(MemoryUsage::DeviceLocal); ReleaseCache(MemoryUsage::Upload); @@ -128,10 +126,12 @@ void StagingBufferPool::TickFrame() { } StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) { - if (AreRegionsActive(Region(free_iterator) + 1, - (std::min)(Region(iterator + size) + 1, NUM_SYNCS))) { - // Avoid waiting for the previous usages to be free - return GetStagingBuffer(size, MemoryUsage::Upload); + const size_t wanted_begin = Region(free_iterator) + 1; + const size_t wanted_end = (std::min)(Region(iterator + size) + 1, NUM_SYNCS); + if (AreRegionsActive(wanted_begin, wanted_end)) { + if (auto ref = OverflowStreamBuffer(size, wanted_begin, wanted_end)) { + return *ref; + } } const u64 current_tick = scheduler.CurrentTick(); std::fill(sync_ticks.begin() + Region(used_iterator), sync_ticks.begin() + Region(iterator), @@ -147,8 +147,9 @@ StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) { free_iterator = size; if (AreRegionsActive(0, Region(size) + 1)) { - // Avoid waiting for the previous usages to be free - return GetStagingBuffer(size, MemoryUsage::Upload); + if (auto ref = OverflowStreamBuffer(size, 0, Region(size) + 1)) { + return *ref; + } } } const size_t offset = iterator; @@ -164,6 +165,27 @@ StagingBufferRef StagingBufferPool::GetStreamBuffer(size_t size) { }; } +u64 StagingBufferPool::MaxRegionTick(size_t region_begin, size_t region_end) const { + u64 tick = 0; + for (size_t region = region_begin; region < region_end; ++region) { + tick = (std::max)(tick, sync_ticks[region]); + } + return tick; +} + +std::optional StagingBufferPool::OverflowStreamBuffer(size_t size, + size_t region_begin, + size_t region_end) { + if (cache_bytes[static_cast(MemoryUsage::Upload)] < stream_buffer_size) { + return GetStagingBuffer(size, MemoryUsage::Upload); + } + if (auto ref = TryGetReservedBuffer(size, MemoryUsage::Upload, false)) { + return ref; + } + scheduler.Wait(MaxRegionTick(region_begin, region_end)); + return std::nullopt; +} + bool StagingBufferPool::AreRegionsActive(size_t region_begin, size_t region_end) const { const u64 gpu_tick = scheduler.GetMasterSemaphore().KnownGpuTick(); return std::any_of(sync_ticks.begin() + region_begin, sync_ticks.begin() + region_end, @@ -211,7 +233,8 @@ StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, MemoryUsage .size = 1ULL << log2_size, .usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | - VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT, + VK_BUFFER_USAGE_INDEX_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, .sharingMode = VK_SHARING_MODE_EXCLUSIVE, .queueFamilyIndexCount = 0, .pQueueFamilyIndices = nullptr, @@ -232,6 +255,7 @@ StagingBufferRef StagingBufferPool::CreateStagingBuffer(size_t size, MemoryUsage device.IsBufferDeviceAddressSupported() ? device.GetLogical().GetBufferDeviceAddress(*buffer) : VkDeviceAddress{}; + cache_bytes[static_cast(usage)] += size_t{1} << log2_size; StagingBuffer& entry = GetCache(usage)[log2_size].entries.emplace_back(StagingBuffer{ .buffer = std::move(buffer), .device_address = buffer_address, @@ -260,12 +284,12 @@ StagingBufferPool::StagingBuffersCache& StagingBufferPool::GetCache(MemoryUsage } void StagingBufferPool::ReleaseCache(MemoryUsage usage) { - ReleaseLevel(GetCache(usage), current_delete_level); + ReleaseLevel(usage, current_delete_level); } -void StagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, size_t log2) { +void StagingBufferPool::ReleaseLevel(MemoryUsage usage, size_t log2) { constexpr size_t deletions_per_tick = 16; - auto& staging = cache[log2]; + auto& staging = GetCache(usage)[log2]; auto& entries = staging.entries; const size_t old_size = entries.size(); @@ -279,6 +303,7 @@ void StagingBufferPool::ReleaseLevel(StagingBuffersCache& cache, size_t log2) { entries.erase(std::remove_if(begin, end, is_deletable), end); const size_t new_size = entries.size(); + cache_bytes[static_cast(usage)] -= (old_size - new_size) << log2; staging.delete_index += deletions_per_tick; if (staging.delete_index >= new_size) { staging.delete_index = 0; diff --git a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h index 6d967abb41..d684a1289b 100644 --- a/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h +++ b/src/video_core/renderer_vulkan/vk_staging_buffer_pool.h @@ -7,6 +7,7 @@ #pragma once #include +#include #include #include "common/common_types.h" @@ -88,6 +89,11 @@ private: bool AreRegionsActive(size_t region_begin, size_t region_end) const; + u64 MaxRegionTick(size_t region_begin, size_t region_end) const; + + std::optional OverflowStreamBuffer(size_t size, size_t region_begin, + size_t region_end); + StagingBufferRef GetStagingBuffer(size_t size, MemoryUsage usage, bool deferred = false); std::optional TryGetReservedBuffer(size_t size, MemoryUsage usage, @@ -99,7 +105,7 @@ private: void ReleaseCache(MemoryUsage usage); - void ReleaseLevel(StagingBuffersCache& cache, size_t log2); + void ReleaseLevel(MemoryUsage usage, size_t log2); size_t Region(size_t iter) const noexcept { return iter / region_size; } @@ -119,6 +125,8 @@ private: size_t free_iterator = 0; std::array sync_ticks{}; + std::array cache_bytes{}; + StagingBuffersCache device_local_cache; StagingBuffersCache upload_cache; StagingBuffersCache download_cache; diff --git a/src/video_core/renderer_vulkan/vk_state_tracker.cpp b/src/video_core/renderer_vulkan/vk_state_tracker.cpp index 0b132c97d6..49b5dded33 100644 --- a/src/video_core/renderer_vulkan/vk_state_tracker.cpp +++ b/src/video_core/renderer_vulkan/vk_state_tracker.cpp @@ -63,7 +63,6 @@ Maxwell3D::DirtyState::Flags MakeInvalidationFlags() { BlendEnable, ConservativeRasterizationMode, LineStippleEnable, - LineStippleParams, }; Maxwell3D::DirtyState::Flags flags{}; for (const int flag : INVALIDATION_FLAGS) { @@ -228,7 +227,6 @@ void SetupDirtyVertexBindings(Maxwell3D::DirtyState::Tables& tables) { void SetupRasterModes(Maxwell3D::DirtyState::Tables &tables) { auto& table = tables[0]; - table[OFF(line_stipple_params)] = LineStippleParams; table[OFF(conservative_raster_enable)] = ConservativeRasterizationMode; table[OFF(line_anti_alias_enable)] = LineRasterizationMode; } diff --git a/src/video_core/renderer_vulkan/vk_state_tracker.h b/src/video_core/renderer_vulkan/vk_state_tracker.h index 6b47ba4176..eebd25a81e 100644 --- a/src/video_core/renderer_vulkan/vk_state_tracker.h +++ b/src/video_core/renderer_vulkan/vk_state_tracker.h @@ -56,7 +56,6 @@ enum : u8 { ConservativeRasterizationMode, LineRasterizationMode, LineStippleEnable, - LineStippleParams, DepthBiasEnable, StateEnable, LogicOp, @@ -89,14 +88,6 @@ public: stencil_reset = true; } - void InvalidateViewports() { - (*flags)[Dirty::Viewports] = true; - } - - void InvalidateScissors() { - (*flags)[Dirty::Scissors] = true; - } - void InvalidateStateEnableFlag() { (*flags)[Dirty::StateEnable] = true; } @@ -220,7 +211,6 @@ public: bool TouchLineStippleEnable() { return Exchange(Dirty::LineStippleEnable, false); } - bool TouchLineStipple() { return Exchange(Dirty::LineStippleParams, false); } bool TouchDepthBiasEnable() { return Exchange(Dirty::DepthBiasEnable, false); } diff --git a/src/video_core/renderer_vulkan/vk_swapchain.cpp b/src/video_core/renderer_vulkan/vk_swapchain.cpp index fea5398557..328bb28897 100644 --- a/src/video_core/renderer_vulkan/vk_swapchain.cpp +++ b/src/video_core/renderer_vulkan/vk_swapchain.cpp @@ -9,10 +9,6 @@ #include #include -#ifdef __ANDROID__ -#include -#endif - #include "common/logging.h" #include "common/settings.h" #include "common/settings_enums.h" @@ -179,35 +175,7 @@ bool Swapchain::AcquireNextImage() { break; } - const auto wait_with_frame_pacing = [this] { - switch (Settings::values.frame_pacing_mode.GetValue()) { - case Settings::FramePacingMode::Target_Auto: - scheduler.Wait(resource_ticks[image_index]); - break; - case Settings::FramePacingMode::Target_30: - scheduler.Wait(resource_ticks[image_index], 30.0); - break; - case Settings::FramePacingMode::Target_60: - scheduler.Wait(resource_ticks[image_index], 60.0); - break; - case Settings::FramePacingMode::Target_90: - scheduler.Wait(resource_ticks[image_index], 90.0); - break; - case Settings::FramePacingMode::Target_120: - scheduler.Wait(resource_ticks[image_index], 120.0); - break; - } - }; - -#ifdef __ANDROID__ - if (android_get_device_api_level() >= 30) { - scheduler.Wait(resource_ticks[image_index]); - } else { - wait_with_frame_pacing(); - } -#else - wait_with_frame_pacing(); -#endif + scheduler.Wait(resource_ticks[image_index]); resource_ticks[image_index] = scheduler.CurrentTick(); diff --git a/src/video_core/renderer_vulkan/vk_swapchain.h b/src/video_core/renderer_vulkan/vk_swapchain.h index d926cc118a..62559355a9 100644 --- a/src/video_core/renderer_vulkan/vk_swapchain.h +++ b/src/video_core/renderer_vulkan/vk_swapchain.h @@ -47,11 +47,6 @@ public: return IsSubOptimal() || NeedsPresentModeUpdate(); } - /// Returns true when the swapchain is outdated. - bool IsOutDated() const { - return is_outdated; - } - /// Returns true when the swapchain is suboptimal. bool IsSubOptimal() const { return is_suboptimal; @@ -69,10 +64,6 @@ public: return image_index; } - std::size_t GetFrameIndex() const { - return frame_index; - } - VkImage GetImageIndex(std::size_t index) const { return images[index]; } @@ -112,7 +103,6 @@ public: private: void CreateSwapchain(const VkSurfaceCapabilitiesKHR& capabilities); void CreateSemaphores(); - void CreateImageViews(); void Destroy(); diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.cpp b/src/video_core/renderer_vulkan/vk_texture_cache.cpp index bc2e82acae..55837ba8c0 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.cpp +++ b/src/video_core/renderer_vulkan/vk_texture_cache.cpp @@ -155,9 +155,75 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { if (Settings::values.accelerate_astc.GetValue() != Settings::AstcDecodeMode::Gpu) { return false; } - return Settings::values.astc_recompression.GetValue() == - Settings::AstcRecompression::Uncompressed && - info.size.depth == 1; + return info.size.depth == 1; +} + +[[nodiscard]] VkFormat UnswizzleStorageFormat(u32 bytes_per_block) { + switch (bytes_per_block) { + case 1: + return VK_FORMAT_R8_UINT; + case 2: + return VK_FORMAT_R16_UINT; + case 4: + return VK_FORMAT_R32_UINT; + case 8: + return VK_FORMAT_R32G32_UINT; + case 16: + return VK_FORMAT_R32G32B32A32_UINT; + default: + return VK_FORMAT_UNDEFINED; + } +} + +[[nodiscard]] bool IsUnswizzleAcceleratedFormat(const Device& device, PixelFormat format) { + if (!device.IsStorageBuffer8BitAccessSupported() || + !device.IsStorageBuffer16BitAccessSupported()) { + return false; + } + if (DefaultBlockWidth(format) != 1 || DefaultBlockHeight(format) != 1) { + return false; + } + if (VideoCore::Surface::GetFormatType(format) != SurfaceType::ColorTexture) { + return false; + } + const VkFormat storage_format = UnswizzleStorageFormat(BytesPerBlock(format)); + if (storage_format == VK_FORMAT_UNDEFINED) { + return false; + } + return device.IsFormatSupported(storage_format, VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT, + FormatType::Optimal); +} + +[[nodiscard]] bool WillUseAcceleratedUnswizzle(const Device& device, const ImageInfo& info) { + const bool supported_type = info.type == ImageType::e2D || info.type == ImageType::e3D || + info.type == ImageType::Linear; + if (!supported_type || info.num_samples > 1) { + return false; + } + if (!device.IsKhrImageFormatListSupported()) { + return false; + } + return IsUnswizzleAcceleratedFormat(device, info.format); +} + +[[nodiscard]] std::array ShaderStorageViewFormats(PixelFormat format) { + if (DefaultBlockWidth(format) != 1 || DefaultBlockHeight(format) != 1) { + return {}; + } + switch (BytesPerBlock(format)) { + case 1: + return {VK_FORMAT_R8_UINT, VK_FORMAT_R8_SINT}; + case 2: + return {VK_FORMAT_R16_UINT, VK_FORMAT_R16_SINT}; + case 4: + return {VK_FORMAT_R32_UINT, VK_FORMAT_UNDEFINED}; + case 8: + return {VK_FORMAT_R32G32_UINT, VK_FORMAT_UNDEFINED}; + case 16: + return {VK_FORMAT_R32G32B32A32_UINT, VK_FORMAT_UNDEFINED}; + default: + return {}; + } } [[nodiscard]] VkImageCreateInfo MakeImageCreateInfo(const Device& device, const ImageInfo& info, @@ -217,11 +283,13 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { [[nodiscard]] vk::Image MakeImage(const Device& device, const MemoryAllocator& allocator, const ImageInfo& info, std::span view_formats, - std::optional format_override = {}) { + std::optional format_override = {}, + VkImageUsageFlags host_usage = 0) { if (info.type == ImageType::Buffer) { return vk::Image{}; } VkImageCreateInfo image_ci = MakeImageCreateInfo(device, info, format_override); + image_ci.usage |= host_usage; const VkImageFormatListCreateInfo image_format_list = { .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_LIST_CREATE_INFO, .pNext = nullptr, @@ -249,7 +317,12 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { } [[nodiscard]] vk::ImageView MakeStorageView(const vk::Device& device, u32 level, VkImage image, - VkFormat format) { + VkFormat format, + VkImageViewType view_type) { + u32 layer_count = VK_REMAINING_ARRAY_LAYERS; + if (view_type != VK_IMAGE_VIEW_TYPE_2D_ARRAY) { + layer_count = 1; + } static constexpr VkImageViewUsageCreateInfo storage_image_view_usage_create_info{ .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO, .pNext = nullptr, @@ -260,7 +333,7 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { .pNext = &storage_image_view_usage_create_info, .flags = 0, .image = image, - .viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY, + .viewType = view_type, .format = format, .components{ .r = VK_COMPONENT_SWIZZLE_IDENTITY, @@ -273,7 +346,7 @@ constexpr VkBorderColor ConvertBorderColor(const std::array& color) { .baseMipLevel = level, .levelCount = 1, .baseArrayLayer = 0, - .layerCount = VK_REMAINING_ARRAY_LAYERS, + .layerCount = layer_count, }, }); } @@ -432,9 +505,11 @@ void SanitizeDepthStencilSwizzle(std::array& swizzle, }; } -[[nodiscard]] VkImageCopy MakeImageCopy(const VideoCommon::ImageCopy& copy, - VkImageAspectFlags aspect_mask) noexcept { - return VkImageCopy{ +[[nodiscard]] VkImageCopy2 MakeImageCopy(const VideoCommon::ImageCopy& copy, + VkImageAspectFlags aspect_mask) noexcept { + return VkImageCopy2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2, + .pNext = nullptr, .srcSubresource = MakeImageSubresourceLayers(copy.src_subresource, aspect_mask), .srcOffset = MakeOffset3D(copy.src_offset), .dstSubresource = MakeImageSubresourceLayers(copy.dst_subresource, aspect_mask), @@ -443,9 +518,12 @@ void SanitizeDepthStencilSwizzle(std::array& swizzle, }; } -[[nodiscard]] VkBufferImageCopy MakeBufferImageCopy(const VideoCommon::ImageCopy& copy, bool is_src, - VkImageAspectFlags aspect_mask) noexcept { - return VkBufferImageCopy{ +[[nodiscard]] VkBufferImageCopy2 MakeBufferImageCopy(const VideoCommon::ImageCopy& copy, + bool is_src, + VkImageAspectFlags aspect_mask) noexcept { + return VkBufferImageCopy2{ + .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2, + .pNext = nullptr, .bufferOffset = 0, .bufferRowLength = 0, .bufferImageHeight = 0, @@ -456,25 +534,13 @@ void SanitizeDepthStencilSwizzle(std::array& swizzle, }; } -[[maybe_unused]] [[nodiscard]] boost::container::small_vector -TransformBufferCopies(std::span copies, size_t buffer_offset) { - boost::container::small_vector result(copies.size()); - std::ranges::transform( - copies, result.begin(), [buffer_offset](const VideoCommon::BufferCopy& copy) { - return VkBufferCopy{ - .srcOffset = static_cast(copy.src_offset + buffer_offset), - .dstOffset = static_cast(copy.dst_offset), - .size = static_cast(copy.size), - }; - }); - return result; -} - -[[nodiscard]] boost::container::small_vector TransformBufferImageCopies( +[[nodiscard]] boost::container::small_vector TransformBufferImageCopies( std::span copies, size_t buffer_offset, VkImageAspectFlags aspect_mask) { struct Maker { - VkBufferImageCopy operator()(const BufferImageCopy& copy) const { - return VkBufferImageCopy{ + VkBufferImageCopy2 operator()(const BufferImageCopy& copy) const { + return VkBufferImageCopy2{ + .sType = VK_STRUCTURE_TYPE_BUFFER_IMAGE_COPY_2, + .pNext = nullptr, .bufferOffset = copy.buffer_offset + buffer_offset, .bufferRowLength = copy.buffer_row_length, .bufferImageHeight = copy.buffer_image_height, @@ -503,14 +569,14 @@ TransformBufferCopies(std::span copies, size_t bu VkImageAspectFlags aspect_mask; }; if (aspect_mask == (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { - boost::container::small_vector result(copies.size() * 2); + boost::container::small_vector result(copies.size() * 2); std::ranges::transform(copies, result.begin(), Maker{buffer_offset, VK_IMAGE_ASPECT_DEPTH_BIT}); std::ranges::transform(copies, result.begin() + copies.size(), Maker{buffer_offset, VK_IMAGE_ASPECT_STENCIL_BIT}); return result; } else { - boost::container::small_vector result(copies.size()); + boost::container::small_vector result(copies.size()); std::ranges::transform(copies, result.begin(), Maker{buffer_offset, aspect_mask}); return result; } @@ -617,14 +683,7 @@ struct RangedBarrierRange { }; void CopyBufferToImage(vk::CommandBuffer cmdbuf, VkBuffer src_buffer, VkImage image, VkImageAspectFlags aspect_mask, bool is_initialized, - std::span copies) { - static constexpr VkAccessFlags WRITE_ACCESS_FLAGS = - VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - static constexpr VkAccessFlags READ_ACCESS_FLAGS = VK_ACCESS_SHADER_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT; - + std::span copies) { // Compute exact mip/layer range being written to RangedBarrierRange range; for (const auto& region : copies) { @@ -632,11 +691,13 @@ void CopyBufferToImage(vk::CommandBuffer cmdbuf, VkBuffer src_buffer, VkImage im } const VkImageSubresourceRange subresource_range = range.SubresourceRange(aspect_mask); - const VkImageMemoryBarrier read_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = WRITE_ACCESS_FLAGS, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = is_initialized ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -645,11 +706,13 @@ void CopyBufferToImage(vk::CommandBuffer cmdbuf, VkBuffer src_buffer, VkImage im .subresourceRange = subresource_range, }; - const VkImageMemoryBarrier write_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 write_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = WRITE_ACCESS_FLAGS | READ_ACCESS_FLAGS, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -658,24 +721,18 @@ void CopyBufferToImage(vk::CommandBuffer cmdbuf, VkBuffer src_buffer, VkImage im .subresourceRange = subresource_range, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, - read_barrier); + cmdbuf.PipelineBarrier(read_barrier); cmdbuf.CopyBufferToImage(src_buffer, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, copies); // TODO: Move this to another API - cmdbuf.PipelineBarrier( - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, - 0, nullptr, nullptr, write_barrier); + cmdbuf.PipelineBarrier(0, {}, {}, write_barrier); } -[[nodiscard]] VkImageBlit MakeImageBlit(const Region2D& dst_region, const Region2D& src_region, - const VkImageSubresourceLayers& dst_layers, - const VkImageSubresourceLayers& src_layers) { - return VkImageBlit{ +[[nodiscard]] VkImageBlit2 MakeImageBlit(const Region2D& dst_region, const Region2D& src_region, + const VkImageSubresourceLayers& dst_layers, + const VkImageSubresourceLayers& src_layers) { + return VkImageBlit2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_BLIT_2, + .pNext = nullptr, .srcSubresource = src_layers, .srcOffsets = { @@ -712,11 +769,13 @@ void CopyBufferToImage(vk::CommandBuffer cmdbuf, VkBuffer src_buffer, VkImage im dst_region.end.y - dst_region.start.y == src_region.end.y - src_region.start.y; } -[[nodiscard]] VkImageResolve MakeImageResolve(const Region2D& dst_region, - const Region2D& src_region, - const VkImageSubresourceLayers& dst_layers, - const VkImageSubresourceLayers& src_layers) { - return VkImageResolve{ +[[nodiscard]] VkImageResolve2 MakeImageResolve(const Region2D& dst_region, + const Region2D& src_region, + const VkImageSubresourceLayers& dst_layers, + const VkImageSubresourceLayers& src_layers) { + return VkImageResolve2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2, + .pNext = nullptr, .srcSubresource = src_layers, .srcOffset = { @@ -819,10 +878,12 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const .x = static_cast(up_scaling ? scaled_width : extent.width), .y = static_cast(up_scaling ? scaled_height : extent.height), }; - boost::container::small_vector regions; + boost::container::small_vector regions; regions.reserve(resources.levels); for (s32 level = 0; level < resources.levels; level++) { regions.push_back({ + .sType = VK_STRUCTURE_TYPE_IMAGE_BLIT_2, + .pNext = nullptr, .srcSubresource{ .aspectMask = aspect_mask, .mipLevel = static_cast(level), @@ -869,11 +930,13 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const .layerCount = VK_REMAINING_ARRAY_LAYERS, }; const std::array read_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -881,13 +944,13 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const .image = src_image, .subresourceRange = subresource_range, }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, // Discard contents .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -897,11 +960,13 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const }, }; const std::array write_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -909,11 +974,13 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const .image = src_image, .subresourceRange = subresource_range, }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_MEMORY_READ_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -922,12 +989,35 @@ void BlitScale(Scheduler& scheduler, VkImage src_image, VkImage dst_image, const .subresourceRange = subresource_range, }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, nullptr, nullptr, read_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, read_barriers); cmdbuf.BlitImage(src_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, regions, vk_filter); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, nullptr, nullptr, write_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, write_barriers); + }); +} +void RecordInitialLayout(Scheduler& scheduler, VkImage image, VkImageAspectFlags aspect_mask) { + const VkImageMemoryBarrier2 barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_NONE, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = image, + .subresourceRange{ + .aspectMask = aspect_mask, + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }; + scheduler.RecordWithUploadBuffer([barrier](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + upload_cmdbuf.PipelineBarrier(barrier); }); } } // Anonymous namespace @@ -936,16 +1026,35 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched MemoryAllocator& memory_allocator_, StagingBufferPool& staging_buffer_pool_, BlitImageHelper& blit_image_helper_, - RenderPassCache& render_pass_cache_, DescriptorPool& descriptor_pool, ComputePassDescriptorQueue& compute_pass_descriptor_queue) : device{device_}, scheduler{scheduler_}, memory_allocator{memory_allocator_}, staging_buffer_pool{staging_buffer_pool_}, blit_image_helper{blit_image_helper_}, - render_pass_cache{render_pass_cache_}, resolution{Settings::values.resolution_info} { + resolution{Settings::values.resolution_info} { if (Settings::values.accelerate_astc.GetValue() == Settings::AstcDecodeMode::Gpu) { astc_decoder_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, compute_pass_descriptor_queue, memory_allocator); } + if (device.IsKhrImageFormatListSupported() && device.IsStorageBuffer8BitAccessSupported() && + device.IsStorageBuffer16BitAccessSupported()) { + bl_unswizzle_2d_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); + bl_unswizzle_3d_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); + pitch_unswizzle_pass.emplace(device, scheduler, descriptor_pool, staging_buffer_pool, + compute_pass_descriptor_queue); + } + for (size_t index = 0; index < VideoCore::Surface::MaxPixelFormat; index++) { + const auto format = static_cast(index); + if (!device.IsHostImageCopySupported() || DefaultBlockWidth(format) == 1) { + continue; + } + const VkFormat vk_format = + MaxwellToVK::SurfaceFormat(device, FormatType::Optimal, false, format).format; + host_copy_formats[index] = + (device.GetPhysical().GetFormatProperties3(vk_format).optimalTilingFeatures & + VK_FORMAT_FEATURE_2_HOST_IMAGE_TRANSFER_BIT_EXT) != 0; + } if (!device.IsKhrImageFormatListSupported()) { return; } @@ -962,20 +1071,24 @@ TextureCacheRuntime::TextureCacheRuntime(const Device& device_, Scheduler& sched view_formats[index_a].push_back(view_info.format); } } + auto& formats = view_formats[index_a]; + for (const VkFormat storage_format : ShaderStorageViewFormats(image_format)) { + if (storage_format == VK_FORMAT_UNDEFINED || + std::ranges::find(formats, storage_format) != formats.end()) { + continue; + } + formats.push_back(storage_format); + } } - if (Settings::values.gpu_unswizzle_enabled.GetValue()) { - bl3d_unswizzle_pass.emplace(device, scheduler, descriptor_pool, - staging_buffer_pool, compute_pass_descriptor_queue); - } } void TextureCacheRuntime::Finish() { scheduler.Finish(); } -StagingBufferRef TextureCacheRuntime::UploadStagingBuffer(size_t size, bool deferred) { - return staging_buffer_pool.Request(size, MemoryUsage::Upload, deferred); +StagingBufferRef TextureCacheRuntime::UploadStagingBuffer(size_t size) { + return staging_buffer_pool.Request(size, MemoryUsage::Upload); } StagingBufferRef TextureCacheRuntime::DownloadStagingBuffer(size_t size, bool deferred) { @@ -1007,11 +1120,12 @@ bool TextureCacheRuntime::ShouldReinterpret(Image& dst, Image& src) { } VkBuffer TextureCacheRuntime::GetTemporaryBuffer(size_t needed_size) { - const auto level = (8 * sizeof(size_t)) - std::countl_zero(needed_size - 1ULL); + const size_t wanted_size = (std::max)(needed_size, size_t{1}); + const auto level = (8 * sizeof(size_t)) - std::countl_zero(wanted_size - 1ULL); if (buffers[level]) { return *buffers[level]; } - const auto new_size = Common::NextPow2(needed_size); + const auto new_size = Common::NextPow2(wanted_size); static constexpr VkBufferUsageFlags flags = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_TEXEL_BUFFER_BIT; @@ -1081,6 +1195,7 @@ VkImageView TextureCacheRuntime::GetOrCreateResolveShadow(VkImage msaa_image, Vk .layerCount = layers, }, }); + RecordInitialLayout(scheduler, *shadow.image, aspect_mask); shadow.format = format; shadow.extent = extent; shadow.layers = layers; @@ -1124,7 +1239,9 @@ void TextureCacheRuntime::EraseResolveShadow(VkImage msaa_image) { } void TextureCacheRuntime::BarrierFeedbackLoop() { - scheduler.RequestOutsideRenderPassOperationContext(); + if (scheduler.HasDepthWrites()) { + scheduler.RequestOutsideRenderPassOperationContext(); + } } void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, @@ -1132,8 +1249,8 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, if (ENABLE_MSAA_RESOLVE_CONSUME) { InvalidateResolveShadow(dst.Handle()); } - boost::container::small_vector vk_in_copies(copies.size()); - boost::container::small_vector vk_out_copies(copies.size()); + boost::container::small_vector vk_in_copies(copies.size()); + boost::container::small_vector vk_out_copies(copies.size()); const VkImageAspectFlags src_aspect_mask = src.AspectMask(); const VkImageAspectFlags dst_aspect_mask = dst.AspectMask(); @@ -1162,32 +1279,28 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, vk_in_copies, vk_out_copies](vk::CommandBuffer cmdbuf) { RangedBarrierRange dst_range; RangedBarrierRange src_range; - for (const VkBufferImageCopy& copy : vk_in_copies) { + for (const VkBufferImageCopy2& copy : vk_in_copies) { src_range.AddLayers(copy.imageSubresource); } - for (const VkBufferImageCopy& copy : vk_out_copies) { + for (const VkBufferImageCopy2& copy : vk_out_copies) { dst_range.AddLayers(copy.imageSubresource); } - static constexpr VkMemoryBarrier READ_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + static constexpr VkMemoryBarrier2 READ_BARRIER{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - }; - static constexpr VkMemoryBarrier WRITE_BARRIER{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_BUFFER_USERS, + .srcAccessMask = vk::ACCESS_BUFFER_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = vk::ACCESS_TRANSFER, }; const std::array pre_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1197,11 +1310,13 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }, }; const std::array middle_in_barrier{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, .srcAccessMask = 0, - .dstAccessMask = 0, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1211,13 +1326,13 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }, }; const std::array middle_out_barrier{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1227,16 +1342,13 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, }, }; const std::array post_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1245,19 +1357,15 @@ void TextureCacheRuntime::ReinterpretImage(Image& dst, Image& src, .subresourceRange = dst_range.SubresourceRange(dst_aspect_mask), }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, {}, {}, pre_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, pre_barriers); cmdbuf.CopyImageToBuffer(src_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, copy_buffer, vk_in_copies); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, WRITE_BARRIER, nullptr, middle_in_barrier); + cmdbuf.PipelineBarrier(0, {}, {}, middle_in_barrier); - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, READ_BARRIER, {}, middle_out_barrier); + cmdbuf.PipelineBarrier(0, READ_BARRIER, {}, middle_out_barrier); cmdbuf.CopyBufferToImage(copy_buffer, dst_image, VK_IMAGE_LAYOUT_GENERAL, vk_out_copies); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, {}, {}, post_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, post_barriers); }); } @@ -1278,7 +1386,24 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst return; } ASSERT(src.format == dst.format); - if (is_src_msaa && !is_dst_msaa && + const bool is_resolve = is_src_msaa && !is_dst_msaa; + if (ENABLE_MSAA_RESOLVE_CONSUME && is_resolve && HaveSameExtent(dst_region, src_region)) { + const VkImageResolve2 resolve = MakeImageResolve( + dst_region, src_region, MakeSubresourceLayers(&dst), MakeSubresourceLayers(&src)); + const VkImageCopy2 region{ + .sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2, + .pNext = nullptr, + .srcSubresource = resolve.srcSubresource, + .srcOffset = resolve.srcOffset, + .dstSubresource = resolve.dstSubresource, + .dstOffset = resolve.dstOffset, + .extent = resolve.extent, + }; + if (CopyResolveShadow(src.ImageHandle(), dst.ImageHandle(), region)) { + return; + } + } + if (is_resolve && (aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0) { if ((aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) == 0) { UNIMPLEMENTED_MSG("Stencil-only MSAA resolve is not supported"); @@ -1301,7 +1426,6 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst } }(); // Use shader-based depth/stencil blits if hardware doesn't support the format - // Note: MSAA resolves (MSAA->single) use vkCmdResolveImage which works fine if (!can_blit_depth_stencil) { UNIMPLEMENTED_IF(is_src_msaa || is_dst_msaa); blit_image_helper.BlitDepthStencil(dst_framebuffer, src, dst_region, src_region, @@ -1322,7 +1446,6 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst return; } - const bool is_resolve = is_src_msaa && !is_dst_msaa; if (is_resolve && !HaveSameExtent(dst_region, src_region)) { blit_image_helper.BlitColorMSAA(dst_framebuffer, src, dst_region, src_region); return; @@ -1336,13 +1459,13 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst scheduler.Record([filter, dst_region, src_region, dst_image, src_image, dst_layers, src_layers, aspect_mask, is_resolve](vk::CommandBuffer cmdbuf) { const std::array read_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1356,13 +1479,13 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1377,14 +1500,13 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst }, }, }; - VkImageMemoryBarrier write_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2 write_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1398,8 +1520,7 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, nullptr, nullptr, read_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, read_barriers); if (is_resolve) { cmdbuf.ResolveImage(src_image, VK_IMAGE_LAYOUT_GENERAL, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, @@ -1411,16 +1532,11 @@ void TextureCacheRuntime::BlitImage(Framebuffer* dst_framebuffer, ImageView& dst src_image, VK_IMAGE_LAYOUT_GENERAL, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, MakeImageBlit(dst_region, src_region, dst_layers, src_layers), vk_filter); } - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, write_barrier); + cmdbuf.PipelineBarrier(write_barrier); }); } void TextureCacheRuntime::ConvertImage(Framebuffer* dst, ImageView& dst_view, ImageView& src_view) { - if (!dst->RenderPass()) { - return; - } - switch (dst_view.format) { case PixelFormat::R16_UNORM: if (src_view.format == PixelFormat::D16_UNORM) { @@ -1558,7 +1674,7 @@ void TextureCacheRuntime::CopyImage(Image& dst, Image& src, }; return ReinterpretImage(dst, src, std::span{&oneCopy, 1}); } - boost::container::small_vector vk_copies(copies.size()); + boost::container::small_vector vk_copies(copies.size()); const VkImageAspectFlags aspect_mask = dst.AspectMask(); ASSERT(aspect_mask == src.AspectMask()); @@ -1571,18 +1687,18 @@ void TextureCacheRuntime::CopyImage(Image& dst, Image& src, scheduler.Record([dst_image, src_image, aspect_mask, vk_copies](vk::CommandBuffer cmdbuf) { RangedBarrierRange dst_range; RangedBarrierRange src_range; - for (const VkImageCopy& copy : vk_copies) { + for (const VkImageCopy2& copy : vk_copies) { dst_range.AddLayers(copy.dstSubresource); src_range.AddLayers(copy.srcSubresource); } const std::array pre_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1590,13 +1706,13 @@ void TextureCacheRuntime::CopyImage(Image& dst, Image& src, .image = src_image, .subresourceRange = src_range.SubresourceRange(aspect_mask), }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1606,11 +1722,13 @@ void TextureCacheRuntime::CopyImage(Image& dst, Image& src, }, }; const std::array post_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, .srcAccessMask = 0, - .dstAccessMask = 0, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1618,16 +1736,13 @@ void TextureCacheRuntime::CopyImage(Image& dst, Image& src, .image = src_image, .subresourceRange = src_range.SubresourceRange(aspect_mask), }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | - VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | - VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -1636,138 +1751,133 @@ void TextureCacheRuntime::CopyImage(Image& dst, Image& src, .subresourceRange = dst_range.SubresourceRange(aspect_mask), }, }; - cmdbuf.PipelineBarrier( - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, nullptr, nullptr, pre_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, pre_barriers); cmdbuf.CopyImage(src_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VideoCommon::FixSmallVectorADL(vk_copies)); - cmdbuf.PipelineBarrier( - VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | - VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, nullptr, nullptr, post_barriers); + cmdbuf.PipelineBarrier(0, {}, {}, post_barriers); }); } +bool TextureCacheRuntime::CopyResolveShadow(VkImage msaa_image, VkImage dst_image, + const VkImageCopy2& region) { + const ResolveShadow* const shadow = GetValidResolveShadow(msaa_image); + if (shadow == nullptr || shadow->aspect_mask != region.srcSubresource.aspectMask || + region.srcSubresource.mipLevel != 0 || region.srcOffset.x < 0 || region.srcOffset.y < 0 || + static_cast(region.srcOffset.x) + region.extent.width > shadow->extent.width || + static_cast(region.srcOffset.y) + region.extent.height > shadow->extent.height || + region.srcSubresource.baseArrayLayer + region.srcSubresource.layerCount > + shadow->layers) { + return false; + } + const VkImageAspectFlags aspect_mask = shadow->aspect_mask; + const VkImage shadow_image = *shadow->image; + scheduler.RequestOutsideRenderPassOperationContext(); + scheduler.Record([shadow_image, dst_image, region, aspect_mask](vk::CommandBuffer cmdbuf) { + const std::array pre_barriers{ + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = shadow_image, + .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, + VK_REMAINING_ARRAY_LAYERS}, + }, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .oldLayout = VK_IMAGE_LAYOUT_GENERAL, + .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, + VK_REMAINING_ARRAY_LAYERS}, + }, + }; + const std::array post_barriers{ + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = 0, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = shadow_image, + .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, + VK_REMAINING_ARRAY_LAYERS}, + }, + VkImageMemoryBarrier2{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, + .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, + .image = dst_image, + .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, + VK_REMAINING_ARRAY_LAYERS}, + }, + }; + cmdbuf.PipelineBarrier(0, {}, {}, pre_barriers); + cmdbuf.CopyImage(shadow_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_image, + VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, region); + cmdbuf.PipelineBarrier(0, {}, {}, post_barriers); + }); + const bool is_color = aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT; + if ((is_color && ENABLE_MSAA_COLOR_DISCARD) || + (!is_color && ENABLE_MSAA_DEPTH_STENCIL_DISCARD)) { + scheduler.DiscardResolvedAttachments(*shadow->view); + } + return true; +} + void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, std::span copies) { const bool msaa_to_non_msaa = src.info.num_samples > 1 && dst.info.num_samples == 1; const u32 num_samples = msaa_to_non_msaa ? src.info.num_samples : dst.info.num_samples; if (ENABLE_MSAA_RESOLVE_CONSUME && msaa_to_non_msaa && copies.size() == 1 && - src.info.format == dst.info.format) { + src.info.format == dst.info.format && copies.front().src_offset.x == 0 && + copies.front().src_offset.y == 0) { const VideoCommon::ImageCopy& copy = copies.front(); - const ResolveShadow* const shadow = GetValidResolveShadow(src.Handle()); - if (shadow != nullptr && shadow->aspect_mask == dst.AspectMask() && - copy.src_offset.x == 0 && copy.src_offset.y == 0 && - copy.src_subresource.base_level == 0 && - static_cast(copy.extent.width) <= shadow->extent.width && - static_cast(copy.extent.height) <= shadow->extent.height && - static_cast(copy.src_subresource.base_layer + copy.src_subresource.num_layers) <= - shadow->layers) { - const VkImageAspectFlags aspect_mask = shadow->aspect_mask; - VkPipelineStageFlags attachment_stage = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; - VkAccessFlags attachment_write = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - VkAccessFlags attachment_read_write = - VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; - if ((aspect_mask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) != 0) { - attachment_stage = VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; - attachment_write = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - attachment_read_write = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; - } - const VkImage shadow_image = *shadow->image; - const VkImage dst_image = dst.Handle(); - const VkImageCopy region{ - .srcSubresource{ - .aspectMask = aspect_mask, - .mipLevel = 0, - .baseArrayLayer = static_cast(copy.src_subresource.base_layer), - .layerCount = static_cast(copy.src_subresource.num_layers), - }, - .srcOffset = {0, 0, 0}, - .dstSubresource{ - .aspectMask = aspect_mask, - .mipLevel = static_cast(copy.dst_subresource.base_level), - .baseArrayLayer = static_cast(copy.dst_subresource.base_layer), - .layerCount = static_cast(copy.dst_subresource.num_layers), - }, - .dstOffset = {copy.dst_offset.x, copy.dst_offset.y, copy.dst_offset.z}, - .extent = {copy.extent.width, copy.extent.height, 1}, - }; - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([shadow_image, dst_image, region, aspect_mask, attachment_stage, - attachment_write, - attachment_read_write](vk::CommandBuffer cmdbuf) { - const std::array pre_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = attachment_write, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, - .oldLayout = VK_IMAGE_LAYOUT_GENERAL, - .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = shadow_image, - .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, - VK_REMAINING_ARRAY_LAYERS}, - }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | attachment_write | - VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .oldLayout = VK_IMAGE_LAYOUT_GENERAL, - .newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = dst_image, - .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, - VK_REMAINING_ARRAY_LAYERS}, - }, - }; - const std::array post_barriers{ - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = 0, - .dstAccessMask = 0, - .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = shadow_image, - .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, - VK_REMAINING_ARRAY_LAYERS}, - }, - VkImageMemoryBarrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT, - .dstAccessMask = VK_ACCESS_SHADER_READ_BIT | attachment_read_write | - VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, - .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = dst_image, - .subresourceRange{aspect_mask, 0, VK_REMAINING_MIP_LEVELS, 0, - VK_REMAINING_ARRAY_LAYERS}, - }, - }; - cmdbuf.PipelineBarrier(attachment_stage | VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_TRANSFER_BIT, 0, nullptr, nullptr, - pre_barriers); - cmdbuf.CopyImage(shadow_image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, dst_image, - VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, region); - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, - VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, nullptr, nullptr, - post_barriers); - }); + const VkImageCopy2 region{ + .sType = VK_STRUCTURE_TYPE_IMAGE_COPY_2, + .pNext = nullptr, + .srcSubresource{ + .aspectMask = dst.AspectMask(), + .mipLevel = static_cast(copy.src_subresource.base_level), + .baseArrayLayer = static_cast(copy.src_subresource.base_layer), + .layerCount = static_cast(copy.src_subresource.num_layers), + }, + .srcOffset = {0, 0, 0}, + .dstSubresource{ + .aspectMask = dst.AspectMask(), + .mipLevel = static_cast(copy.dst_subresource.base_level), + .baseArrayLayer = static_cast(copy.dst_subresource.base_layer), + .layerCount = static_cast(copy.dst_subresource.num_layers), + }, + .dstOffset = {copy.dst_offset.x, copy.dst_offset.y, copy.dst_offset.z}, + .extent = {copy.extent.width, copy.extent.height, 1}, + }; + if (CopyResolveShadow(src.Handle(), dst.Handle(), region)) { return; } } @@ -1775,17 +1885,17 @@ void TextureCacheRuntime::CopyImageMSAA(Image& dst, Image& src, if ((dst_aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT) != 0) { const bool copies_stencil = (dst_aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) != 0 && device.IsExtShaderStencilExportSupported(); - blit_image_helper.CopyMSAADepth(render_pass_cache, dst.Handle(), dst.info.format, - src.Handle(), src.info.format, num_samples, copies, - copies_stencil, msaa_to_non_msaa); + blit_image_helper.CopyMSAADepth(dst.Handle(), dst.info.format, src.Handle(), + src.info.format, num_samples, copies, copies_stencil, + msaa_to_non_msaa); return; } if ((dst_aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT) == 0) { UNIMPLEMENTED_MSG("Copying images with different samples is not supported."); return; } - blit_image_helper.CopyMSAA(render_pass_cache, dst.Handle(), dst.info.format, src.Handle(), - src.info.format, num_samples, copies, msaa_to_non_msaa); + blit_image_helper.CopyMSAA(dst.Handle(), dst.info.format, src.Handle(), src.info.format, + num_samples, copies, msaa_to_non_msaa); } u64 TextureCacheRuntime::GetDeviceLocalMemory() const { @@ -1814,10 +1924,6 @@ bool TextureCacheRuntime::CanReportMemoryUsage() const { return device.CanReportMemoryUsage(); } -void TextureCacheRuntime::FlushDeferredClear() { - scheduler.FlushDeferredClear(); -} - VkImage TextureCacheRuntime::AcquireMsaaScratchImage(const VkImageCreateInfo& image_ci) { const MsaaScratchKey key{ .format = image_ci.format, @@ -1875,17 +1981,15 @@ void TextureCacheRuntime::TickFrame() { Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu_addr_, VAddr cpu_addr_) : VideoCommon::ImageBase(info_, gpu_addr_, cpu_addr_), scheduler{&runtime_.scheduler}, - runtime{&runtime_}, - original_image(MakeImage(runtime_.device, runtime_.memory_allocator, info, - runtime->ViewFormats(info.format))), - aspect_mask(ImageAspectMask(info.format)) { + runtime{&runtime_}, aspect_mask(ImageAspectMask(info.format)) { if (IsPixelFormatASTC(info.format) && !runtime->device.IsOptimalAstcSupported()) { switch (Settings::values.accelerate_astc.GetValue()) { case Settings::AstcDecodeMode::Gpu: if (WillUseAcceleratedAstcDecode(runtime->device, info)) { flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + break; } - break; + [[fallthrough]]; case Settings::AstcDecodeMode::CpuAsynchronous: flags |= VideoCommon::ImageFlagBits::AsynchronousDecode; break; @@ -1899,18 +2003,24 @@ Image::Image(TextureCacheRuntime& runtime_, const ImageInfo& info_, GPUVAddr gpu flags |= VideoCommon::ImageFlagBits::Converted; flags |= VideoCommon::ImageFlagBits::CostlyLoad; } + if (False(flags & VideoCommon::ImageFlagBits::Converted) && + WillUseAcceleratedUnswizzle(runtime->device, info)) { + flags |= VideoCommon::ImageFlagBits::AcceleratedUpload; + } + VkImageUsageFlags host_usage{}; + if (False(flags & VideoCommon::ImageFlagBits::Converted) && + runtime->host_copy_formats[static_cast(info.format)]) { + host_usage = VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT; + } + original_image = MakeImage(runtime->device, runtime->memory_allocator, info, + runtime->ViewFormats(info.format), {}, host_usage); if (runtime->device.HasDebuggingToolAttached()) { original_image.SetObjectNameEXT(VideoCommon::Name(*this).c_str()); } current_image = &Image::original_image; storage_image_views.resize(info.resources.levels); - if (WillUseAcceleratedAstcDecode(runtime->device, info)) { - const auto& device = runtime->device.GetLogical(); - const VkFormat storage_format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; - for (s32 level = 0; level < info.resources.levels; ++level) { - storage_image_views[level] = - MakeStorageView(device, level, *original_image, storage_format); - } + if (original_image) { + runtime->TransitionImageLayout(*this); } } @@ -1927,48 +2037,6 @@ Image::~Image() { } } -void Image::AllocateComputeUnswizzleBuffer(u32 max_slices) { - using VideoCore::Surface::BytesPerBlock; - - const u32 block_bytes = BytesPerBlock(info.format); // 8 for BC1, 16 for BC6H - const u32 block_width = 4; - const u32 block_height = 4; - - // BCn is 4x4x1 blocks - const u32 blocks_x = (info.size.width + block_width - 1) / block_width; - const u32 blocks_y = (info.size.height + block_height - 1) / block_height; - const u32 blocks_z = (std::min)(max_slices, info.size.depth); - - const u64 block_count = - static_cast(blocks_x) * - static_cast(blocks_y) * - static_cast(blocks_z); - - const VkDeviceSize required_size = block_count * block_bytes; - if (has_compute_unswizzle_buffer && required_size <= compute_unswizzle_buffer_size) { - return; - } - - compute_unswizzle_buffer_size = required_size; - - VkBufferCreateInfo ci{ - .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .size = compute_unswizzle_buffer_size, - .usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | - VK_BUFFER_USAGE_TRANSFER_SRC_BIT, - .sharingMode = VK_SHARING_MODE_EXCLUSIVE, - .queueFamilyIndexCount = 0, - .pQueueFamilyIndices = nullptr, - }; - - compute_unswizzle_buffer = - runtime->memory_allocator.CreateBuffer(ci, MemoryUsage::DeviceLocal); - - has_compute_unswizzle_buffer = true; -} - void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, std::span copies) { // TODO: Move this to another API @@ -2025,14 +2093,13 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, } if (msaa_upload_is_depth) { - runtime->blit_image_helper.CopyMSAADepth(runtime->render_pass_cache, Handle(), - info.format, temp_vk_image, info.format, - info.num_samples, + runtime->blit_image_helper.CopyMSAADepth(Handle(), info.format, temp_vk_image, + info.format, info.num_samples, {image_copies.data(), image_copies.size()}, msaa_upload_copies_stencil, false); } else { - runtime->blit_image_helper.CopyMSAA(runtime->render_pass_cache, Handle(), info.format, - temp_vk_image, info.format, info.num_samples, + runtime->blit_image_helper.CopyMSAA(Handle(), info.format, temp_vk_image, info.format, + info.num_samples, {image_copies.data(), image_copies.size()}, false); } initialized = true; @@ -2051,17 +2118,25 @@ void Image::UploadMemory(VkBuffer buffer, VkDeviceSize offset, return; } - scheduler->RequestOutsideRenderPassOperationContext(); auto vk_copies = TransformBufferImageCopies(copies, offset, aspect_mask); const VkBuffer src_buffer = buffer; const VkImage vk_image = *original_image; const VkImageAspectFlags vk_aspect_mask = aspect_mask; const bool was_initialized = std::exchange(initialized, true); - scheduler->Record([src_buffer, vk_image, vk_aspect_mask, was_initialized, - vk_copies](vk::CommandBuffer cmdbuf) { + auto copy = [src_buffer, vk_image, vk_aspect_mask, was_initialized, + vk_copies](vk::CommandBuffer cmdbuf) { CopyBufferToImage(cmdbuf, src_buffer, vk_image, vk_aspect_mask, was_initialized, VideoCommon::FixSmallVectorADL(vk_copies)); - }); + }; + if (True(flags & ImageFlagBits::ReorderableUpload)) { + scheduler->RecordWithUploadBuffer( + [copy = std::move(copy)](vk::CommandBuffer, vk::CommandBuffer upload_cmdbuf) { + copy(upload_cmdbuf); + }); + } else { + scheduler->RequestOutsideRenderPassOperationContext(); + scheduler->Record(std::move(copy)); + } if (is_rescaled) { ScaleUp(); @@ -2072,6 +2147,33 @@ void Image::UploadMemory(const StagingBufferRef& map, std::span memory, + std::span copies) { + const auto vk_copies = TransformBufferImageCopies(copies, 0, aspect_mask); + boost::container::small_vector regions; + for (const VkBufferImageCopy2& copy : vk_copies) { + regions.push_back(VkMemoryToImageCopyEXT{ + .sType = VK_STRUCTURE_TYPE_MEMORY_TO_IMAGE_COPY_EXT, + .pNext = nullptr, + .pHostPointer = memory.data() + copy.bufferOffset, + .memoryRowLength = copy.bufferRowLength, + .memoryImageHeight = copy.bufferImageHeight, + .imageSubresource = copy.imageSubresource, + .imageOffset = copy.imageOffset, + .imageExtent = copy.imageExtent, + }); + } + runtime->device.GetLogical().CopyMemoryToImageEXT(VkCopyMemoryToImageInfoEXT{ + .sType = VK_STRUCTURE_TYPE_COPY_MEMORY_TO_IMAGE_INFO_EXT, + .pNext = nullptr, + .flags = 0, + .dstImage = *original_image, + .dstImageLayout = VK_IMAGE_LAYOUT_GENERAL, + .regionCount = static_cast(regions.size()), + .pRegions = regions.data(), + }); +} + void Image::DownloadMemory(VkBuffer buffer, size_t offset, std::span copies) { std::array buffer_handles{ @@ -2108,23 +2210,25 @@ void Image::DownloadMemory(std::span buffers_span, std::span o const VkImage temp_vk_image = runtime->AcquireMsaaScratchImage(image_ci); const VkImageAspectFlags temp_aspect_mask = aspect_mask; - const VkAccessFlags attachment_access = - msaa_download_is_depth ? (VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | - VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT) - : (VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | - VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT); - const VkPipelineStageFlags attachment_stage = - msaa_download_is_depth ? (VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | - VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT) - : VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; + const VkAccessFlags2 attachment_access = + msaa_download_is_depth ? (VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT | + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT) + : (VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | + VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT); + const VkPipelineStageFlags2 attachment_stage = + msaa_download_is_depth ? (VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT) + : VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT; scheduler->RequestOutsideRenderPassOperationContext(); scheduler->Record([temp_vk_image, temp_aspect_mask, attachment_access, attachment_stage](vk::CommandBuffer cmdbuf) { - const VkImageMemoryBarrier init_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 init_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, .srcAccessMask = 0, + .dstStageMask = attachment_stage, .dstAccessMask = attachment_access, .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, .newLayout = VK_IMAGE_LAYOUT_GENERAL, @@ -2139,8 +2243,7 @@ void Image::DownloadMemory(std::span buffers_span, std::span o .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, attachment_stage, 0, - init_barrier); + cmdbuf.PipelineBarrier(init_barrier); }); boost::container::small_vector image_copies; @@ -2157,19 +2260,18 @@ void Image::DownloadMemory(std::span buffers_span, std::span o if (msaa_download_is_depth) { runtime->blit_image_helper.CopyMSAADepth( - runtime->render_pass_cache, temp_vk_image, info.format, Handle(), info.format, - info.num_samples, {image_copies.data(), image_copies.size()}, - msaa_download_copies_stencil, true); + temp_vk_image, info.format, Handle(), info.format, info.num_samples, + {image_copies.data(), image_copies.size()}, msaa_download_copies_stencil, true); } else { - runtime->blit_image_helper.CopyMSAA(runtime->render_pass_cache, temp_vk_image, - info.format, Handle(), info.format, - info.num_samples, + runtime->blit_image_helper.CopyMSAA(temp_vk_image, info.format, Handle(), + info.format, info.num_samples, {image_copies.data(), image_copies.size()}, true); } boost::container::small_vector buffers_vector{}; - boost::container::small_vector, 8> + boost::container::small_vector< + boost::container::small_vector, 8> vk_copies; for (size_t index = 0; index < buffers_span.size(); index++) { buffers_vector.emplace_back(buffers_span[index]); @@ -2179,12 +2281,17 @@ void Image::DownloadMemory(std::span buffers_span, std::span o scheduler->RequestOutsideRenderPassOperationContext(); scheduler->Record([buffers = std::move(buffers_vector), image = temp_vk_image, - aspect_mask_ = aspect_mask, vk_copies](vk::CommandBuffer cmdbuf) { - const VkImageMemoryBarrier read_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + aspect_mask_ = aspect_mask, vk_copies, + consumer_stages = runtime->device.GetBufferConsumerStages(), + consumer_access = runtime->device.GetBufferConsumerAccess()]( + vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -2198,25 +2305,28 @@ void Image::DownloadMemory(std::span buffers_span, std::span o .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, read_barrier); + cmdbuf.PipelineBarrier(read_barrier); for (size_t index = 0; index < buffers.size(); index++) { cmdbuf.CopyImageToBuffer(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, buffers[index], vk_copies[index]); } - const VkMemoryBarrier memory_write_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + const VkMemoryBarrier2 memory_write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = consumer_stages, + .dstAccessMask = consumer_access, }; - const VkImageMemoryBarrier image_write_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 image_write_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -2230,8 +2340,7 @@ void Image::DownloadMemory(std::span buffers_span, std::span o .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, memory_write_barrier, nullptr, image_write_barrier); + cmdbuf.PipelineBarrier(0, memory_write_barrier, {}, image_write_barrier); }); runtime->ReleaseMsaaScratchImage(temp_vk_image); } @@ -2241,7 +2350,7 @@ void Image::DownloadMemory(std::span buffers_span, std::span o return; } else { boost::container::small_vector buffers_vector{}; - boost::container::small_vector, 8> + boost::container::small_vector, 8> vk_copies; for (size_t index = 0; index < buffers_span.size(); index++) { buffers_vector.emplace_back(buffers_span[index]); @@ -2250,12 +2359,17 @@ void Image::DownloadMemory(std::span buffers_span, std::span o } scheduler->RequestOutsideRenderPassOperationContext(); scheduler->Record([buffers = std::move(buffers_vector), image = *original_image, - aspect_mask_ = aspect_mask, vk_copies](vk::CommandBuffer cmdbuf) { - const VkImageMemoryBarrier read_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + aspect_mask_ = aspect_mask, vk_copies, + consumer_stages = runtime->device.GetBufferConsumerStages(), + consumer_access = runtime->device.GetBufferConsumerAccess()]( + vk::CommandBuffer cmdbuf) { + const VkImageMemoryBarrier2 read_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT, + .srcStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .srcAccessMask = vk::ACCESS_IMAGE_WRITES, + .dstStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, .oldLayout = VK_IMAGE_LAYOUT_GENERAL, .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -2269,25 +2383,28 @@ void Image::DownloadMemory(std::span buffers_span, std::span o .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER, VK_PIPELINE_STAGE_TRANSFER_BIT, - 0, read_barrier); + cmdbuf.PipelineBarrier(read_barrier); for (size_t index = 0; index < buffers.size(); index++) { cmdbuf.CopyImageToBuffer(image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, buffers[index], vk_copies[index]); } - const VkMemoryBarrier memory_write_barrier{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER, + const VkMemoryBarrier2 memory_write_barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, .pNext = nullptr, - .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, + .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, + .dstStageMask = consumer_stages, + .dstAccessMask = consumer_access, }; - const VkImageMemoryBarrier image_write_barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, + const VkImageMemoryBarrier2 image_write_barrier{ + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, .pNext = nullptr, + .srcStageMask = VK_PIPELINE_STAGE_2_TRANSFER_BIT, .srcAccessMask = 0, - .dstAccessMask = VK_ACCESS_MEMORY_WRITE_BIT, + .dstStageMask = vk::PIPELINE_STAGE_IMAGE_USERS, + .dstAccessMask = vk::ACCESS_IMAGE_USERS, .oldLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, .newLayout = VK_IMAGE_LAYOUT_GENERAL, .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, @@ -2301,8 +2418,7 @@ void Image::DownloadMemory(std::span buffers_span, std::span o .layerCount = VK_REMAINING_ARRAY_LAYERS, }, }; - cmdbuf.PipelineBarrier(VK_PIPELINE_STAGE_TRANSFER_BIT, vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - 0, memory_write_barrier, nullptr, image_write_barrier); + cmdbuf.PipelineBarrier(0, memory_write_barrier, {}, image_write_barrier); }); } @@ -2326,11 +2442,19 @@ VkImageView Image::StorageImageView(s32 level) noexcept { if (!view) { auto format_info = MaxwellToVK::SurfaceFormat(runtime->device, FormatType::Optimal, true, info.format); + VkImageViewType view_type = VK_IMAGE_VIEW_TYPE_2D_ARRAY; if (WillUseAcceleratedAstcDecode(runtime->device, info)) { format_info.format = VK_FORMAT_A8B8G8R8_UNORM_PACK32; + } else if (True(flags & VideoCommon::ImageFlagBits::AcceleratedUpload)) { + format_info.format = UnswizzleStorageFormat(BytesPerBlock(info.format)); + if (info.type == ImageType::e3D) { + view_type = VK_IMAGE_VIEW_TYPE_3D; + } else if (info.type == ImageType::Linear) { + view_type = VK_IMAGE_VIEW_TYPE_2D; + } } - view = MakeStorageView(runtime->device.GetLogical(), level, *(this->*current_image), - format_info.format); + view = MakeStorageView(runtime->device.GetLogical(), level, *original_image, + format_info.format, view_type); } return *view; } @@ -2522,13 +2646,9 @@ ImageView::ImageView(TextureCacheRuntime& runtime, const VideoCommon::ImageViewI if (device->ApiVersion() >= VK_API_VERSION_1_3) { const VkFormatProperties3 properties3 = device->GetPhysical().GetFormatProperties3(format_info.format); - supports_depth_comparison = - (properties3.optimalTilingFeatures & - VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_DEPTH_COMPARISON_BIT) != 0; supports_minmax_filter = (properties3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_FILTER_MINMAX_BIT) != 0; } else { - supports_depth_comparison = true; supports_minmax_filter = (device->GetPhysical().GetFormatProperties(format_info.format).optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_MINMAX_BIT) != 0; @@ -2797,7 +2917,6 @@ Sampler::Sampler(TextureCacheRuntime& runtime, const Tegra::Texture::TSCEntry& t has_added_anisotropy = max_anisotropy > default_anisotropy; has_linear_filtering = mag_filter == VK_FILTER_LINEAR || min_filter == VK_FILTER_LINEAR || mipmap_mode == VK_SAMPLER_MIPMAP_MODE_LINEAR; - has_depth_comparison = tsc.depth_compare_enabled != 0; has_minmax_reduction = reduction_mode != VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT; has_srgb_border_color = tsc.srgb_conversion != 0 && srgb_border_color != border_color; if (has_minmax_reduction && !device.IsExtSamplerFilterMinmaxSupported()) { @@ -2956,9 +3075,6 @@ Framebuffer::Framebuffer(TextureCacheRuntime& runtime, std::span color_buffers, ImageView* depth_buffer, bool is_rescaled_) { - boost::container::small_vector attachments; - RenderPassKey renderpass_key{}; + std::array color_formats{}; + std::array color_views{}; + PixelFormat depth_format = PixelFormat::Invalid; + VkImageView depth_view = VK_NULL_HANDLE; s32 num_layers = 1; is_rescaled = is_rescaled_; @@ -2985,15 +3103,15 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, for (size_t index = 0; index < NUM_RT; ++index) { const ImageView* const color_buffer = color_buffers[index]; if (!color_buffer) { - renderpass_key.color_formats[index] = PixelFormat::Invalid; + color_formats[index] = PixelFormat::Invalid; continue; } width = (std::min)(width, is_rescaled ? resolution.ScaleUp(color_buffer->size.width) : color_buffer->size.width); height = (std::min)(height, is_rescaled ? resolution.ScaleUp(color_buffer->size.height) : color_buffer->size.height); - attachments.push_back(color_buffer->RenderTarget()); - renderpass_key.color_formats[index] = color_buffer->format; + color_views[index] = color_buffer->RenderTarget(); + color_formats[index] = color_buffer->format; num_layers = (std::max)(num_layers, color_buffer->range.extent.layers); images[num_images] = color_buffer->ImageHandle(); image_ranges[num_images] = MakeSubresourceRange(color_buffer); @@ -3001,7 +3119,7 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, samples = color_buffer->Samples(); ++num_images; } - const size_t num_colors = attachments.size(); + const size_t num_colors = num_images; VkImage depth_image = VK_NULL_HANDLE; VkImageAspectFlags depth_aspect_mask = 0; if (depth_buffer) { @@ -3009,8 +3127,8 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, : depth_buffer->size.width); height = (std::min)(height, is_rescaled ? resolution.ScaleUp(depth_buffer->size.height) : depth_buffer->size.height); - attachments.push_back(depth_buffer->RenderTarget()); - renderpass_key.depth_format = depth_buffer->format; + depth_view = depth_buffer->RenderTarget(); + depth_format = depth_buffer->format; num_layers = (std::max)(num_layers, depth_buffer->range.extent.layers); images[num_images] = depth_buffer->ImageHandle(); const VkImageSubresourceRange subresource_range = MakeSubresourceRange(depth_buffer); @@ -3021,71 +3139,55 @@ void Framebuffer::CreateFramebuffer(TextureCacheRuntime& runtime, has_stencil = (subresource_range.aspectMask & VK_IMAGE_ASPECT_STENCIL_BIT) != 0; depth_image = depth_buffer->ImageHandle(); depth_aspect_mask = subresource_range.aspectMask; - } else { - renderpass_key.depth_format = PixelFormat::Invalid; } - renderpass_key.samples = samples; const bool do_resolve_color = ENABLE_MSAA_TILER_RESOLVE && samples != VK_SAMPLE_COUNT_1_BIT && num_colors > 0 && runtime.device.IsTiler(); - renderpass_key.resolve_color = do_resolve_color; const bool do_resolve_depth_stencil = ENABLE_MSAA_TILER_RESOLVE && samples != VK_SAMPLE_COUNT_1_BIT && depth_image != VK_NULL_HANDLE && - runtime.device.IsTiler() && - SupportsDepthStencilResolve(runtime.device, renderpass_key.depth_format); - renderpass_key.resolve_depth_stencil = do_resolve_depth_stencil; + runtime.device.IsTiler() && SupportsDepthStencilResolve(runtime.device, depth_format); - discard_msaa_color = - ENABLE_MSAA_RESOLVE_CONSUME && ENABLE_MSAA_COLOR_DISCARD && do_resolve_color; - discard_msaa_depth_stencil = ENABLE_MSAA_RESOLVE_CONSUME && - ENABLE_MSAA_DEPTH_STENCIL_DISCARD && do_resolve_depth_stencil; - - renderpass = runtime.render_pass_cache.Get(renderpass_key); - render_pass_key = renderpass_key; - render_pass_cache = &runtime.render_pass_cache; runtime_ptr = &runtime; + id = ++runtime.framebuffer_serial; render_area.width = (std::min)(render_area.width, width); render_area.height = (std::min)(render_area.height, height); + const u32 layers = static_cast((std::max)(num_layers, 1)); + formats = MakeRenderingFormats(runtime.device, color_formats, depth_format); + attachments = MakeRenderingAttachments(formats, color_views, depth_view, + VkRect2D{.offset = {0, 0}, .extent = render_area}, + layers); + if (do_resolve_color) { - const u32 layers = static_cast((std::max)(num_layers, 1)); for (size_t index = 0; index < NUM_RT; ++index) { - const PixelFormat format = renderpass_key.color_formats[index]; - if (format == PixelFormat::Invalid) { + if (color_formats[index] == PixelFormat::Invalid) { continue; } - const VkFormat vk_format = - MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, format).format; const VkImage msaa_image = images[rt_map[index]]; - attachments.push_back(runtime.GetOrCreateResolveShadow( - msaa_image, vk_format, render_area, layers, VK_IMAGE_ASPECT_COLOR_BIT)); + VkRenderingAttachmentInfo& attachment = attachments.colors[index]; + attachment.resolveMode = VK_RESOLVE_MODE_AVERAGE_BIT; + if (VideoCore::Surface::IsPixelFormatInteger(color_formats[index])) { + attachment.resolveMode = VK_RESOLVE_MODE_SAMPLE_ZERO_BIT; + } + attachment.resolveImageView = runtime.GetOrCreateResolveShadow( + msaa_image, formats.colors[index], render_area, layers, VK_IMAGE_ASPECT_COLOR_BIT); resolve_shadow_images[num_resolve_shadows++] = msaa_image; } } if (do_resolve_depth_stencil) { - const u32 layers = static_cast((std::max)(num_layers, 1)); const VkFormat vk_format = - MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, - renderpass_key.depth_format) + MaxwellToVK::SurfaceFormat(runtime.device, FormatType::Optimal, true, depth_format) .format; - attachments.push_back(runtime.GetOrCreateResolveShadow(depth_image, vk_format, render_area, - layers, depth_aspect_mask)); + const VkImageView resolve_view = runtime.GetOrCreateResolveShadow( + depth_image, vk_format, render_area, layers, depth_aspect_mask); + const ResolveModes modes = PickResolveModes(runtime.device, depth_format); + attachments.depth.resolveMode = modes.depth; + attachments.depth.resolveImageView = resolve_view; + attachments.stencil.resolveMode = modes.stencil; + attachments.stencil.resolveImageView = resolve_view; resolve_shadow_images[num_resolve_shadows++] = depth_image; } - - num_color_buffers = static_cast(num_colors); - framebuffer = runtime.device.GetLogical().CreateFramebuffer({ - .sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .renderPass = renderpass, - .attachmentCount = static_cast(attachments.size()), - .pAttachments = attachments.data(), - .width = render_area.width, - .height = render_area.height, - .layers = static_cast((std::max)(num_layers, 1)), - }); } void Framebuffer::MarkResolveShadowsUpToDate() const { @@ -3097,86 +3199,69 @@ void Framebuffer::MarkResolveShadowsUpToDate() const { } } -VkRenderPass Framebuffer::RenderPassVariant(u32 color_clear_mask, bool depth_stencil_clear, - u32 color_discard_mask, - bool depth_stencil_discard) const { - if (color_clear_mask == 0 && !depth_stencil_clear && color_discard_mask == 0 && - !depth_stencil_discard) { - return renderpass; - } - static_assert(NUM_RT <= 8); - const u32 variant_key = color_clear_mask | (color_discard_mask << 8) | - (static_cast(depth_stencil_clear) << 16) | - (static_cast(depth_stencil_discard) << 17); - for (u32 index = 0; index < num_memoized_variants; ++index) { - if (variant_keys[index] == variant_key) { - return variant_render_passes[index]; - } - } - RenderPassKey key = render_pass_key; - key.color_clear_mask = color_clear_mask; - key.depth_stencil_clear = depth_stencil_clear; - key.color_discard_mask = color_discard_mask; - key.depth_stencil_discard = depth_stencil_discard; - const VkRenderPass variant = render_pass_cache->Get(key); - if (num_memoized_variants < variant_keys.size()) { - variant_keys[num_memoized_variants] = variant_key; - variant_render_passes[num_memoized_variants] = variant; - ++num_memoized_variants; - } - return variant; -} - void TextureCacheRuntime::AccelerateImageUpload( Image& image, const StagingBufferRef& map, - std::span swizzles, - u32 z_start, u32 z_count) { - + std::span swizzles) { + const bool is_rescaled = True(image.flags & ImageFlagBits::Rescaled); + if (is_rescaled) { + image.ScaleDown(true); + } + const bool reorder = !is_rescaled && True(image.flags & ImageFlagBits::ReorderableUpload); if (IsPixelFormatASTC(image.info.format)) { - return astc_decoder_pass->Assemble(image, map, swizzles); + astc_decoder_pass->Assemble(image, map, swizzles, reorder); + } else if (image.info.type == ImageType::e3D) { + bl_unswizzle_3d_pass->Unswizzle(image, map, swizzles, reorder); + } else if (image.info.type == ImageType::Linear) { + pitch_unswizzle_pass->Unswizzle(image, map, swizzles, reorder); + } else { + bl_unswizzle_2d_pass->Unswizzle(image, map, swizzles, reorder); } - - if (!Settings::values.gpu_unswizzle_enabled.GetValue() || !bl3d_unswizzle_pass) { - if (IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D) { - ASSERT(false && "GPU unswizzle is disabled for BCn 3D texture"); - } - ASSERT(false); - return; + if (is_rescaled) { + image.ScaleUp(); } +} - if (bl3d_unswizzle_pass && IsPixelFormatBCn(image.info.format) && image.info.type == ImageType::e3D && image.info.resources.levels == 1 && image.info.resources.layers == 1) { - return bl3d_unswizzle_pass->Unswizzle(image, map, swizzles, z_start, z_count); +void TextureCacheRuntime::ImportGuestMemory(const Common::HostMemory& host_memory) { + guest_memory.emplace(device, memory_allocator, scheduler, host_memory); + if (guest_memory->Empty()) { + guest_memory.reset(); } +} - ASSERT(false); +std::optional TextureCacheRuntime::GuestMemorySource(const u8* pointer, + size_t size) const { + if (pointer == nullptr) { + return std::nullopt; + } + const std::optional range = guest_memory->Find(pointer, size); + if (!range || range->offset % device.GetStorageBufferAlignment() != 0) { + return std::nullopt; + } + return StagingBufferRef{.buffer = range->buffer, .offset = range->offset}; } void TextureCacheRuntime::TransitionImageLayout(Image& image) { - if (!image.ExchangeInitialization()) { - VkImageMemoryBarrier barrier{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, - .pNext = nullptr, - .srcAccessMask = VK_ACCESS_NONE, - .dstAccessMask = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT, - .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, - .newLayout = VK_IMAGE_LAYOUT_GENERAL, - .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, - .image = image.Handle(), - .subresourceRange{ - .aspectMask = image.AspectMask(), - .baseMipLevel = 0, - .levelCount = VK_REMAINING_MIP_LEVELS, - .baseArrayLayer = 0, - .layerCount = VK_REMAINING_ARRAY_LAYERS, - }, - }; - scheduler.RequestOutsideRenderPassOperationContext(); - scheduler.Record([barrier](vk::CommandBuffer cmdbuf) { - cmdbuf.PipelineBarrier(vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, - vk::PIPELINE_STAGE_GRAPHICS_COMPUTE, 0, barrier); - }); + if (image.ExchangeInitialization()) { + return; } + if (!image.CanUploadHostMemory()) { + RecordInitialLayout(scheduler, image.Handle(), image.AspectMask()); + return; + } + device.GetLogical().TransitionImageLayoutEXT(VkHostImageLayoutTransitionInfoEXT{ + .sType = VK_STRUCTURE_TYPE_HOST_IMAGE_LAYOUT_TRANSITION_INFO_EXT, + .pNext = nullptr, + .image = image.Handle(), + .oldLayout = VK_IMAGE_LAYOUT_UNDEFINED, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .subresourceRange{ + .aspectMask = image.AspectMask(), + .baseMipLevel = 0, + .levelCount = VK_REMAINING_MIP_LEVELS, + .baseArrayLayer = 0, + .layerCount = VK_REMAINING_ARRAY_LAYERS, + }, + }); } } // namespace Vulkan diff --git a/src/video_core/renderer_vulkan/vk_texture_cache.h b/src/video_core/renderer_vulkan/vk_texture_cache.h index 2e9736f707..37b476a489 100644 --- a/src/video_core/renderer_vulkan/vk_texture_cache.h +++ b/src/video_core/renderer_vulkan/vk_texture_cache.h @@ -6,12 +6,14 @@ #pragma once +#include #include #include "video_core/texture_cache/texture_cache_base.h" #include "shader_recompiler/shader_info.h" #include "video_core/renderer_vulkan/vk_compute_pass.h" +#include "video_core/renderer_vulkan/vk_guest_memory.h" #include "video_core/renderer_vulkan/vk_render_pass_cache.h" #include "video_core/renderer_vulkan/vk_staging_buffer_pool.h" #include "video_core/texture_cache/image_view_base.h" @@ -36,7 +38,6 @@ class Device; class Image; class ImageView; class Framebuffer; -class RenderPassCache; class StagingBufferPool; class Scheduler; @@ -46,13 +47,12 @@ public: MemoryAllocator& memory_allocator_, StagingBufferPool& staging_buffer_pool_, BlitImageHelper& blit_image_helper_, - RenderPassCache& render_pass_cache_, DescriptorPool& descriptor_pool, ComputePassDescriptorQueue& compute_pass_descriptor_queue); void Finish(); - StagingBufferRef UploadStagingBuffer(size_t size, bool deferred = false); + StagingBufferRef UploadStagingBuffer(size_t size); StagingBufferRef DownloadStagingBuffer(size_t size, bool deferred = false); @@ -60,8 +60,6 @@ public: void TickFrame(); - void FlushDeferredClear(); - u64 GetDeviceLocalMemory() const; u64 GetDeviceMemoryUsage() const; @@ -89,17 +87,21 @@ public: void ConvertImage(Framebuffer* dst, ImageView& dst_view, ImageView& src_view); - bool CanAccelerateImageUpload(Image&) const noexcept { - return false; - } - bool CanUploadMSAA() const noexcept { return true; } void AccelerateImageUpload(Image&, const StagingBufferRef&, - std::span, - u32 z_start, u32 z_count); + std::span); + + void ImportGuestMemory(const Common::HostMemory& host_memory); + + [[nodiscard]] bool HasGuestMemory() const noexcept { + return guest_memory.has_value(); + } + + [[nodiscard]] std::optional GuestMemorySource(const u8* pointer, + size_t size) const; void InsertUploadMemoryBarrier() {} @@ -139,6 +141,8 @@ public: void EraseResolveShadow(VkImage msaa_image); + bool CopyResolveShadow(VkImage msaa_image, VkImage dst_image, const VkImageCopy2& region); + std::span ViewFormats(PixelFormat format) { return view_formats[static_cast(format)]; } @@ -155,12 +159,14 @@ public: MemoryAllocator& memory_allocator; StagingBufferPool& staging_buffer_pool; BlitImageHelper& blit_image_helper; - RenderPassCache& render_pass_cache; std::optional astc_decoder_pass; - - std::optional bl3d_unswizzle_pass; + std::optional bl_unswizzle_2d_pass; + std::optional bl_unswizzle_3d_pass; + std::optional pitch_unswizzle_pass; + std::optional guest_memory; const Settings::ResolutionScalingInfo& resolution; std::array, VideoCore::Surface::MaxPixelFormat> view_formats; + std::bitset host_copy_formats; static constexpr size_t indexing_slots = 8 * sizeof(size_t); std::array buffers{}; @@ -188,6 +194,7 @@ public: std::vector msaa_scratch_images; ::Common::unordered_map resolve_shadows; std::vector> pending_resolve_shadows; + u64 framebuffer_serial{}; }; class Framebuffer { @@ -210,22 +217,18 @@ public: std::span color_buffers, ImageView* depth_buffer, bool is_rescaled = false); - [[nodiscard]] VkFramebuffer Handle() const noexcept { - return *framebuffer; + [[nodiscard]] u64 Id() const noexcept { + return id; } - [[nodiscard]] VkRenderPass RenderPass() const noexcept { - return renderpass; + [[nodiscard]] const RenderingFormats& Formats() const noexcept { + return formats; } - [[nodiscard]] const RenderPassKey& RenderPassKeyBase() const noexcept { - return render_pass_key; + [[nodiscard]] const RenderingAttachments& Attachments() const noexcept { + return attachments; } - [[nodiscard]] VkRenderPass RenderPassVariant(u32 color_clear_mask, bool depth_stencil_clear, - u32 color_discard_mask, - bool depth_stencil_discard) const; - [[nodiscard]] VkExtent2D RenderArea() const noexcept { return render_area; } @@ -234,10 +237,6 @@ public: return samples; } - [[nodiscard]] u32 NumColorBuffers() const noexcept { - return num_color_buffers; - } - [[nodiscard]] u32 NumImages() const noexcept { return num_images; } @@ -266,26 +265,16 @@ public: return is_rescaled; } - [[nodiscard]] bool DiscardsMsaaColor() const noexcept { - return discard_msaa_color; - } - - [[nodiscard]] bool DiscardsMsaaDepthStencil() const noexcept { - return discard_msaa_depth_stencil; - } - /// Records that a render pass has begun, so its resolve attachments will hold valid contents /// once it ends. void MarkResolveShadowsUpToDate() const; private: - static constexpr size_t NUM_MEMOIZED_RENDER_PASS_VARIANTS = 8; - - vk::Framebuffer framebuffer; - VkRenderPass renderpass{}; + RenderingFormats formats{}; + RenderingAttachments attachments{}; + u64 id{}; VkExtent2D render_area{}; VkSampleCountFlagBits samples = VK_SAMPLE_COUNT_1_BIT; - u32 num_color_buffers = 0; u32 num_images = 0; std::array images{}; std::array image_ranges{}; @@ -296,13 +285,6 @@ private: std::array resolve_shadow_images{}; u32 num_resolve_shadows = 0; TextureCacheRuntime* runtime_ptr{nullptr}; - RenderPassKey render_pass_key{}; - RenderPassCache* render_pass_cache{nullptr}; - bool discard_msaa_color{}; - bool discard_msaa_depth_stencil{}; - mutable std::array variant_keys{}; - mutable std::array variant_render_passes{}; - mutable u32 num_memoized_variants{}; }; class Image : public VideoCommon::ImageBase { @@ -325,6 +307,13 @@ public: void UploadMemory(const StagingBufferRef& map, std::span copies); + void UploadHostMemory(std::span memory, + std::span copies); + + [[nodiscard]] bool CanUploadHostMemory() const noexcept { + return (original_image.UsageFlags() & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT) != 0; + } + void DownloadMemory(VkBuffer buffer, size_t offset, std::span copies); @@ -334,8 +323,6 @@ public: void DownloadMemory(const StagingBufferRef& map, std::span copies); - void AllocateComputeUnswizzleImage(); - [[nodiscard]] VkImage Handle() const noexcept { return *(this->*current_image); } @@ -361,10 +348,6 @@ public: bool ScaleDown(bool ignore = false); - u64 allocation_tick; - - friend class BlockLinearUnswizzle3DPass; - private: bool BlitScaleHelper(bool scale_up); @@ -376,12 +359,6 @@ private: vk::Image original_image; vk::Image scaled_image; - vk::Buffer compute_unswizzle_buffer; - VkDeviceSize compute_unswizzle_buffer_size = 0; - bool has_compute_unswizzle_buffer = false; - - void AllocateComputeUnswizzleBuffer(u32 max_slices); - // Use a pointer to field because it is relative, so that the object can be // moved without breaking the reference. vk::Image Image::*current_image{}; @@ -440,10 +417,6 @@ public: return samples; } - [[nodiscard]] bool SupportsDepthComparison() const noexcept { - return supports_depth_comparison; - } - [[nodiscard]] bool RequiresBorderColorFormat() const noexcept { return requires_border_color_format; } @@ -494,7 +467,6 @@ private: VkComponentMapping swizzle_mapping{}; - bool supports_depth_comparison = false; bool requires_border_color_format = false; bool supports_minmax_filter = false; bool has_identity_swizzle = true; @@ -571,7 +543,6 @@ private: bool has_added_anisotropy{}; bool has_linear_filtering{}; - bool has_depth_comparison{}; bool has_minmax_reduction{}; bool has_custom_border_colors{}; bool has_srgb_border_color{}; @@ -579,7 +550,6 @@ private: }; struct TextureCacheParams { - static constexpr bool ENABLE_VALIDATION = true; static constexpr bool FRAMEBUFFER_BLITS = false; static constexpr bool HAS_EMULATED_COPIES = false; static constexpr bool HAS_DEVICE_MEMORY_INFO = true; diff --git a/src/video_core/renderer_vulkan/vk_turbo_mode.cpp b/src/video_core/renderer_vulkan/vk_turbo_mode.cpp index 095908b6cf..8b18ae7d51 100644 --- a/src/video_core/renderer_vulkan/vk_turbo_mode.cpp +++ b/src/video_core/renderer_vulkan/vk_turbo_mode.cpp @@ -208,16 +208,22 @@ void TurboMode::Run(std::stop_token stop_token) { // Finish. cmdbuf.End(); - const VkSubmitInfo submit_info{ - .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO, + const VkCommandBufferSubmitInfo cmdbuf_info{ + .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, .pNext = nullptr, - .waitSemaphoreCount = 0, - .pWaitSemaphores = nullptr, - .pWaitDstStageMask = nullptr, - .commandBufferCount = 1, - .pCommandBuffers = cmdbuf.address(), - .signalSemaphoreCount = 0, - .pSignalSemaphores = nullptr, + .commandBuffer = *cmdbuf, + .deviceMask = 0, + }; + const VkSubmitInfo2 submit_info{ + .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2, + .pNext = nullptr, + .flags = 0, + .waitSemaphoreInfoCount = 0, + .pWaitSemaphoreInfos = nullptr, + .commandBufferInfoCount = 1, + .pCommandBufferInfos = &cmdbuf_info, + .signalSemaphoreInfoCount = 0, + .pSignalSemaphoreInfos = nullptr, }; m_device.GetGraphicsQueue().Submit(std::array{submit_info}, *fence); diff --git a/src/video_core/renderer_vulkan/vk_update_descriptor.h b/src/video_core/renderer_vulkan/vk_update_descriptor.h index 9f30e65dbd..1efe39180f 100644 --- a/src/video_core/renderer_vulkan/vk_update_descriptor.h +++ b/src/video_core/renderer_vulkan/vk_update_descriptor.h @@ -40,17 +40,13 @@ class UpdateDescriptorQueue final { static constexpr size_t FRAMES_IN_FLIGHT = 8; public: - static constexpr size_t GUEST_FRAME_PAYLOAD_SIZE = 0x80000; - static constexpr size_t COMPUTE_FRAME_PAYLOAD_SIZE = 0x20000; + static constexpr size_t GUEST_FRAME_PAYLOAD_SIZE = 0x20000; + static constexpr size_t COMPUTE_FRAME_PAYLOAD_SIZE = 0x8000; explicit UpdateDescriptorQueue(const Device& device_, size_t frame_payload_size_, bool supports_descriptor_buffer_ = false); ~UpdateDescriptorQueue(); - [[nodiscard]] bool UsesDescriptorBuffer() const noexcept { - return use_descriptor_buffer; - } - void TickFrame(); void Acquire(Scheduler& scheduler, size_t required_entries = 0, bool use_descriptor_buffer_ = false); diff --git a/src/video_core/shader_cache.cpp b/src/video_core/shader_cache.cpp index f1aff2810f..d2d6ffeca1 100644 --- a/src/video_core/shader_cache.cpp +++ b/src/video_core/shader_cache.cpp @@ -49,6 +49,7 @@ bool ShaderCache::RefreshStages(std::array& unique_hashes) { return last_shaders_valid; } dirty[VideoCommon::Dirty::Shaders] = false; + fragment_color_outputs = 0; const GPUVAddr base_addr{maxwell3d->regs.program_region.Address()}; for (size_t index = 0; index < Tegra::Engines::Maxwell3D::Regs::MaxShaderProgram; ++index) { @@ -78,6 +79,9 @@ bool ShaderCache::RefreshStages(std::array& unique_hashes) { } shader_infos[index] = shader_info; unique_hashes[index] = shader_info->unique_hash; + if (program == Tegra::Engines::Maxwell3D::Regs::ShaderType::Pixel) { + fragment_color_outputs = shader_info->color_outputs; + } } last_shaders_valid = true; return true; @@ -241,6 +245,13 @@ const ShaderInfo* ShaderCache::MakeShaderInfo(GenericEnvironment& env, VAddr cpu info->unique_hash = env.CalculateHash(); info->size_bytes = env.ReadSizeBytes(); } + if (env.ShaderStage() == Shader::Stage::Fragment) { + for (u32 target = 0; target < Tegra::Engines::Maxwell3D::Regs::NumRenderTargets; ++target) { + if (env.SPH().ps.HasOutputComponents(target)) { + info->color_outputs |= 1U << target; + } + } + } const size_t size_bytes{info->size_bytes}; const ShaderInfo* const result{info.get()}; Register(std::move(info), cpu_addr, size_bytes); diff --git a/src/video_core/shader_cache.h b/src/video_core/shader_cache.h index 997f2b0371..aec003d166 100644 --- a/src/video_core/shader_cache.h +++ b/src/video_core/shader_cache.h @@ -36,6 +36,7 @@ class GenericEnvironment; struct ShaderInfo { u64 unique_hash{}; size_t size_bytes{}; + u32 color_outputs{}; }; class ShaderCache : public VideoCommon::ChannelSetupCaches { @@ -98,6 +99,7 @@ protected: std::array shader_infos{}; bool last_shaders_valid = false; + u32 fragment_color_outputs{}; private: /// @brief Tries to obtain a cached shader starting in a given address diff --git a/src/video_core/surface.cpp b/src/video_core/surface.cpp index 4603870bfa..d1b9bf4151 100644 --- a/src/video_core/surface.cpp +++ b/src/video_core/surface.cpp @@ -6,7 +6,6 @@ #include "common/common_types.h" #include "common/math_util.h" -#include "common/settings.h" #include "video_core/surface.h" namespace VideoCore::Surface { @@ -344,10 +343,6 @@ bool IsPixelFormatETC2(PixelFormat format) { case PixelFormat::ETC2_RGB_SRGB: case PixelFormat::ETC2_RGBA_SRGB: case PixelFormat::ETC2_RGB_PTA_SRGB: - case PixelFormat::EAC_R11_UNORM: - case PixelFormat::EAC_R11_SNORM: - case PixelFormat::EAC_R11G11_UNORM: - case PixelFormat::EAC_R11G11_SNORM: return true; default: return false; @@ -469,14 +464,7 @@ u64 TranscodedAstcSize(u64 base_size, PixelFormat format) { static_cast(DefaultBlockHeight(format)) * RGBA8_PIXEL_SIZE; const u64 uncompressed_size = (base_size * base_block_size) / BytesPerBlock(format); - switch (Settings::values.astc_recompression.GetValue()) { - case Settings::AstcRecompression::Bc1: - return uncompressed_size / 8; - case Settings::AstcRecompression::Bc3: - return uncompressed_size / 4; - default: - return uncompressed_size; - } + return uncompressed_size; } } // namespace VideoCore::Surface diff --git a/src/video_core/surface.h b/src/video_core/surface.h index 8e1973d3b9..5611dbdffa 100644 --- a/src/video_core/surface.h +++ b/src/video_core/surface.h @@ -119,10 +119,6 @@ namespace VideoCore::Surface { PIXEL_FORMAT_ELEM(ETC2_RGB_SRGB, 4, 4, 64) \ PIXEL_FORMAT_ELEM(ETC2_RGBA_SRGB, 4, 4, 128) \ PIXEL_FORMAT_ELEM(ETC2_RGB_PTA_SRGB, 4, 4, 64) \ - PIXEL_FORMAT_ELEM(EAC_R11_UNORM, 4, 4, 64) \ - PIXEL_FORMAT_ELEM(EAC_R11_SNORM, 4, 4, 64) \ - PIXEL_FORMAT_ELEM(EAC_R11G11_UNORM, 4, 4, 128) \ - PIXEL_FORMAT_ELEM(EAC_R11G11_SNORM, 4, 4, 128) \ /* Depth formats */ \ PIXEL_FORMAT_ELEM(D32_FLOAT, 1, 1, 32) \ PIXEL_FORMAT_ELEM(D16_UNORM, 1, 1, 16) \ diff --git a/src/video_core/texture_cache/descriptor_table.h b/src/video_core/texture_cache/descriptor_table.h index e40c128ab5..37022842fe 100644 --- a/src/video_core/texture_cache/descriptor_table.h +++ b/src/video_core/texture_cache/descriptor_table.h @@ -7,12 +7,16 @@ #pragma once #include +#include +#include +#include #include #include "common/alignment.h" #include "common/common_types.h" #include "common/div_ceil.h" #include "common/assert.h" +#include "common/slot_vector.h" #include "video_core/memory_manager.h" #include "video_core/rasterizer_interface.h" @@ -21,53 +25,55 @@ namespace VideoCommon { template class DescriptorTable { public: - [[nodiscard]] bool Synchronize(GPUVAddr gpu_addr, u32 limit) noexcept { - bool ret = !(current_gpu_addr == gpu_addr && current_limit == limit); - if (ret) { + struct Entry { + T descriptor; + Common::SlotId id; + u32 generation; + }; + + void Synchronize(GPUVAddr gpu_addr, u32 limit) noexcept { + if (current_gpu_addr != gpu_addr || current_limit != limit) { Refresh(gpu_addr, limit); } - return ret; } void Invalidate() noexcept { - std::ranges::fill(read_descriptors, 0); + ++generation; } - [[nodiscard]] std::pair Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { + [[nodiscard]] std::pair Read(Tegra::MemoryManager const& gpu_memory, u32 index) noexcept { DEBUG_ASSERT(index <= current_limit); const GPUVAddr gpu_addr = current_gpu_addr + index * sizeof(T); - std::pair result; - gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(result.first), sizeof(T)); - if ((read_descriptors[index / 64] & (1ULL << (index % 64))) != 0) { - result.second = result.first != descriptors[index]; - } else { - read_descriptors[index / 64] |= 1ULL << (index % 64); - result.second = true; + T value{}; + if (!aligned) { + gpu_memory.ReadBlockUnsafe(gpu_addr, std::addressof(value), sizeof(T)); + } else if (const u8* const ptr = gpu_memory.GetPointer(gpu_addr)) { + std::memcpy(std::addressof(value), ptr, sizeof(T)); } - if (result.second) { - descriptors[index] = result.first; + Entry& entry = entries[index]; + const bool is_new = entry.generation != generation || entry.descriptor != value; + if (is_new) { + entry.descriptor = value; + entry.generation = generation; } - return result; + return {entry, is_new}; } void Refresh(GPUVAddr gpu_addr, u32 limit) noexcept { current_gpu_addr = gpu_addr; current_limit = limit; - // Mario Brothership reallocates a lot of times, so use aggressive pre-alloc sizes - // std::vector by default uses quadratic growth, but that isn't even enough to satisfy brothership - const size_t num_descriptors = ((limit + 0x80000) & (~0x7ffff)) + 1; - size_t old_size = read_descriptors.size(); - read_descriptors.resize(Common::DivCeil(num_descriptors, 64U)); - old_size = (std::min)(old_size, read_descriptors.size()); - std::fill(read_descriptors.begin(), read_descriptors.begin() + old_size, 0); - // - descriptors.resize(num_descriptors); + aligned = gpu_addr % sizeof(T) == 0; + ++generation; + if (entries.size() <= limit) { + entries.resize(std::bit_ceil(size_t{limit} + 1)); + } } - std::vector read_descriptors; - std::vector descriptors; + std::vector entries; GPUVAddr current_gpu_addr{}; u32 current_limit{}; + u32 generation{1}; + bool aligned{}; }; } // namespace VideoCommon diff --git a/src/video_core/texture_cache/format_lookup_table.cpp b/src/video_core/texture_cache/format_lookup_table.cpp index d6e019cddf..ea68a1c063 100644 --- a/src/video_core/texture_cache/format_lookup_table.cpp +++ b/src/video_core/texture_cache/format_lookup_table.cpp @@ -204,15 +204,6 @@ PixelFormat PixelFormatFromTextureInfo(TextureFormat format, ComponentType red, return PixelFormat::ETC2_RGB_PTA_SRGB; case Hash(TextureFormat::ETC2_RGBA, UNORM, SRGB): return PixelFormat::ETC2_RGBA_SRGB; - /* EAC */ - case Hash(TextureFormat::EAC, UNORM): - return PixelFormat::EAC_R11_UNORM; - case Hash(TextureFormat::EAC, SNORM): - return PixelFormat::EAC_R11_SNORM; - case Hash(TextureFormat::EACX2, UNORM): - return PixelFormat::EAC_R11G11_UNORM; - case Hash(TextureFormat::EACX2, SNORM): - return PixelFormat::EAC_R11G11_SNORM; /* ASTC */ case Hash(TextureFormat::ASTC_2D_4X4, UNORM, LINEAR): return PixelFormat::ASTC_2D_4X4_UNORM; diff --git a/src/video_core/texture_cache/image_base.h b/src/video_core/texture_cache/image_base.h index 79fc8505d6..3c597a7f1d 100644 --- a/src/video_core/texture_cache/image_base.h +++ b/src/video_core/texture_cache/image_base.h @@ -45,6 +45,7 @@ enum class ImageFlagBits : u32 { AsynchronousDecode = 1 << 16, IsDecoding = 1 << 17, ///< Is currently being decoded asynchronously. + ReorderableUpload = 1 << 18, }; DECLARE_ENUM_FLAG_OPERATORS(ImageFlagBits) @@ -95,6 +96,7 @@ struct ImageBase { u32 scale_rating = 0; u64 scale_tick = 0; bool has_scaled = false; + u32 guest_data_checks = 0; size_t channel = 0; @@ -105,6 +107,7 @@ struct ImageBase { VAddr cpu_addr_end = 0; u64 modification_tick = 0; + u64 guest_data_hash = 0; size_t lru_index = SIZE_MAX; std::array mip_level_offsets{}; diff --git a/src/video_core/texture_cache/texture_cache.h b/src/video_core/texture_cache/texture_cache.h index 640bd38407..2e26880868 100644 --- a/src/video_core/texture_cache/texture_cache.h +++ b/src/video_core/texture_cache/texture_cache.h @@ -8,11 +8,11 @@ #include #include -#include #include "common/container/unordered_map.h" #include #include "common/alignment.h" +#include "common/cityhash.h" #include "common/settings.h" #include "common/slot_vector.h" #include "video_core/control/channel_state.h" @@ -58,59 +58,16 @@ TextureCache

::TextureCache(Runtime& runtime_, Tegra::MaxwellDeviceMemoryManag void(slot_samplers.insert(runtime, sampler_descriptor)); if constexpr (HAS_DEVICE_MEMORY_INFO) { - const s64 device_local_memory = static_cast(runtime.GetDeviceLocalMemory()); - const s64 min_spacing_expected = device_local_memory - 1_GiB; - const s64 min_spacing_critical = device_local_memory - 512_MiB; - const s64 mem_threshold = (std::min)(device_local_memory, TARGET_THRESHOLD); - const s64 min_vacancy_expected = (6 * mem_threshold) / 10; - const s64 min_vacancy_critical = (2 * mem_threshold) / 10; - expected_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_expected, min_spacing_expected), - DEFAULT_EXPECTED_MEMORY)); - critical_memory = static_cast( - (std::max)((std::min)(device_local_memory - min_vacancy_critical, min_spacing_critical), - DEFAULT_CRITICAL_MEMORY)); - minimum_memory = static_cast((device_local_memory - mem_threshold) / 2); - } else { - expected_memory = DEFAULT_EXPECTED_MEMORY + 512_MiB; - critical_memory = DEFAULT_CRITICAL_MEMORY + 1_GiB; - minimum_memory = 0; - } - - const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); - - if (gpu_unswizzle_enabled) { - switch (Settings::values.gpu_unswizzle_texture_size.GetValue()) { - case Settings::GpuUnswizzleSize::VerySmall: gpu_unswizzle_maxsize = 16_MiB; break; - case Settings::GpuUnswizzleSize::Small: gpu_unswizzle_maxsize = 32_MiB; break; - case Settings::GpuUnswizzleSize::Normal: gpu_unswizzle_maxsize = 128_MiB; break; - case Settings::GpuUnswizzleSize::Large: gpu_unswizzle_maxsize = 256_MiB; break; - case Settings::GpuUnswizzleSize::VeryLarge: gpu_unswizzle_maxsize = 512_MiB; break; - default: gpu_unswizzle_maxsize = 128_MiB; break; - } - - switch (Settings::values.gpu_unswizzle_stream_size.GetValue()) { - case Settings::GpuUnswizzle::VeryLow: swizzle_chunk_size = 4_MiB; break; - case Settings::GpuUnswizzle::Low: swizzle_chunk_size = 8_MiB; break; - case Settings::GpuUnswizzle::Normal: swizzle_chunk_size = 16_MiB; break; - case Settings::GpuUnswizzle::Medium: swizzle_chunk_size = 32_MiB; break; - case Settings::GpuUnswizzle::High: swizzle_chunk_size = 64_MiB; break; - default: swizzle_chunk_size = 16_MiB; - } - - switch (Settings::values.gpu_unswizzle_chunk_size.GetValue()) { - case Settings::GpuUnswizzleChunk::VeryLow: swizzle_slices_per_batch = 32; break; - case Settings::GpuUnswizzleChunk::Low: swizzle_slices_per_batch = 64; break; - case Settings::GpuUnswizzleChunk::Normal: swizzle_slices_per_batch = 128; break; - case Settings::GpuUnswizzleChunk::Medium: swizzle_slices_per_batch = 256; break; - case Settings::GpuUnswizzleChunk::High: swizzle_slices_per_batch = 512; break; - default: swizzle_slices_per_batch = 128; - } - } else { - gpu_unswizzle_maxsize = 0; - swizzle_chunk_size = 0; - swizzle_slices_per_batch = 0; + device_local_memory = runtime.GetDeviceLocalMemory(); } + const auto thresholds = VideoCommon::MakeReclaimThresholds( + device_local_memory, static_cast(TARGET_THRESHOLD), + static_cast(DEFAULT_EXPECTED_MEMORY), static_cast(DEFAULT_CRITICAL_MEMORY), + HEAP_PRESSURE_HEADROOM); + minimum_memory = thresholds.minimum; + expected_memory = thresholds.expected; + critical_memory = thresholds.critical; + heap_headroom = thresholds.headroom; } template @@ -120,8 +77,9 @@ void TextureCache

::RunGarbageCollector() { u64 ticks_to_destroy = 0; size_t num_iterations = 0; const auto Configure = [&](bool allow_aggressive) { - high_priority_mode = total_used_memory >= expected_memory; - aggressive_mode = allow_aggressive && total_used_memory >= critical_memory; + high_priority_mode = heap_pressure || total_used_memory >= expected_memory; + aggressive_mode = + allow_aggressive && (heap_pressure || total_used_memory >= critical_memory); ticks_to_destroy = aggressive_mode ? 10ULL : high_priority_mode ? 25ULL : 50ULL; num_iterations = aggressive_mode ? 40 : (high_priority_mode ? 20 : 10); }; @@ -135,7 +93,7 @@ void TextureCache

::RunGarbageCollector() { return false; } const bool must_download = IsDownloadable(image) && False(image.flags & ImageFlagBits::BadOverlap); - if ((!aggressive_mode && True(image.flags & ImageFlagBits::CostlyLoad)) || (!high_priority_mode && must_download)) { + if (!aggressive_mode && (must_download || True(image.flags & ImageFlagBits::CostlyLoad))) { return false; } if (must_download) { @@ -161,7 +119,7 @@ void TextureCache

::RunGarbageCollector() { }; Configure(false); lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); - if (total_used_memory >= critical_memory) { + if (heap_pressure || total_used_memory >= critical_memory) { Configure(true); lru_cache.ForEachItemBelow(frame_tick - ticks_to_destroy, Cleanup); } @@ -169,18 +127,17 @@ void TextureCache

::RunGarbageCollector() { template void TextureCache

::TickFrame() { - // If we can obtain the memory info, use it instead of the estimate. - if (runtime.CanReportMemoryUsage()) { - total_used_memory = runtime.GetDeviceMemoryUsage(); + heap_pressure = false; + if (device_local_memory != 0 && runtime.CanReportMemoryUsage()) { + heap_pressure = runtime.GetDeviceMemoryUsage() + heap_headroom >= device_local_memory; } - if (total_used_memory > minimum_memory) { + if (total_used_memory > minimum_memory || heap_pressure) { RunGarbageCollector(); } sentenced_images.Tick(); sentenced_framebuffers.Tick(); sentenced_image_view.Tick(); TickAsyncDecode(); - TickAsyncUnswizzle(); runtime.TickFrame(); ++frame_tick; @@ -239,65 +196,16 @@ void TextureCache

::FillImageViews(std::span views, bool compu template void TextureCache

::CheckFeedbackLoop(std::span views) { - if (!Settings::values.barrier_feedback_loops.GetValue()) { + if (!rt_depth_image_id || !Settings::values.barrier_feedback_loops.GetValue()) { return; } - - if (render_targets_serial == last_feedback_loop_serial && - texture_bindings_serial == last_feedback_texture_serial) { - if (last_feedback_loop_result) { + const ImageViewId depth_view_id = render_targets.depth_buffer_id; + for (const auto& view : views) { + if (view.id && view.id != depth_view_id && + slot_image_views[view.id].image_id == rt_depth_image_id) { runtime.BarrierFeedbackLoop(); + return; } - return; - } - - if (rt_active_mask == 0) { - last_feedback_loop_serial = render_targets_serial; - last_feedback_texture_serial = texture_bindings_serial; - last_feedback_loop_result = false; - return; - } - const u32 depth_bit = 1u << NUM_RT; - const bool depth_active = (rt_active_mask & depth_bit) != 0; - - const bool requires_barrier = [&] { - for (const auto& view : views) { - if (!view.id) { - continue; - } - - { - bool is_continue = false; - for (size_t i = 0; i < 8; ++i) - is_continue |= (rt_active_mask & (1u << i)) && view.id == render_targets.color_buffer_ids[i]; - if (is_continue) - continue; - } - - if (depth_active && view.id == render_targets.depth_buffer_id) - continue; - - const ImageId view_image_id = slot_image_views[view.id].image_id; - { - bool is_continue = false; - for (size_t i = 0; i < 8; ++i) - is_continue |= (rt_active_mask & (1u << i)) && view_image_id == rt_image_id[i]; - if (is_continue) - continue; - } - if (depth_active && view_image_id == rt_depth_image_id) { - return true; - } - } - - return false; - }(); - - last_feedback_loop_serial = render_targets_serial; - last_feedback_texture_serial = texture_bindings_serial; - last_feedback_loop_result = requires_barrier; - if (requires_barrier) { - runtime.BarrierFeedbackLoop(); } } @@ -313,14 +221,11 @@ SamplerId TextureCache

::GetSamplerId(u32 index, bool compute) { LOG_DEBUG(HW_GPU, "Invalid sampler index={}", index); return NULL_SAMPLER_ID; } - auto const map_index = index | (compute ? Common::SlotId::TAGGED_VALUE : 0); - auto const [descriptor, is_new] = table.Read(*gpu_memory, index); + auto const [entry, is_new] = table.Read(*gpu_memory, index); if (is_new) { - auto const id = FindSampler(descriptor, compute); - channel_state->sampler_ids.insert_or_assign(map_index, id); - return id; + entry.id = FindSampler(entry.descriptor, compute); } - return channel_state->sampler_ids.find(map_index)->second; + return entry.id; } template @@ -339,26 +244,14 @@ void TextureCache

::SynchronizeDescriptors(bool compute) { const bool linked_tsc = kepler_compute->launch_description.linked_tsc; const u32 tic_limit = kepler_compute->regs.tic.limit; const u32 tsc_limit = linked_tsc ? tic_limit : kepler_compute->regs.tsc.limit; - bool bindings_changed = false; - if (channel_state->compute_sampler_table.Synchronize(kepler_compute->regs.tsc.Address(), tsc_limit)) - bindings_changed = true; - if (channel_state->compute_image_table.Synchronize(kepler_compute->regs.tic.Address(), tic_limit)) - bindings_changed = true; - if (bindings_changed) { - ++texture_bindings_serial; - } + channel_state->compute_sampler_table.Synchronize(kepler_compute->regs.tsc.Address(), tsc_limit); + channel_state->compute_image_table.Synchronize(kepler_compute->regs.tic.Address(), tic_limit); } else { const bool linked_tsc = maxwell3d->regs.sampler_binding == Tegra::Engines::Maxwell3D::Regs::SamplerBinding::ViaHeaderBinding; const u32 tic_limit = maxwell3d->regs.tex_header.limit; const u32 tsc_limit = linked_tsc ? tic_limit : maxwell3d->regs.tex_sampler.limit; - bool bindings_changed = false; - if (channel_state->graphics_sampler_table.Synchronize(maxwell3d->regs.tex_sampler.Address(), tsc_limit)) - bindings_changed = true; - if (channel_state->graphics_image_table.Synchronize(maxwell3d->regs.tex_header.Address(), tic_limit)) - bindings_changed = true; - if (bindings_changed) { - ++texture_bindings_serial; - } + channel_state->graphics_sampler_table.Synchronize(maxwell3d->regs.tex_sampler.Address(), tsc_limit); + channel_state->graphics_image_table.Synchronize(maxwell3d->regs.tex_header.Address(), tic_limit); } } @@ -488,19 +381,9 @@ void TextureCache

::UpdateRenderTargets(bool is_clear) { PrepareImageView(depth_buffer_id, true, is_clear && IsFullClear(depth_buffer_id)); - rt_active_mask = 0; - rt_image_id = {}; - for (size_t i = 0; i < rt_image_id.size(); ++i) { - if (ImageViewId const view = render_targets.color_buffer_ids[i]; view) { - rt_active_mask |= 1u << i; - rt_image_id[i] = slot_image_views[view].image_id; - } - } + rt_depth_image_id = {}; if (depth_buffer_id) { - rt_active_mask |= (1u << NUM_RT); rt_depth_image_id = slot_image_views[depth_buffer_id].image_id; - } else { - rt_depth_image_id = ImageId{}; } for (size_t index = 0; index < NUM_RT; ++index) { @@ -543,26 +426,19 @@ ImageViewId TextureCache

::VisitImageView(u32 index, bool compute) { LOG_DEBUG(HW_GPU, "Invalid image view index={}", index); return NULL_IMAGE_VIEW_ID; } - auto const map_index = index | (compute ? Common::SlotId::TAGGED_VALUE : 0); - // Is new (on the tegra engine side)? - auto const [descriptor, is_new] = table.Read(*gpu_memory, index); + auto const [entry, is_new] = table.Read(*gpu_memory, index); if (is_new) { - if (IsValidEntry(*gpu_memory, descriptor)) { - // Is new (registered view) on the texture cache side? - const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(descriptor); + entry.id = NULL_IMAGE_VIEW_ID; + if (IsValidEntry(*gpu_memory, entry.descriptor)) { + const auto [pair, is_new_tc] = channel_state->image_views.try_emplace(entry.descriptor); if (is_new_tc) - pair->second = CreateImageView(descriptor); - PrepareImageView(pair->second, false, false); - channel_state->image_view_ids.insert_or_assign(map_index, pair->second); - return pair->second; + pair->second = CreateImageView(entry.descriptor); + entry.id = pair->second; } - channel_state->image_view_ids.insert_or_assign(map_index, NULL_IMAGE_VIEW_ID); - return NULL_IMAGE_VIEW_ID; } - auto const it = channel_state->image_view_ids.find(map_index); - if (it->second != NULL_IMAGE_VIEW_ID) - PrepareImageView(it->second, false, false); - return it->second; + if (entry.id != NULL_IMAGE_VIEW_ID) + PrepareImageView(entry.id, false, false); + return entry.id; } template @@ -577,7 +453,6 @@ FramebufferId TextureCache

::GetFramebufferId(const RenderTargets& key) { return id ? &slot_image_views[id] : nullptr; }); ImageView* const depth_buffer = key.depth_buffer_id ? &slot_image_views[key.depth_buffer_id] : nullptr; - runtime.FlushDeferredClear(); framebuffer_id = slot_framebuffers.insert(runtime, color_buffers, depth_buffer, key); return framebuffer_id; } @@ -1111,6 +986,29 @@ void TextureCache

::DownloadImageIntoBuffer(typename TextureCache

::Image* i } } +template +bool TextureCache

::IsAstcDataUnchanged(Image& image) { + static constexpr u32 CHECK_THRESHOLD = 4; + if (False(image.flags & ImageFlagBits::Converted) || + True(image.flags & ImageFlagBits::GpuModified) || + !IsPixelFormatASTC(image.info.format)) { + return false; + } + if (image.guest_data_checks < CHECK_THRESHOLD) { + ++image.guest_data_checks; + return false; + } + Tegra::Memory::GpuGuestMemory guest_data( + *gpu_memory, image.gpu_addr, image.guest_size_bytes, &swizzle_data_buffer); + const u64 hash = Common::CityHash64(reinterpret_cast(guest_data.data()), + image.guest_size_bytes); + if (image.guest_data_hash == hash) { + return true; + } + image.guest_data_hash = hash; + return false; +} + template void TextureCache

::RefreshContents(Image& image, ImageId image_id) { if (False(image.flags & ImageFlagBits::CpuModified)) { @@ -1122,29 +1020,50 @@ void TextureCache

::RefreshContents(Image& image, ImageId image_id) { TrackImage(image, image_id); + if (IsAstcDataUnchanged(image)) { + return; + } + if (image.info.num_samples > 1 && !runtime.CanUploadMSAA()) { LOG_WARNING(HW_GPU, "MSAA image uploads are not implemented"); runtime.TransitionImageLayout(image); return; } + gpu_memory->FlushRegion(image.gpu_addr, image.guest_size_bytes, + VideoCommon::CacheType::NoTextureCache); if (True(image.flags & ImageFlagBits::AsynchronousDecode)) { QueueAsyncDecode(image, image_id); return; } - - const bool gpu_unswizzle_enabled = Settings::values.gpu_unswizzle_enabled.GetValue(); - - if (gpu_unswizzle_enabled && - IsPixelFormatBCn(image.info.format) && - image.info.type == ImageType::e3D && - image.info.resources.levels == 1 && - image.info.resources.layers == 1 && - MapSizeBytes(image) >= gpu_unswizzle_maxsize && - False(image.flags & ImageFlagBits::GpuModified)) { - - QueueAsyncUnswizzle(image, image_id); - return; + if constexpr (requires { runtime.GuestMemorySource(nullptr, size_t{}); }) { + if (True(image.flags & ImageFlagBits::AcceleratedUpload) && runtime.HasGuestMemory() && + gpu_memory->IsContinuousRange(image.gpu_addr, image.guest_size_bytes)) { + const std::optional device_addr = gpu_memory->GpuToCpuAddress(image.gpu_addr); + if (device_addr) { + const auto source = runtime.GuestMemorySource( + device_memory.GetSpan(*device_addr, image.guest_size_bytes), + image.guest_size_bytes); + if (source) { + runtime.AccelerateImageUpload( + image, *source, FixSmallVectorADL(FullUploadSwizzles(image.info))); + return; + } + } + } } + if constexpr (requires { image.UploadHostMemory(unswizzle_data_buffer, {}); }) { + if (True(image.flags & ImageFlagBits::ReorderableUpload) && image.CanUploadHostMemory()) { + Tegra::Memory::GpuGuestMemory + swizzle_data(*gpu_memory, image.gpu_addr, image.guest_size_bytes, + &swizzle_data_buffer); + unswizzle_data_buffer.resize_destructive(image.unswizzled_size_bytes); + const auto copies = FixSmallVectorADL(UnswizzleImage( + *gpu_memory, image.gpu_addr, image.info, swizzle_data, unswizzle_data_buffer)); + image.UploadHostMemory(unswizzle_data_buffer, copies); + return; + } + } + auto staging = runtime.UploadStagingBuffer(MapSizeBytes(image)); UploadImageContents(image, staging); runtime.InsertUploadMemoryBarrier(); @@ -1157,16 +1076,21 @@ void TextureCache

::UploadImageContents(Image& image, StagingBuffer& staging) const GPUVAddr gpu_addr = image.gpu_addr; if (True(image.flags & ImageFlagBits::AcceleratedUpload)) { - gpu_memory->ReadBlock(gpu_addr, mapped_span.data(), mapped_span.size_bytes(), - VideoCommon::CacheType::NoTextureCache); + gpu_memory->ReadBlockUnsafe(gpu_addr, mapped_span.data(), image.guest_size_bytes); const auto uploads = FullUploadSwizzles(image.info); - runtime.AccelerateImageUpload(image, staging, FixSmallVectorADL(uploads), 0, 0); + runtime.AccelerateImageUpload(image, staging, FixSmallVectorADL(uploads)); return; } Tegra::Memory::GpuGuestMemory swizzle_data( *gpu_memory, gpu_addr, image.guest_size_bytes, &swizzle_data_buffer); if (True(image.flags & ImageFlagBits::Converted)) { + if (CanConvertFromGuest(image.info)) { + const auto copies = + FixSmallVectorADL(ConvertImageFromGuest(swizzle_data, image.info, mapped_span)); + image.UploadMemory(staging, copies); + return; + } unswizzle_data_buffer.resize_destructive(image.unswizzled_size_bytes); auto copies = FixSmallVectorADL(UnswizzleImage(*gpu_memory, gpu_addr, image.info, swizzle_data, unswizzle_data_buffer)); ConvertImage(unswizzle_data_buffer, image.info, mapped_span, copies); @@ -1319,9 +1243,6 @@ void TextureCache

::InvalidateScale(Image& image) { for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; - if constexpr (ENABLE_VALIDATION) - for (auto& e : channel_info.image_view_ids) - e.second = CORRUPT_ID; channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -1344,18 +1265,33 @@ u64 TextureCache

::GetScaledImageSizeBytes(const ImageBase& image) { template void TextureCache

::QueueAsyncDecode(Image& image, ImageId image_id) { UNIMPLEMENTED_IF(False(image.flags & ImageFlagBits::Converted)); - LOG_INFO(HW_GPU, "Queuing async texture decode"); - image.flags |= ImageFlagBits::IsDecoding; auto decode = std::make_unique(); auto* decode_ptr = decode.get(); decode->image_id = image_id; async_decodes.push_back(std::move(decode)); + const size_t out_size = MapSizeBytes(image); + if (CanConvertFromGuest(image.info)) { + decode_ptr->input_data.resize_destructive(image.guest_size_bytes); + gpu_memory->ReadBlockUnsafe(image.gpu_addr, decode_ptr->input_data.data(), + image.guest_size_bytes); + + texture_decode_worker.QueueWork([out_size, info = image.info, async_decode = decode_ptr] { + async_decode->decoded_data.resize_destructive(out_size); + auto copies = + ConvertImageFromGuest(async_decode->input_data, info, async_decode->decoded_data); + + std::unique_lock lock{async_decode->mutex}; + async_decode->copies = std::move(copies); + async_decode->complete = true; + }); + return; + } + std::vector local_unswizzle_data_buffer(image.unswizzled_size_bytes, 0); Tegra::Memory::GpuGuestMemory swizzle_data(*gpu_memory, image.gpu_addr, image.guest_size_bytes, &swizzle_data_buffer); auto copies = UnswizzleImage(*gpu_memory, image.gpu_addr, image.info, swizzle_data, local_unswizzle_data_buffer); - const size_t out_size = MapSizeBytes(image); auto func = [out_size, copies, info = image.info, input = std::move(local_unswizzle_data_buffer), @@ -1372,20 +1308,6 @@ void TextureCache

::QueueAsyncDecode(Image& image, ImageId image_id) { texture_decode_worker.QueueWork(std::move(func)); } -template -void TextureCache

::QueueAsyncUnswizzle(Image& image, ImageId image_id) { - if (True(image.flags & ImageFlagBits::IsDecoding)) { - return; - } - - image.flags |= ImageFlagBits::IsDecoding; - - unswizzle_queue.push_back({ - .image_id = image_id, - .info = image.info - }); -} - template void TextureCache

::TickAsyncDecode() { bool has_uploads{}; @@ -1411,83 +1333,6 @@ void TextureCache

::TickAsyncDecode() { } } -template -void TextureCache

::TickAsyncUnswizzle() { - if (unswizzle_queue.empty()) { - return; - } - - if(current_unswizzle_frame > 0) { - current_unswizzle_frame--; - return; - } - - PendingUnswizzle& task = unswizzle_queue.front(); - Image& image = slot_images[task.image_id]; - - if (!task.initialized) { - task.total_size = MapSizeBytes(image); - task.staging_buffer = runtime.UploadStagingBuffer(task.total_size, true); - - const auto& info = image.info; - const u32 bytes_per_block = BytesPerBlock(info.format); - const u32 width_blocks = Common::DivCeil(info.size.width, 4u); - const u32 height_blocks = Common::DivCeil(info.size.height, 4u); - - const u32 stride = width_blocks * bytes_per_block; - const u32 aligned_height = height_blocks; - task.bytes_per_slice = static_cast(stride) * aligned_height; - task.last_submitted_offset = 0; - task.initialized = true; - } - - // Read data - if (task.current_offset < task.total_size) { - const size_t remaining = task.total_size - task.current_offset; - - size_t copy_amount = (std::min)(swizzle_chunk_size, remaining); - - if (remaining > swizzle_chunk_size) { - copy_amount = (copy_amount / task.bytes_per_slice) * task.bytes_per_slice; - if (copy_amount == 0) copy_amount = task.bytes_per_slice; - } - - gpu_memory->ReadBlock(image.gpu_addr + task.current_offset, - task.staging_buffer.mapped_span.data() + task.current_offset, - copy_amount); - task.current_offset += copy_amount; - } - - const bool is_final_batch = task.current_offset >= task.total_size; - const size_t bytes_ready = task.current_offset - task.last_submitted_offset; - const u32 complete_slices = static_cast(bytes_ready / task.bytes_per_slice); - - if (complete_slices >= swizzle_slices_per_batch || (is_final_batch && complete_slices > 0)) { - const u32 z_start = static_cast(task.last_submitted_offset / task.bytes_per_slice); - const u32 slices_to_process = (std::min)(complete_slices, swizzle_slices_per_batch); - const u32 z_count = (std::min)(slices_to_process, image.info.size.depth - z_start); - - if (z_count > 0) { - const auto uploads = FullUploadSwizzles(task.info); - runtime.AccelerateImageUpload(image, task.staging_buffer, FixSmallVectorADL(uploads), z_start, z_count); - task.last_submitted_offset += (static_cast(z_count) * task.bytes_per_slice); - } - } - - // Check if complete - const u32 slices_submitted = static_cast(task.last_submitted_offset / task.bytes_per_slice); - const bool all_slices_submitted = slices_submitted >= image.info.size.depth; - - if (is_final_batch && all_slices_submitted) { - runtime.FreeDeferredStagingBuffer(task.staging_buffer); - image.flags &= ~ImageFlagBits::IsDecoding; - unswizzle_queue.pop_front(); - - // Wait 4 frames to process the next entry - current_unswizzle_frame = 4u; - } -} - template bool TextureCache

::ScaleUp(Image& image) { const bool has_copy = image.HasScaled(); @@ -1637,8 +1482,6 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA const ImageId new_image_id = slot_images.insert(runtime, new_info, gpu_addr, cpu_addr); Image& new_image = slot_images[new_image_id]; - new_image.allocation_tick = frame_tick; - if (!gpu_memory->IsContinuousRange(new_image.gpu_addr, new_image.guest_size_bytes) && new_info.is_sparse) { new_image.flags |= ImageFlagBits::Sparse; @@ -1656,8 +1499,9 @@ ImageId TextureCache

::JoinImages(const ImageInfo& info, GPUVAddr gpu_addr, DA DeleteImage(overlap_id); } - // TODO: Only upload what we need + new_image.flags |= ImageFlagBits::ReorderableUpload; RefreshContents(new_image, new_image_id); + new_image.flags &= ~ImageFlagBits::ReorderableUpload; if (can_rescale) { ScaleUp(new_image); @@ -2396,9 +2240,6 @@ void TextureCache

::DeleteImage(ImageId image_id, bool immediate_delete) { } for (size_t c : active_channel_ids) { auto& channel_info = channel_storage[c]; - if constexpr (ENABLE_VALIDATION) - for (auto& e : channel_info.image_view_ids) - e.second = CORRUPT_ID; channel_info.graphics_image_table.Invalidate(); channel_info.compute_image_table.Invalidate(); } @@ -2423,7 +2264,6 @@ void TextureCache

::RemoveImageViewReferences(std::span rem template void TextureCache

::RemoveFramebuffers(std::span removed_views) { - runtime.FlushDeferredClear(); auto it = framebuffers.begin(); while (it != framebuffers.end()) { if (it->first.Contains(removed_views)) { diff --git a/src/video_core/texture_cache/texture_cache_base.h b/src/video_core/texture_cache/texture_cache_base.h index 8a2318cc72..0243a68a2d 100644 --- a/src/video_core/texture_cache/texture_cache_base.h +++ b/src/video_core/texture_cache/texture_cache_base.h @@ -31,6 +31,7 @@ #include "common/thread_worker.h" #include "video_core/compatible_formats.h" #include "video_core/control/channel_state_cache.h" +#include "video_core/cache_reclaim.h" #include "video_core/delayed_destruction_ring.h" #include "video_core/engines/fermi_2d.h" #include "video_core/surface.h" @@ -63,6 +64,7 @@ struct ImageViewInOut { struct AsyncDecodeContext { ImageId image_id; + Common::ScratchBuffer input_data; Common::ScratchBuffer decoded_data; boost::container::small_vector copies; std::mutex mutex; @@ -87,9 +89,6 @@ public: std::unordered_map image_views; std::unordered_map samplers; - ::Common::unordered_map sampler_ids; - ::Common::unordered_map image_view_ids; - TextureCacheGPUMap* gpu_page_table = nullptr; TextureCacheGPUMap* sparse_page_table = nullptr; }; @@ -99,8 +98,6 @@ class TextureCache : public VideoCommon::ChannelSetupCaches void UploadImageContents(Image& image, StagingBuffer& staging_buffer); @@ -422,9 +411,6 @@ private: void QueueAsyncDecode(Image& image, ImageId image_id); void TickAsyncDecode(); - void QueueAsyncUnswizzle(Image& image, ImageId image_id); - void TickAsyncUnswizzle(); - Runtime& runtime; Tegra::MaxwellDeviceMemoryManager& device_memory; @@ -432,13 +418,7 @@ private: RenderTargets render_targets; u64 render_targets_serial = 0; - u32 rt_active_mask = 0; - std::array rt_image_id{}; ImageId rt_depth_image_id{}; - u64 texture_bindings_serial = 0; - u64 last_feedback_loop_serial = 0; - u64 last_feedback_texture_serial = 0; - bool last_feedback_loop_result = false; FramebufferId last_framebuffer_id{}; u64 last_framebuffer_serial = 0; @@ -451,12 +431,12 @@ private: bool has_deleted_images = false; bool is_rescaling = false; u64 total_used_memory = 0; - u64 minimum_memory; - u64 expected_memory; - u64 critical_memory; - size_t gpu_unswizzle_maxsize = 0; - size_t swizzle_chunk_size = 0; - u32 swizzle_slices_per_batch = 0; + u64 device_local_memory = 0; + u64 minimum_memory = 0; + u64 expected_memory = 0; + u64 critical_memory = 0; + u64 heap_headroom = 0; + bool heap_pressure = false; struct BufferDownload { GPUVAddr address; @@ -508,13 +488,9 @@ private: u64 modification_tick = 0; u64 frame_tick = 0; - Common::ThreadWorker texture_decode_worker{1, "TextureDecoder", {}, - Common::ThreadPlacement::Efficiency}; + Common::ThreadWorker texture_decode_worker{1, "TextureDecoder"}; std::vector> async_decodes; - std::deque unswizzle_queue; - u8 current_unswizzle_frame; - // Join caching boost::container::small_vector join_overlap_ids; ::Common::unordered_set join_overlaps_found; diff --git a/src/video_core/texture_cache/util.cpp b/src/video_core/texture_cache/util.cpp index e55d0752ec..d8cae4f3ee 100644 --- a/src/video_core/texture_cache/util.cpp +++ b/src/video_core/texture_cache/util.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project @@ -22,7 +22,6 @@ #include "common/common_types.h" #include "common/div_ceil.h" #include "common/scratch_buffer.h" -#include "common/settings.h" #include "video_core/compatible_formats.h" #include "video_core/engines/maxwell_3d.h" #include "video_core/guest_memory.h" @@ -34,7 +33,6 @@ #include "video_core/texture_cache/samples_helper.h" #include "video_core/texture_cache/util.h" #include "video_core/textures/astc.h" -#include "video_core/textures/bcn.h" #include "video_core/textures/decoders.h" namespace VideoCommon { @@ -608,21 +606,6 @@ u32 CalculateConvertedSizeBytes(const ImageInfo& info) noexcept { return info.size.width * BytesPerBlock(info.format); } static constexpr Extent2D TILE_SIZE{1, 1}; - if (IsPixelFormatASTC(info.format) && Settings::values.astc_recompression.GetValue() != - Settings::AstcRecompression::Uncompressed) { - const u32 bpp_div = - Settings::values.astc_recompression.GetValue() == Settings::AstcRecompression::Bc1 ? 2 - : 1; - // NumBlocksPerLayer doesn't account for this correctly, so we have to do it manually. - u32 output_size = 0; - for (s32 i = 0; i < info.resources.levels; i++) { - const auto mip_size = AdjustMipSize(info.size, i); - const u32 plane_dim = - Common::AlignUp(mip_size.width, 4U) * Common::AlignUp(mip_size.height, 4U); - output_size += (plane_dim * info.size.depth * info.resources.layers) / bpp_div; - } - return output_size; - } return NumBlocksPerLayer(info, TILE_SIZE) * info.resources.layers * ConvertedBytesPerBlock(info.format); } @@ -925,7 +908,6 @@ boost::container::small_vector UnswizzleImage(Tegra::Memory void ConvertImage(std::span input, const ImageInfo& info, std::span output, std::span copies) { u32 output_offset = 0; - Common::ScratchBuffer decode_scratch; const Extent2D tile_size = DefaultBlockSize(info.format); for (BufferImageCopy& copy : copies) { @@ -940,52 +922,19 @@ void ConvertImage(std::span input, const ImageInfo& info, std::span(copy.buffer_size); } else { DecompressBCn(input_offset, output.subspan(output_offset), copy, info.format); output_offset += copy.image_extent.width * copy.image_extent.height * - copy.image_subresource.num_layers * + copy.image_extent.depth * copy.image_subresource.num_layers * ConvertedBytesPerBlock(info.format); } @@ -994,6 +943,74 @@ void ConvertImage(std::span input, const ImageInfo& info, std::span ConvertImageFromGuest( + std::span input, const ImageInfo& info, std::span output) { + const u32 bpp_log2 = BytesPerBlockLog2(info.format); + const Extent2D tile_size = DefaultBlockSize(info.format); + const Extent3D size = info.size; + const LevelInfo level_info = MakeLevelInfo(info); + const s32 num_layers = info.resources.layers; + const s32 num_levels = info.resources.levels; + const std::array level_sizes = CalculateLevelSizes(level_info, num_levels); + const Extent2D gob = GobSize(bpp_log2, info.block.height, info.tile_width_spacing); + const u32 layer_size = CalculateLevelBytes(level_sizes, num_levels); + const u32 layer_stride = AlignLayerSize(layer_size, size, level_info.block, tile_size.height, + info.tile_width_spacing); + const u32 out_bytes_per_texel = BytesPerBlock(PixelFormat::A8B8G8R8_UNORM); + size_t guest_offset = 0; + u32 output_offset = 0; + boost::container::small_vector copies(num_levels); + + for (s32 level = 0; level < num_levels; ++level) { + const Extent3D level_size = AdjustMipSize(size, level); + const Extent3D num_tiles = AdjustTileSize(level_size, tile_size); + const Extent3D block = + AdjustMipBlockSize(num_tiles, level_info.block, level, level_info.num_levels); + const u32 stride_alignment = StrideAlignment(num_tiles, info.block, gob, bpp_log2); + const u32 stride = Common::AlignUpLog2(num_tiles.width, stride_alignment) << bpp_log2; + const u32 gobs_in_x = Common::DivCeilLog2(stride, GOB_SIZE_X_SHIFT); + const u32 gob_block_size = gobs_in_x << (GOB_SIZE_SHIFT + block.height + block.depth); + const u32 level_bytes = level_size.width * level_size.height * level_size.depth * + num_layers * out_bytes_per_texel; + copies[level] = BufferImageCopy{ + .buffer_offset = output_offset, + .buffer_size = level_bytes, + .buffer_row_length = level_size.width, + .buffer_image_height = level_size.height, + .image_subresource = + { + .base_level = level, + .base_layer = 0, + .num_layers = num_layers, + }, + .image_offset = {0, 0, 0}, + .image_extent = level_size, + }; + const Tegra::Texture::ASTC::BlockLinearLayout layout{ + .layer_stride = layer_stride, + .slice_size = + Common::DivCeilLog2(num_tiles.height, block.height + GOB_SIZE_Y_SHIFT) * + gob_block_size, + .block_size = gob_block_size, + .x_shift = GOB_SIZE_SHIFT + block.height + block.depth, + .gob_height = block.height, + .gob_height_mask = (1U << block.height) - 1, + .gob_depth = block.depth, + .gob_depth_mask = (1U << block.depth) - 1, + }; + Tegra::Texture::ASTC::DecompressBlockLinear( + input.subspan(guest_offset), level_size.width, level_size.height, level_size.depth, + num_layers, tile_size.width, tile_size.height, layout, output.subspan(output_offset)); + output_offset += level_bytes; + guest_offset += level_sizes[level]; + } + return copies; +} + boost::container::small_vector FullDownloadCopies(const ImageInfo& info) { const Extent3D size = info.size; const u32 bytes_per_block = BytesPerBlock(info.format); diff --git a/src/video_core/texture_cache/util.h b/src/video_core/texture_cache/util.h index 3e8bb00032..1bc4b53582 100644 --- a/src/video_core/texture_cache/util.h +++ b/src/video_core/texture_cache/util.h @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2020 yuzu Emulator Project @@ -72,6 +72,11 @@ struct OverlapResult { void ConvertImage(std::span input, const ImageInfo& info, std::span output, std::span copies); +[[nodiscard]] bool CanConvertFromGuest(const ImageInfo& info); + +[[nodiscard]] boost::container::small_vector ConvertImageFromGuest( + std::span input, const ImageInfo& info, std::span output); + [[nodiscard]] boost::container::small_vector FullDownloadCopies( const ImageInfo& info); diff --git a/src/video_core/textures/astc.cpp b/src/video_core/textures/astc.cpp index 0b6d06c49a..678e52a242 100644 --- a/src/video_core/textures/astc.cpp +++ b/src/video_core/textures/astc.cpp @@ -20,6 +20,7 @@ #include "common/common_types.h" #include #include "video_core/textures/astc.h" +#include "video_core/textures/decoders.h" #include "video_core/textures/workers.h" class InputBitStream { @@ -1671,7 +1672,7 @@ static void ComputeEndpoints(Pixel& ep1, Pixel& ep2, const u32*& colorValues, } static void FillVoidExtentLDR(InputBitStream& strm, std::span outBuf, u32 blockWidth, - u32 blockHeight) { + u32 blockHeight, u32 stride) { // Don't actually care about the void extent, just read the bits... for (s32 i = 0; i < 4; ++i) { strm.ReadBits<13>(); @@ -1688,7 +1689,7 @@ static void FillVoidExtentLDR(InputBitStream& strm, std::span outBuf, u32 b for (u32 j = 0; j < blockHeight; j++) { for (u32 i = 0; i < blockWidth; i++) { - outBuf[j * blockWidth + i] = rgba; + outBuf[j * stride + i] = rgba; } } } @@ -1726,52 +1727,52 @@ static u16 HalfToClampedByte(u16 half_bits) { return static_cast(clamped * 255.0f + 0.5f); } -static void FillError(std::span outBuf, u32 blockWidth, u32 blockHeight) { +static void FillError(std::span outBuf, u32 blockWidth, u32 blockHeight, u32 stride) { for (u32 j = 0; j < blockHeight; j++) { for (u32 i = 0; i < blockWidth; i++) { - outBuf[j * blockWidth + i] = 0x00000000; + outBuf[j * stride + i] = 0x00000000; } } } static void DecompressBlock(std::span inBuf, const u32 blockWidth, - const u32 blockHeight, std::span outBuf) { + const u32 blockHeight, std::span outBuf, u32 stride) { InputBitStream strm(inBuf); TexelWeightParams weightParams = DecodeBlockInfo(strm); // Was there an error? if (weightParams.m_bError) { assert(false && "Invalid block mode"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } if (weightParams.m_bVoidExtentLDR) { - FillVoidExtentLDR(strm, outBuf, blockWidth, blockHeight); + FillVoidExtentLDR(strm, outBuf, blockWidth, blockHeight, stride); return; } if (weightParams.m_bVoidExtentHDR) { assert(false && "HDR void extent blocks are unsupported!"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } if (weightParams.m_Width > blockWidth) { assert(false && "Texel weight grid width should be smaller than block width"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } if (weightParams.m_Height > blockHeight) { assert(false && "Texel weight grid height should be smaller than block height"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } if (weightParams.GetNumWeightValues() > 64) { assert(false && "Too many weights in the weight grid"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } @@ -1781,7 +1782,7 @@ static void DecompressBlock(std::span inBuf, const u32 blockWidth, if (nPartitions == 4 && weightParams.m_bDualPlane) { assert(false && "Dual plane mode is incompatible with four partition blocks"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } @@ -1813,7 +1814,7 @@ static void DecompressBlock(std::span inBuf, const u32 blockWidth, u32 nWeightBits = weightParams.GetPackedBitSize(); if (nWeightBits < 24 || nWeightBits > 96) { assert(false && "Invalid weight bit count"); - FillError(outBuf, blockWidth, blockHeight); + FillError(outBuf, blockWidth, blockHeight, stride); return; } s32 remainingBits = 128 - nWeightBits - static_cast(strm.GetBitsRead()); @@ -1995,41 +1996,54 @@ static void DecompressBlock(std::span inBuf, const u32 blockWidth, } } - outBuf[j * blockWidth + i] = p.Pack(); + outBuf[j * stride + i] = p.Pack(); } } +static void DecodeAndStoreBlock(std::span data, size_t src_offset, u32 x, u32 y, + u32 width, u32 height, u32 block_width, u32 block_height, + std::span out_slice) { + const u32 decomp_width = (std::min)(block_width, width - x); + const u32 decomp_height = (std::min)(block_height, height - y); + u32* const dst = out_slice.data() + size_t{y} * width + x; + if (src_offset + 16 > data.size()) { + for (u32 h = 0; h < decomp_height; ++h) { + std::memset(dst + size_t{h} * width, 0, decomp_width * 4); + } + return; + } + const std::span block_ptr{data.subspan(src_offset, 16)}; + if (decomp_width == block_width && decomp_height == block_height) { + const size_t touched = size_t{block_height - 1} * width + block_width; + DecompressBlock(block_ptr, block_width, block_height, std::span{dst, touched}, width); + return; + } + std::array staging; + DecompressBlock(block_ptr, block_width, block_height, staging, block_width); + for (u32 h = 0; h < decomp_height; ++h) { + std::memcpy(dst + size_t{h} * width, staging.data() + h * block_width, decomp_width * 4); + } +} + void Decompress(std::span data, uint32_t width, uint32_t height, uint32_t depth, uint32_t block_width, uint32_t block_height, std::span output) { const u32 rows = Common::DivideUp(height, block_height); const u32 cols = Common::DivideUp(width, block_width); Common::ThreadWorker& workers{GetThreadWorkers()}; + const std::span out{reinterpret_cast(output.data()), output.size() / 4}; + const size_t slice_texels = size_t{height} * width; for (u32 z = 0; z < depth; ++z) { - const u32 depth_offset = z * height * width * 4; + const std::span out_slice = out.subspan(z * slice_texels); for (u32 y_index = 0; y_index < rows; ++y_index) { - auto decompress_stride = [data, width, height, block_width, block_height, output, rows, - cols, z, depth_offset, y_index] { + auto decompress_stride = [data, width, height, block_width, block_height, out_slice, + rows, cols, z, y_index] { const u32 y = y_index * block_height; for (u32 x_index = 0; x_index < cols; ++x_index) { const u32 block_index = (z * rows * cols) + (y_index * cols) + x_index; - const u32 x = x_index * block_width; - - const std::span blockPtr{data.subspan(block_index * 16, 16)}; - - // Blocks can be at most 12x12 - std::array uncompData; - DecompressBlock(blockPtr, block_width, block_height, uncompData); - - u32 decompWidth = (std::min)(block_width, width - x); - u32 decompHeight = (std::min)(block_height, height - y); - - const std::span outRow = output.subspan(depth_offset + (y * width + x) * 4); - for (u32 h = 0; h < decompHeight; ++h) { - std::memcpy(outRow.data() + h * width * 4, - uncompData.data() + h * block_width, decompWidth * 4); - } + DecodeAndStoreBlock(data, size_t{block_index} * 16, x_index * block_width, y, + width, height, block_width, block_height, out_slice); } }; workers.QueueWork(std::move(decompress_stride)); @@ -2038,4 +2052,50 @@ void Decompress(std::span data, uint32_t width, uint32_t height, } } +void DecompressBlockLinear(std::span data, uint32_t width, uint32_t height, + uint32_t depth, uint32_t layers, uint32_t block_width, + uint32_t block_height, const BlockLinearLayout& layout, + std::span output) { + const u32 rows = Common::DivideUp(height, block_height); + const u32 cols = Common::DivideUp(width, block_width); + const size_t slice_texels = size_t{height} * width; + + Common::ThreadWorker& workers{GetThreadWorkers()}; + const std::span out{reinterpret_cast(output.data()), output.size() / 4}; + + for (u32 layer = 0; layer < layers; ++layer) { + const size_t layer_offset = size_t{layer} * layout.layer_stride; + for (u32 z = 0; z < depth; ++z) { + const size_t slice_offset = + layer_offset + (z >> layout.gob_depth) * size_t{layout.slice_size} + + ((z & layout.gob_depth_mask) << (GOB_SIZE_SHIFT + layout.gob_height)); + const std::span out_slice = + out.subspan((size_t{layer} * depth + z) * slice_texels); + for (u32 y_index = 0; y_index < rows; ++y_index) { + auto decompress_stride = [data, width, height, block_width, block_height, out_slice, + cols, layout, slice_offset, y_index] { + const u32 y = y_index * block_height; + const u32 gob_y = y_index >> GOB_SIZE_Y_SHIFT; + const size_t offset_y = + (gob_y >> layout.gob_height) * size_t{layout.block_size} + + ((gob_y & layout.gob_height_mask) << GOB_SIZE_SHIFT); + const u32 swizzled_y = ((y_index & 1) << 4) | ((y_index & 6) << 5); + for (u32 x_index = 0; x_index < cols; ++x_index) { + const u32 byte_x = x_index * 16; + const size_t offset_x = size_t{byte_x >> GOB_SIZE_X_SHIFT} + << layout.x_shift; + const u32 swizzled_x = ((x_index & 1) << 5) | ((x_index & 2) << 7); + const size_t src_offset = slice_offset + offset_y + offset_x + + (swizzled_x | swizzled_y); + DecodeAndStoreBlock(data, src_offset, x_index * block_width, y, width, + height, block_width, block_height, out_slice); + } + }; + workers.QueueWork(std::move(decompress_stride)); + } + } + } + workers.WaitForRequests(); +} + } // namespace Tegra::Texture::ASTC diff --git a/src/video_core/textures/astc.h b/src/video_core/textures/astc.h index afd3933c3e..5d59da7496 100644 --- a/src/video_core/textures/astc.h +++ b/src/video_core/textures/astc.h @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project +// SPDX-License-Identifier: GPL-3.0-or-later + // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later @@ -5,7 +8,23 @@ namespace Tegra::Texture::ASTC { +struct BlockLinearLayout { + uint32_t layer_stride; + uint32_t slice_size; + uint32_t block_size; + uint32_t x_shift; + uint32_t gob_height; + uint32_t gob_height_mask; + uint32_t gob_depth; + uint32_t gob_depth_mask; +}; + void Decompress(std::span data, uint32_t width, uint32_t height, uint32_t depth, uint32_t block_width, uint32_t block_height, std::span output); +void DecompressBlockLinear(std::span data, uint32_t width, uint32_t height, + uint32_t depth, uint32_t layers, uint32_t block_width, + uint32_t block_height, const BlockLinearLayout& layout, + std::span output); + } // namespace Tegra::Texture::ASTC diff --git a/src/video_core/textures/bcn.cpp b/src/video_core/textures/bcn.cpp deleted file mode 100644 index 16ddbe3202..0000000000 --- a/src/video_core/textures/bcn.cpp +++ /dev/null @@ -1,86 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#include -#include -#include "common/alignment.h" -#include "video_core/textures/bcn.h" -#include "video_core/textures/workers.h" - -namespace Tegra::Texture::BCN { - -using BCNCompressor = void(u8* block_output, const u8* block_input, bool any_alpha); - -template -void CompressBCN(std::span data, uint32_t width, uint32_t height, uint32_t depth, - std::span output, BCNCompressor f) { - constexpr u8 alpha_threshold = 128; - constexpr u32 bytes_per_px = 4; - const u32 plane_dim = width * height; - - Common::ThreadWorker& workers{GetThreadWorkers()}; - - for (u32 z = 0; z < depth; z++) { - for (u32 y = 0; y < height; y += 4) { - auto compress_row = [z, y, width, height, plane_dim, f, data, output]() { - for (u32 x = 0; x < width; x += 4) { - // Gather 4x4 block of RGBA texels - u8 input_colors[4][4][4]; - bool any_alpha = false; - - for (u32 j = 0; j < 4; j++) { - for (u32 i = 0; i < 4; i++) { - const size_t coord = - (z * plane_dim + (y + j) * width + (x + i)) * bytes_per_px; - - if ((x + i < width) && (y + j < height)) { - if constexpr (ThresholdAlpha) { - if (data[coord + 3] >= alpha_threshold) { - input_colors[j][i][0] = data[coord + 0]; - input_colors[j][i][1] = data[coord + 1]; - input_colors[j][i][2] = data[coord + 2]; - input_colors[j][i][3] = 255; - } else { - any_alpha = true; - memset(input_colors[j][i], 0, bytes_per_px); - } - } else { - memcpy(input_colors[j][i], &data[coord], bytes_per_px); - } - } else { - memset(input_colors[j][i], 0, bytes_per_px); - } - } - } - - const u32 bytes_per_row = BytesPerBlock * Common::DivideUp(width, 4U); - const u32 bytes_per_plane = bytes_per_row * Common::DivideUp(height, 4U); - f(output.data() + z * bytes_per_plane + (y / 4) * bytes_per_row + - (x / 4) * BytesPerBlock, - reinterpret_cast(input_colors), any_alpha); - } - }; - workers.QueueWork(std::move(compress_row)); - } - workers.WaitForRequests(); - } -} - -void CompressBC1(std::span data, uint32_t width, uint32_t height, uint32_t depth, - std::span output) { - CompressBCN<8, true>(data, width, height, depth, output, - [](u8* block_output, const u8* block_input, bool any_alpha) { - stb_compress_bc1_block(block_output, block_input, any_alpha, - STB_DXT_NORMAL); - }); -} - -void CompressBC3(std::span data, uint32_t width, uint32_t height, uint32_t depth, - std::span output) { - CompressBCN<16, false>(data, width, height, depth, output, - [](u8* block_output, const u8* block_input, bool any_alpha) { - stb_compress_bc3_block(block_output, block_input, STB_DXT_NORMAL); - }); -} - -} // namespace Tegra::Texture::BCN diff --git a/src/video_core/textures/bcn.h b/src/video_core/textures/bcn.h deleted file mode 100644 index d5d2a16c91..0000000000 --- a/src/video_core/textures/bcn.h +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project -// SPDX-License-Identifier: GPL-2.0-or-later - -#pragma once - -#include - -#include "common/common_types.h" - -namespace Tegra::Texture::BCN { - -void CompressBC1(std::span data, u32 width, u32 height, u32 depth, std::span output); - -void CompressBC3(std::span data, u32 width, u32 height, u32 depth, std::span output); - -} // namespace Tegra::Texture::BCN diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 25b0860b0f..29d44b77db 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: Copyright 2025 Eden Emulator Project +// SPDX-FileCopyrightText: Copyright 2026 Eden Emulator Project // SPDX-License-Identifier: GPL-3.0-or-later // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project @@ -41,9 +41,9 @@ void SwizzleImpl(std::span output, std::span input, u32 width, u32 u32 block_height, u32 block_depth, u32 stride) { // The origin of the transformation can be configured here, leave it as zero as the current API // doesn't expose it. - static constexpr u32 origin_x = 0; static constexpr u32 origin_y = 0; static constexpr u32 origin_z = 0; + static constexpr u32 columns_per_gob = GOB_SIZE_X / BYTES_PER_PIXEL; // We can configure here a custom pitch // As it's not exposed 'width * BYTES_PER_PIXEL' will be the expected pitch. @@ -62,30 +62,38 @@ void SwizzleImpl(std::span output, std::span input, u32 width, u32 const u32 z = slice + origin_z; const u32 offset_z = (z >> block_depth) * slice_size + ((z & block_depth_mask) << (GOB_SIZE_SHIFT + block_height)); - for (u32 line = 0; line < height; ++line) { - const u32 y = line + origin_y; - const u32 swizzled_y = pdep(y); - - const u32 block_y = y >> GOB_SIZE_Y_SHIFT; + const u32 slice_base = slice * pitch * height; + for (u32 band = 0; band < height; band += GOB_SIZE_Y) { + const u32 band_end = (std::min)(band + GOB_SIZE_Y, height); + const u32 block_y = (band + origin_y) >> GOB_SIZE_Y_SHIFT; const u32 offset_y = (block_y >> block_height) * block_size + ((block_y & block_height_mask) << GOB_SIZE_SHIFT); - u32 swizzled_x = pdep(origin_x * BYTES_PER_PIXEL); - for (u32 column = 0; column < width; - ++column, incrpdep(swizzled_x)) { - const u32 x = (column + origin_x) * BYTES_PER_PIXEL; - const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; + const u32 band_base = offset_z + offset_y; + u32 band_linear = slice_base + band * pitch; + u32 offset_x = 0; + for (u32 gob = 0; gob < width; gob += columns_per_gob, + offset_x += 1U << x_shift, band_linear += GOB_SIZE_X) { + const u32 columns = (std::min)(columns_per_gob, width - gob); + const u32 gob_base = band_base + offset_x; + u32 linear_row = band_linear; + for (u32 line = band; line < band_end; ++line, linear_row += pitch) { + const u32 y = line + origin_y; + const u32 row_base = gob_base + ((y & 1) << 4) + ((y & 6) << 5); - const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + u32 linear_offset = linear_row; + u32 swizzled_x = 0; + for (u32 column = 0; column < columns; + ++column, incrpdep(swizzled_x), + linear_offset += BYTES_PER_PIXEL) { + const u32 swizzled_offset = row_base + swizzled_x; - const u32 unswizzled_offset = - slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; + u8* const dst = &output[TO_LINEAR ? swizzled_offset : linear_offset]; + const u8* const src = &input[TO_LINEAR ? linear_offset : swizzled_offset]; - u8* const dst = &output[TO_LINEAR ? swizzled_offset : unswizzled_offset]; - const u8* const src = &input[TO_LINEAR ? unswizzled_offset : swizzled_offset]; - - std::memcpy(dst, src, BYTES_PER_PIXEL); + std::memcpy(dst, src, BYTES_PER_PIXEL); + } + } } } } @@ -123,26 +131,25 @@ void SwizzleSubrectImpl(std::span output, std::span input, u32 wid const u32 lines_in_y = (std::min)(unprocessed_lines, extent_y); for (u32 line = 0; line < lines_in_y; ++line) { const u32 y = line + origin_y; - const u32 swizzled_y = pdep(y); + const u32 swizzled_y = ((y & 1) << 4) | ((y & 6) << 5); const u32 block_y = y >> GOB_SIZE_Y_SHIFT; const u32 offset_y = (block_y >> block_height) * block_size + ((block_y & block_height_mask) << GOB_SIZE_SHIFT); + const u32 row_base = offset_z + offset_y + swizzled_y; + u32 linear_offset = slice * pitch * height + line * pitch; u32 swizzled_x = pdep(origin_x * BYTES_PER_PIXEL); for (u32 column = 0; column < extent_x; - ++column, incrpdep(swizzled_x)) { + ++column, incrpdep(swizzled_x), + linear_offset += BYTES_PER_PIXEL) { const u32 x = (column + origin_x) * BYTES_PER_PIXEL; const u32 offset_x = (x >> GOB_SIZE_X_SHIFT) << x_shift; - const u32 base_swizzled_offset = offset_z + offset_y + offset_x; - const u32 swizzled_offset = base_swizzled_offset + (swizzled_x | swizzled_y); + const u32 swizzled_offset = row_base + offset_x + swizzled_x; - const u32 unswizzled_offset = - slice * pitch * height + line * pitch + column * BYTES_PER_PIXEL; - - u8* const dst = &output[TO_LINEAR ? swizzled_offset : unswizzled_offset]; - const u8* const src = &input[TO_LINEAR ? unswizzled_offset : swizzled_offset]; + u8* const dst = &output[TO_LINEAR ? swizzled_offset : linear_offset]; + const u8* const src = &input[TO_LINEAR ? linear_offset : swizzled_offset]; std::memcpy(dst, src, BYTES_PER_PIXEL); } @@ -164,11 +171,8 @@ void Swizzle(std::span output, std::span input, u32 bytes_per_pixe block_depth, stride_alignment); BPP_CASE(1) BPP_CASE(2) - BPP_CASE(3) BPP_CASE(4) - BPP_CASE(6) BPP_CASE(8) - BPP_CASE(12) BPP_CASE(16) #undef BPP_CASE default: diff --git a/src/video_core/textures/decoders.h b/src/video_core/textures/decoders.h index 1e14a38753..8018f444be 100644 --- a/src/video_core/textures/decoders.h +++ b/src/video_core/textures/decoders.h @@ -24,7 +24,6 @@ constexpr u32 GOB_SIZE_Z_SHIFT = 0; constexpr u32 GOB_SIZE_SHIFT = GOB_SIZE_X_SHIFT + GOB_SIZE_Y_SHIFT + GOB_SIZE_Z_SHIFT; constexpr u32 SWIZZLE_X_BITS = 0b100101111; -constexpr u32 SWIZZLE_Y_BITS = 0b011010000; /// Unswizzles a block linear texture into linear memory. void UnswizzleTexture(std::span output, std::span input, u32 bytes_per_pixel, diff --git a/src/video_core/vulkan_common/vulkan_device.cpp b/src/video_core/vulkan_common/vulkan_device.cpp index be09bca8d3..cd5e5a8412 100644 --- a/src/video_core/vulkan_common/vulkan_device.cpp +++ b/src/video_core/vulkan_common/vulkan_device.cpp @@ -7,8 +7,6 @@ #include #include #include -#include -#include #include #include #include "common/container/unordered_map.h" @@ -19,8 +17,6 @@ #include #include "common/assert.h" -#include "common/fs/fs.h" -#include "common/fs/path_util.h" #include "common/literals.h" #include #include "common/settings.h" @@ -305,10 +301,6 @@ VkFormatFeatureFlags GetFormatFeatures(VkFormatProperties properties, FormatType VK_FORMAT_ETC2_R8G8B8_SRGB_BLOCK, VK_FORMAT_ETC2_R8G8B8A8_SRGB_BLOCK, VK_FORMAT_ETC2_R8G8B8A1_SRGB_BLOCK, - VK_FORMAT_EAC_R11_UNORM_BLOCK, - VK_FORMAT_EAC_R11_SNORM_BLOCK, - VK_FORMAT_EAC_R11G11_UNORM_BLOCK, - VK_FORMAT_EAC_R11G11_SNORM_BLOCK, }; ::Common::unordered_map format_properties; for (const auto format : formats) { @@ -398,17 +390,6 @@ std::vector ExtensionListForVulkan( return output; } -constexpr std::array STATIC_CACHE_MAGIC_NUMBER{'e', 'd', 'e', 'n', 's', 't', 'p', 'c'}; -constexpr u32 STATIC_CACHE_VERSION = 1; - -std::filesystem::path StaticPipelineCacheFilename() { - const auto shader_dir = Common::FS::GetEdenPath(Common::FS::EdenPath::ShaderDir); - if (!Common::FS::CreateDir(shader_dir)) { - return {}; - } - return shader_dir / "vulkan_static_pipelines.bin"; -} - } // Anonymous namespace void Device::RemoveExtension(bool& extension, const std::string& extension_name) { @@ -452,6 +433,13 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR const vk::InstanceDispatch& dld_) : instance{instance_}, dld{dld_}, physical{physical_}, format_properties(GetFormatProperties(physical)) { + const VkPhysicalDeviceMemoryProperties memory_properties = + physical.GetMemoryProperties().memoryProperties; + is_uma = std::all_of(memory_properties.memoryTypes, + memory_properties.memoryTypes + memory_properties.memoryTypeCount, + [](const VkMemoryType& type) { + return (type.propertyFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0; + }); // Get suitability and device properties. const bool is_suitable = GetSuitability(surface != VkSurfaceKHR{}); @@ -506,9 +494,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR properties.subgroup_size_control.maxSubgroupSize > GuestWarpSize; is_integrated = properties.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; - is_virtual = properties.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU; - is_non_gpu = properties.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_OTHER || - properties.properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_CPU; supports_d24_depth = IsFormatSupported(VK_FORMAT_D24_UNORM_S8_UINT, @@ -586,7 +571,7 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR const auto arch = GetNvidiaArch(); if (arch >= NvidiaArchitecture::Arch_AmpereOrNewer) { LOG_WARNING(Render_Vulkan, "Ampere and newer have broken float16 math"); - features.shader_float16_int8.shaderFloat16 = false; + has_broken_float16_math = true; } // Use hardware depth/stencil blits instead when available @@ -618,13 +603,6 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR "AMD GCN4 and earlier have broken VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT"); has_broken_cube_compatibility = true; } - - // AMD drivers (2026+) have broken float16 math on DKCR - if (features.shader_float16_int8.shaderFloat16) { - LOG_WARNING(Render_Vulkan, - "AMD drivers (2026+) have broken float16 math"); - features.shader_float16_int8.shaderFloat16 = false; - } } if (extensions.sampler_filter_minmax && is_amd) { @@ -760,106 +738,21 @@ Device::Device(VkInstance instance_, vk::PhysicalDevice physical_, VkSurfaceKHR .pHeapSizeLimit = nullptr, .pVulkanFunctions = &functions, .instance = instance, - .vulkanApiVersion = ApiVersion(), + .vulkanApiVersion = VK_API_VERSION_1_1, .pTypeExternalMemoryHandleTypes = nullptr, }; vk::Check(vmaCreateAllocator(&allocator_info, &allocator)); - owns_static_pipeline_cache = surface != VkSurfaceKHR{}; - LoadStaticPipelineCache(); - // Initialize GPU logging if enabled InitializeGPULogging(); } Device::~Device() { - SaveStaticPipelineCache(); ShutdownGPULogging(); vmaDestroyAllocator(allocator); } -void Device::LoadStaticPipelineCache() { - const auto create = [this](size_t size, const void* data) { - static_pipeline_cache = logical.CreatePipelineCache({ - .sType = VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - .initialDataSize = size, - .pInitialData = data, - }); - }; - if (!owns_static_pipeline_cache) { - create(0, nullptr); - return; - } - const auto filename = StaticPipelineCacheFilename(); - if (filename.empty()) { - create(0, nullptr); - return; - } - std::vector data; - try { - std::ifstream file(filename, std::ios::binary | std::ios::ate); - if (!file.is_open()) { - create(0, nullptr); - return; - } - file.exceptions(std::ifstream::failbit | std::ifstream::badbit); - const size_t total = static_cast(file.tellg()); - file.seekg(0, std::ios::beg); - std::array magic{}; - u32 version{}; - if (total < magic.size() + sizeof(version)) { - create(0, nullptr); - return; - } - file.read(magic.data(), magic.size()) - .read(reinterpret_cast(&version), sizeof(version)); - if (magic != STATIC_CACHE_MAGIC_NUMBER || version != STATIC_CACHE_VERSION) { - create(0, nullptr); - return; - } - data.resize(total - magic.size() - sizeof(version)); - file.read(data.data(), static_cast(data.size())); - } catch (const std::ios_base::failure& e) { - create(0, nullptr); - return; - } - create(data.size(), data.empty() ? nullptr : data.data()); -} - -void Device::SaveStaticPipelineCache() const { - if (!owns_static_pipeline_cache || !static_pipeline_cache) { - return; - } - const auto filename = StaticPipelineCacheFilename(); - if (filename.empty()) { - return; - } - size_t size = 0; - std::vector data; - static_pipeline_cache.Read(&size, nullptr); - if (size == 0) { - return; - } - data.resize(size); - static_pipeline_cache.Read(&size, data.data()); - try { - std::ofstream file(filename, std::ios::binary | std::ios::trunc); - file.exceptions(std::ofstream::failbit); - if (!file.is_open()) { - return; - } - file.write(STATIC_CACHE_MAGIC_NUMBER.data(), STATIC_CACHE_MAGIC_NUMBER.size()) - .write(reinterpret_cast(&STATIC_CACHE_VERSION), - sizeof(STATIC_CACHE_VERSION)) - .write(data.data(), static_cast(size)); - } catch (const std::ios_base::failure& e) { - Common::FS::RemoveFile(filename); - } -} - VkFormat Device::GetSupportedFormat(VkFormat wanted_format, VkFormatFeatureFlags wanted_usage, FormatType format_type) const { if (IsFormatSupported(wanted_format, wanted_usage, format_type)) { @@ -1000,10 +893,6 @@ bool Device::GetSuitability(bool requires_swapchain) { // Assume we will be suitable. bool suitable = true; - // Configure properties. - VkPhysicalDeviceVulkan12Features features_1_2{}; - VkPhysicalDeviceVulkan13Features features_1_3{}; - // Configure properties. properties.properties = physical.GetProperties(); @@ -1049,6 +938,8 @@ bool Device::GetSuitability(bool requires_swapchain) { extensions.depth_stencil_resolve = extensions.depth_stencil_resolve && (instance_version >= VK_API_VERSION_1_2 || extensions.create_renderpass2); + extensions.draw_indirect_count = + extensions.draw_indirect_count || instance_version >= VK_API_VERSION_1_2; RemoveExtensionIfUnsuitable(extensions.depth_stencil_resolve, VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME); @@ -1098,16 +989,6 @@ bool Device::GetSuitability(bool requires_swapchain) { // Set next pointer. void** next = &features2.pNext; - // Vulkan 1.2 and 1.3 features - if (instance_version >= VK_API_VERSION_1_2) { - features_1_2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES; - features_1_3.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES; - - features_1_2.pNext = &features_1_3; - - *next = &features_1_2; - } - // Test all features we know about. If the feature is not available in core at our // current API version, and was not enabled by an extension, skip testing the feature. // We set the structure sType explicitly here as it is zeroed by the constructor. @@ -1214,6 +1095,16 @@ bool Device::GetSuitability(bool requires_swapchain) { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_SIZE_CONTROL_PROPERTIES; SetNext(next, properties.subgroup_size_control); } + properties.texel_buffer_alignment.storageTexelBufferOffsetAlignmentBytes = + properties.properties.limits.minTexelBufferOffsetAlignment; + properties.texel_buffer_alignment.uniformTexelBufferOffsetAlignmentBytes = + properties.properties.limits.minTexelBufferOffsetAlignment; + if (features.texel_buffer_alignment.texelBufferAlignment == VK_TRUE || + instance_version >= VK_API_VERSION_1_3) { + properties.texel_buffer_alignment.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TEXEL_BUFFER_ALIGNMENT_PROPERTIES_EXT; + SetNext(next, properties.texel_buffer_alignment); + } if (extensions.transform_feedback) { properties.transform_feedback.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_PROPERTIES_EXT; @@ -1230,11 +1121,22 @@ bool Device::GetSuitability(bool requires_swapchain) { SetNext(next, properties.custom_border_color); } + if (extensions.vertex_attribute_divisor) { + properties.vertex_attribute_divisor.sType = + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VERTEX_ATTRIBUTE_DIVISOR_PROPERTIES_EXT; + SetNext(next, properties.vertex_attribute_divisor); + } + // Perform the property fetch. physical.GetProperties2(properties2); // Store base properties properties.properties = properties2.properties; + max_vertex_attrib_divisor = + (std::max)(1U, properties.vertex_attribute_divisor.maxVertexAttribDivisor); + if (!features.vertex_attribute_divisor.vertexAttributeInstanceRateDivisor) { + max_vertex_attrib_divisor = 1; + } // Unload extensions if feature support is insufficient. RemoveUnsuitableExtensions(); @@ -1313,6 +1215,11 @@ void Device::RemoveUnsuitableExtensions() { RemoveExtensionFeatureIfUnsuitable(extensions.color_write_enable, features.color_write_enable, VK_EXT_COLOR_WRITE_ENABLE_EXTENSION_NAME); + extensions.conditional_rendering = features.conditional_rendering.conditionalRendering; + RemoveExtensionFeatureIfUnsuitable(extensions.conditional_rendering, + features.conditional_rendering, + VK_EXT_CONDITIONAL_RENDERING_EXTENSION_NAME); + // VK_EXT_custom_border_color if (extensions.custom_border_color) { extensions.custom_border_color = @@ -1348,6 +1255,12 @@ void Device::RemoveUnsuitableExtensions() { RemoveExtensionFeatureIfUnsuitable(extensions.descriptor_buffer, features.descriptor_buffer, VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME); + extensions.dynamic_rendering_unused_attachments = + features.dynamic_rendering_unused_attachments.dynamicRenderingUnusedAttachments; + RemoveExtensionFeatureIfUnsuitable(extensions.dynamic_rendering_unused_attachments, + features.dynamic_rendering_unused_attachments, + VK_EXT_DYNAMIC_RENDERING_UNUSED_ATTACHMENTS_EXTENSION_NAME); + // VK_EXT_extended_dynamic_state extensions.extended_dynamic_state = features.extended_dynamic_state.extendedDynamicState; RemoveExtensionFeatureIfUnsuitable(extensions.extended_dynamic_state, @@ -1500,18 +1413,6 @@ void Device::RemoveUnsuitableExtensions() { features.workgroup_memory_explicit_layout, VK_KHR_WORKGROUP_MEMORY_EXPLICIT_LAYOUT_EXTENSION_NAME); - // VK_KHR_maintenance1 - extensions.maintenance1 = loaded_extensions.contains(VK_KHR_MAINTENANCE_1_EXTENSION_NAME); - RemoveExtensionIfUnsuitable(extensions.maintenance1, VK_KHR_MAINTENANCE_1_EXTENSION_NAME); - - // VK_KHR_maintenance2 - extensions.maintenance2 = loaded_extensions.contains(VK_KHR_MAINTENANCE_2_EXTENSION_NAME); - RemoveExtensionIfUnsuitable(extensions.maintenance2, VK_KHR_MAINTENANCE_2_EXTENSION_NAME); - - // VK_KHR_maintenance3 - extensions.maintenance3 = loaded_extensions.contains(VK_KHR_MAINTENANCE_3_EXTENSION_NAME); - RemoveExtensionIfUnsuitable(extensions.maintenance3, VK_KHR_MAINTENANCE_3_EXTENSION_NAME); - // VK_KHR_maintenance4 extensions.maintenance4 = features.maintenance4.maintenance4; RemoveExtensionFeatureIfUnsuitable(extensions.maintenance4, features.maintenance4, @@ -1522,23 +1423,20 @@ void Device::RemoveUnsuitableExtensions() { RemoveExtensionFeatureIfUnsuitable(extensions.maintenance5, features.maintenance5, VK_KHR_MAINTENANCE_5_EXTENSION_NAME); - // VK_KHR_maintenance6 - extensions.maintenance6 = features.maintenance6.maintenance6; - RemoveExtensionFeatureIfUnsuitable(extensions.maintenance6, features.maintenance6, - VK_KHR_MAINTENANCE_6_EXTENSION_NAME); + if (!is_uma || instance_version < VK_API_VERSION_1_3 || + !features.host_image_copy.hostImageCopy) { + RemoveExtensionFeature(extensions.host_image_copy, features.host_image_copy, + VK_EXT_HOST_IMAGE_COPY_EXTENSION_NAME); + } - // VK_KHR_maintenance7 - extensions.maintenance7 = loaded_extensions.contains(VK_KHR_MAINTENANCE_7_EXTENSION_NAME); - RemoveExtensionIfUnsuitable(extensions.maintenance7, VK_KHR_MAINTENANCE_7_EXTENSION_NAME); - // VK_KHR_maintenance8 - extensions.maintenance8 = loaded_extensions.contains(VK_KHR_MAINTENANCE_8_EXTENSION_NAME); - RemoveExtensionIfUnsuitable(extensions.maintenance8, VK_KHR_MAINTENANCE_8_EXTENSION_NAME); - - // VK_KHR_synchronization2 extensions.synchronization2 = features.synchronization2.synchronization2; - RemoveExtensionFeatureIfUnsuitable(extensions.synchronization2, features.synchronization2, - VK_KHR_SYNCHRONIZATION_2_EXTENSION_NAME); + extensions.dynamic_rendering = features.dynamic_rendering.dynamicRendering; + const bool copy_commands2 = + extensions.copy_commands2 || instance_version >= VK_API_VERSION_1_3; + if (!extensions.synchronization2 || !extensions.dynamic_rendering || !copy_commands2) { + throw vk::Exception(VK_ERROR_FEATURE_NOT_PRESENT); + } } void Device::SetupFamilies(VkSurfaceKHR surface) { diff --git a/src/video_core/vulkan_common/vulkan_device.h b/src/video_core/vulkan_common/vulkan_device.h index 077b1c3e23..63a2e5b7fa 100644 --- a/src/video_core/vulkan_common/vulkan_device.h +++ b/src/video_core/vulkan_common/vulkan_device.h @@ -6,6 +6,7 @@ #pragma once +#include #include #include #include @@ -42,6 +43,7 @@ VK_DEFINE_HANDLE(VmaAllocator) FEATURE(KHR, VulkanMemoryModel, VULKAN_MEMORY_MODEL, vulkan_memory_model) #define FOR_EACH_VK_FEATURE_1_3(FEATURE) \ + FEATURE(KHR, DynamicRendering, DYNAMIC_RENDERING, dynamic_rendering) \ FEATURE(EXT, ImageRobustness, IMAGE_ROBUSTNESS, robust_image_access) \ FEATURE(EXT, ShaderDemoteToHelperInvocation, SHADER_DEMOTE_TO_HELPER_INVOCATION, \ shader_demote_to_helper_invocation) \ @@ -55,24 +57,29 @@ VK_DEFINE_HANDLE(VmaAllocator) #define FOR_EACH_VK_FEATURE_EXT(FEATURE) \ FEATURE(EXT, BorderColorSwizzle, BORDER_COLOR_SWIZZLE, border_color_swizzle) \ FEATURE(EXT, ColorWriteEnable, COLOR_WRITE_ENABLE, color_write_enable) \ + FEATURE(EXT, ConditionalRendering, CONDITIONAL_RENDERING, conditional_rendering) \ FEATURE(EXT, CustomBorderColor, CUSTOM_BORDER_COLOR, custom_border_color) \ FEATURE(EXT, DepthBiasControl, DEPTH_BIAS_CONTROL, depth_bias_control) \ FEATURE(EXT, DepthClipControl, DEPTH_CLIP_CONTROL, depth_clip_control) \ FEATURE(EXT, DescriptorBuffer, DESCRIPTOR_BUFFER, descriptor_buffer) \ + FEATURE(EXT, DynamicRenderingUnusedAttachments, DYNAMIC_RENDERING_UNUSED_ATTACHMENTS, \ + dynamic_rendering_unused_attachments) \ FEATURE(EXT, ExtendedDynamicState, EXTENDED_DYNAMIC_STATE, extended_dynamic_state) \ FEATURE(EXT, ExtendedDynamicState2, EXTENDED_DYNAMIC_STATE_2, extended_dynamic_state2) \ FEATURE(EXT, ExtendedDynamicState3, EXTENDED_DYNAMIC_STATE_3, extended_dynamic_state3) \ FEATURE(EXT, 4444Formats, 4444_FORMATS, format_a4b4g4r4) \ + FEATURE(EXT, HostImageCopy, HOST_IMAGE_COPY, host_image_copy) \ FEATURE(EXT, IndexTypeUint8, INDEX_TYPE_UINT8, index_type_uint8) \ FEATURE(EXT, LineRasterization, LINE_RASTERIZATION, line_rasterization) \ FEATURE(EXT, PrimitiveTopologyListRestart, PRIMITIVE_TOPOLOGY_LIST_RESTART, \ primitive_topology_list_restart) \ FEATURE(EXT, ProvokingVertex, PROVOKING_VERTEX, provoking_vertex) \ FEATURE(EXT, Robustness2, ROBUSTNESS_2, robustness2) \ + FEATURE(EXT, TexelBufferAlignment, TEXEL_BUFFER_ALIGNMENT, texel_buffer_alignment) \ FEATURE(EXT, TransformFeedback, TRANSFORM_FEEDBACK, transform_feedback) \ + FEATURE(EXT, VertexAttributeDivisor, VERTEX_ATTRIBUTE_DIVISOR, vertex_attribute_divisor) \ FEATURE(EXT, VertexInputDynamicState, VERTEX_INPUT_DYNAMIC_STATE, vertex_input_dynamic_state) \ FEATURE(KHR, Maintenance5, MAINTENANCE_5, maintenance5) \ - FEATURE(KHR, Maintenance6, MAINTENANCE_6, maintenance6) \ FEATURE(KHR, PipelineExecutableProperties, PIPELINE_EXECUTABLE_PROPERTIES, \ pipeline_executable_properties) \ FEATURE(KHR, ShaderQuadControl, SHADER_QUAD_CONTROL, shader_quad_control) \ @@ -83,7 +90,6 @@ VK_DEFINE_HANDLE(VmaAllocator) // Define miscellaneous extensions which may be used by the implementation here. #define FOR_EACH_VK_EXTENSION(EXTENSION) \ EXTENSION(EXT, ASTC_DECODE_MODE, astc_decode_mode) \ - EXTENSION(EXT, CONDITIONAL_RENDERING, conditional_rendering) \ EXTENSION(EXT, CONSERVATIVE_RASTERIZATION, conservative_rasterization) \ EXTENSION(EXT, DEPTH_RANGE_UNRESTRICTED, depth_range_unrestricted) \ EXTENSION(EXT, MEMORY_BUDGET, memory_budget) \ @@ -92,7 +98,7 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION(EXT, SHADER_STENCIL_EXPORT, shader_stencil_export) \ EXTENSION(EXT, SHADER_VIEWPORT_INDEX_LAYER, shader_viewport_index_layer) \ EXTENSION(EXT, TOOLING_INFO, tooling_info) \ - EXTENSION(EXT, VERTEX_ATTRIBUTE_DIVISOR, vertex_attribute_divisor) \ + EXTENSION(KHR, COPY_COMMANDS_2, copy_commands2) \ EXTENSION(KHR, CREATE_RENDERPASS_2, create_renderpass2) \ EXTENSION(KHR, DEPTH_STENCIL_RESOLVE, depth_stencil_resolve) \ EXTENSION(KHR, DRAW_INDIRECT_COUNT, draw_indirect_count) \ @@ -104,17 +110,13 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION(KHR, SWAPCHAIN, swapchain) \ EXTENSION(KHR, SWAPCHAIN_MUTABLE_FORMAT, swapchain_mutable_format) \ EXTENSION(KHR, IMAGE_FORMAT_LIST, image_format_list) \ - EXTENSION(KHR, MAINTENANCE_1, maintenance1) \ - EXTENSION(KHR, MAINTENANCE_2, maintenance2) \ - EXTENSION(KHR, MAINTENANCE_3, maintenance3) \ - EXTENSION(KHR, MAINTENANCE_7, maintenance7) \ - EXTENSION(KHR, MAINTENANCE_8, maintenance8) \ + EXTENSION(KHR, LOAD_STORE_OP_NONE, load_store_op_none) \ + EXTENSION(EXT, LOAD_STORE_OP_NONE, load_store_op_none_ext) \ EXTENSION(NV, DEVICE_DIAGNOSTICS_CONFIG, device_diagnostics_config) \ EXTENSION(NV, GEOMETRY_SHADER_PASSTHROUGH, geometry_shader_passthrough) \ EXTENSION(NV, VIEWPORT_ARRAY2, viewport_array2) \ EXTENSION(NV, VIEWPORT_SWIZZLE, viewport_swizzle) \ EXTENSION(EXT, FILTER_CUBIC, filter_cubic) \ - EXTENSION(IMG, FILTER_CUBIC, filter_cubic_img) \ EXTENSION(QCOM, FILTER_CUBIC_WEIGHTS, filter_cubic_weights) // Define extensions which must be supported. @@ -133,7 +135,6 @@ VK_DEFINE_HANDLE(VmaAllocator) EXTENSION_NAME(VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_EXTENDED_DYNAMIC_STATE_2_EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_EXTENDED_DYNAMIC_STATE_3_EXTENSION_NAME) \ - EXTENSION_NAME(VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_4444_FORMATS_EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_LINE_RASTERIZATION_EXTENSION_NAME) \ EXTENSION_NAME(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME) \ @@ -273,10 +274,6 @@ public: return physical; } - VkPipelineCache StaticPipelineCache() const noexcept { - return *static_pipeline_cache; - } - /// Returns the main graphics queue. vk::Queue GetGraphicsQueue() const { return graphics_queue; @@ -302,6 +299,23 @@ public: return properties.properties.apiVersion; } + VkDeviceSize TexelBufferAlignment(u32 texel_size) const { + VkDeviceSize texel = texel_size; + if (texel % 3 == 0) { + texel /= 3; + } + const auto& limits = properties.texel_buffer_alignment; + VkDeviceSize storage = limits.storageTexelBufferOffsetAlignmentBytes; + if (limits.storageTexelBufferOffsetSingleTexelAlignment != VK_FALSE) { + storage = (std::min)(storage, texel); + } + VkDeviceSize uniform = limits.uniformTexelBufferOffsetAlignmentBytes; + if (limits.uniformTexelBufferOffsetSingleTexelAlignment != VK_FALSE) { + uniform = (std::min)(uniform, texel); + } + return (std::max)((std::max)(storage, uniform), VkDeviceSize{1}); + } + /// Returns the current driver version provided in Vulkan-formatted version numbers. u32 GetDriverVersion() const { return properties.properties.driverVersion; @@ -356,11 +370,6 @@ public: return {count[0], count[1], count[2]}; } - /// Returns the maximum size for push constants. - VkDeviceSize GetMaxPushConstantsSize() const { - return properties.properties.limits.maxPushConstantsSize; - } - #define FN_MAX_LIMIT_LIST \ FN_MAX_LIMIT_ELEM(ComputeSharedMemorySize) \ FN_MAX_LIMIT_ELEM(PerStageDescriptorSampledImages) \ @@ -426,6 +435,10 @@ FN_MAX_LIMIT_LIST /// Returns true if the device supports float16 natively. bool IsFloat16Supported() const { + return features.shader_float16_int8.shaderFloat16 && !has_broken_float16_math; + } + + bool IsNativeFloat16Enabled() const { return features.shader_float16_int8.shaderFloat16; } @@ -483,6 +496,11 @@ FN_MAX_LIMIT_LIST return properties.subgroup_properties.supportedStages; } + u32 GetMaxSubgroupSize() const { + return (std::max)(properties.subgroup_properties.subgroupSize, + properties.subgroup_size_control.maxSubgroupSize); + } + /// Returns the maximum number of push descriptors. u32 MaxPushDescriptors() const { return properties.push_descriptor.maxPushDescriptors; @@ -553,11 +571,6 @@ FN_MAX_LIMIT_LIST return extensions.geometry_shader_passthrough; } - /// Returns true if the device supports VK_KHR_uniform_buffer_standard_layout. - bool IsKhrUniformBufferStandardLayoutSupported() const { - return extensions.uniform_buffer_standard_layout; - } - /// Returns true if the device supports VK_KHR_push_descriptor. bool IsKhrPushDescriptorSupported() const { return extensions.push_descriptor; @@ -630,15 +643,9 @@ FN_MAX_LIMIT_LIST return extensions.shader_stencil_export; } - /// Returns true if the device supports VK_KHR_create_renderpass2. - bool IsKhrCreateRenderPass2Supported() const { - return extensions.create_renderpass2 || instance_version >= VK_API_VERSION_1_2; - } - /// Returns true if the device supports VK_KHR_depth_stencil_resolve. bool IsKhrDepthStencilResolveSupported() const { - return (extensions.depth_stencil_resolve || instance_version >= VK_API_VERSION_1_2) && - IsKhrCreateRenderPass2Supported(); + return extensions.depth_stencil_resolve || instance_version >= VK_API_VERSION_1_2; } /// Returns the supported resolve modes for the depth aspect. @@ -656,13 +663,6 @@ FN_MAX_LIMIT_LIST return properties.depth_stencil_resolve.independentResolveNone == VK_TRUE; } - /// Returns true if depth/stencil operations can be performed efficiently. - /// Either through shader export or hardware blits. - bool CanPerformDepthStencilOperations() const { - return extensions.shader_stencil_export || is_blit_depth24_stencil8_supported || - is_blit_depth32_stencil8_supported; - } - /// Returns true if the device supports VK_EXT_depth_range_unrestricted. bool IsExtDepthRangeUnrestrictedSupported() const { return extensions.depth_range_unrestricted; @@ -673,6 +673,14 @@ FN_MAX_LIMIT_LIST return extensions.depth_clip_control; } + bool IsLoadOpNoneSupported() const { + return extensions.load_store_op_none || extensions.load_store_op_none_ext; + } + + bool IsExtDynamicRenderingUnusedAttachmentsSupported() const { + return extensions.dynamic_rendering_unused_attachments; + } + /// Returns true if the device supports VK_EXT_depth_bias_control. bool IsExtDepthBiasControlSupported() const { return extensions.depth_bias_control; @@ -703,12 +711,6 @@ FN_MAX_LIMIT_LIST return extensions.transform_feedback && properties.transform_feedback.transformFeedbackDraw; } - /// Returns true if transform feedback query types are supported. - bool IsTransformFeedbackQueriesSupported() const { - return extensions.transform_feedback && - properties.transform_feedback.transformFeedbackQueries; - } - /// Returns true if the device supports VK_EXT_transform_feedback properly. bool AreTransformFeedbackGeometryStreamsSupported() const { return features.transform_feedback.geometryStreams; @@ -743,11 +745,6 @@ FN_MAX_LIMIT_LIST !features.border_color_swizzle.borderColorSwizzleFromImage; } - /// Returns true if borderColorSwizzleFromImage is available. - bool IsBorderColorSwizzleFromImageSupported() const { - return features.border_color_swizzle.borderColorSwizzleFromImage; - } - /// Returns true if the device supports VK_EXT_extended_dynamic_state. bool IsExtExtendedDynamicStateSupported() const { return extensions.extended_dynamic_state; @@ -762,11 +759,6 @@ FN_MAX_LIMIT_LIST return features.extended_dynamic_state2.extendedDynamicState2LogicOp; } - /// Returns true if the device supports VK_EXT_extended_dynamic_state3. - bool IsExtExtendedDynamicState3Supported() const { - return extensions.extended_dynamic_state3; - } - /// Returns true if the device supports VK_EXT_4444_formats. bool IsExt4444FormatsSupported() const { return features.format_a4b4g4r4.formatA4B4G4R4; @@ -891,6 +883,34 @@ FN_MAX_LIMIT_LIST return extensions.conditional_rendering; } + VkPipelineStageFlags2 GetBufferUserStages() const { + VkPipelineStageFlags2 stages = vk::PIPELINE_STAGE_BUFFER_USERS; + if (IsExtConditionalRendering()) { + stages |= VK_PIPELINE_STAGE_2_CONDITIONAL_RENDERING_BIT_EXT; + } + if (IsExtTransformFeedbackSupported()) { + stages |= VK_PIPELINE_STAGE_2_TRANSFORM_FEEDBACK_BIT_EXT; + } + return stages; + } + + VkPipelineStageFlags2 GetBufferConsumerStages() const { + return GetBufferUserStages() | VK_PIPELINE_STAGE_2_HOST_BIT; + } + + VkAccessFlags2 GetBufferConsumerAccess() const { + VkAccessFlags2 access = vk::ACCESS_BUFFER_CONSUMERS; + if (IsExtConditionalRendering()) { + access |= VK_ACCESS_2_CONDITIONAL_RENDERING_READ_BIT_EXT; + } + if (IsExtTransformFeedbackSupported()) { + access |= VK_ACCESS_2_TRANSFORM_FEEDBACK_WRITE_BIT_EXT | + VK_ACCESS_2_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT | + VK_ACCESS_2_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT; + } + return access; + } + bool IsExtAstcDecodeModeSupported() const { return extensions.astc_decode_mode; } @@ -902,11 +922,6 @@ FN_MAX_LIMIT_LIST bool HasTimelineSemaphore() const; - /// Returns true if the device supports VK_KHR_synchronization2. - bool HasSynchronization2() const { - return extensions.synchronization2; - } - /// Returns the minimum supported version of SPIR-V. u32 SupportedSpirvVersion() const { if (instance_version >= VK_API_VERSION_1_3) { @@ -970,16 +985,20 @@ FN_MAX_LIMIT_LIST return must_emulate_scaled_formats; } + bool IsUMA() const { + return is_uma; + } + + bool IsHostImageCopySupported() const { + return extensions.host_image_copy; + } + bool HasNullDescriptor() const { return features.robustness2.nullDescriptor; } bool MustEmulateBGR565() const; - bool HasExactDepthBiasControl() const { - return features.depth_bias_control.depthBiasExact; - } - u32 GetMaxVertexInputAttributes() const { return properties.properties.limits.maxVertexInputAttributes; } @@ -988,6 +1007,10 @@ FN_MAX_LIMIT_LIST return properties.properties.limits.maxVertexInputBindings; } + u32 GetMaxVertexAttribDivisor() const { + return max_vertex_attrib_divisor; + } + u32 GetMaxViewports() const { return properties.properties.limits.maxViewports; } @@ -1004,34 +1027,28 @@ FN_MAX_LIMIT_LIST return features2.features.multiViewport; } - /// Returns true if the device supports VK_KHR_maintenance1. - bool IsKhrMaintenance1Supported() const { - return extensions.maintenance1; - } - - /// Returns true if the device supports VK_KHR_maintenance2. - bool IsKhrMaintenance2Supported() const { - return extensions.maintenance2; - } - - /// Returns true if the device supports VK_KHR_maintenance3. - bool IsKhrMaintenance3Supported() const { - return extensions.maintenance3; - } - /// Returns true if the device supports VK_KHR_maintenance4. bool IsKhrMaintenance4Supported() const { return extensions.maintenance4; } - /// Returns true if the device supports VK_KHR_maintenance5. - bool IsKhrMaintenance5Supported() const { - return extensions.maintenance5; + /// Returns true if the device can read the draw count from a buffer. + bool IsDrawIndirectCountSupported() const { + return extensions.draw_indirect_count; } - /// Returns true if polygon mode POINT supports gl_PointSize. - bool SupportsPolygonModePointSize() const { - return extensions.maintenance5 && properties.maintenance5.polygonModePointSize; + /// Returns true if the device supports VK_EXT_vertex_attribute_divisor. + bool IsExtVertexAttributeDivisorSupported() const { + return extensions.vertex_attribute_divisor; + } + + /// Returns true if the device supports VK_KHR_sampler_mirror_clamp_to_edge. + bool IsKhrSamplerMirrorClampToEdgeSupported() const { + return extensions.sampler_mirror_clamp_to_edge; + } + + bool IsKhrMaintenance5Supported() const { + return extensions.maintenance5; } /// Returns true if depth/stencil swizzle ONE is supported. @@ -1039,28 +1056,6 @@ FN_MAX_LIMIT_LIST return extensions.maintenance5 && properties.maintenance5.depthStencilSwizzleOneSupport; } - /// Returns true if early fragment tests optimizations are available. - bool SupportsEarlyFragmentTests() const { - return extensions.maintenance5 && - properties.maintenance5.earlyFragmentMultisampleCoverageAfterSampleCounting && - properties.maintenance5.earlyFragmentSampleMaskTestBeforeSampleCounting; - } - - /// Returns true if the device supports VK_KHR_maintenance6. - bool IsKhrMaintenance6Supported() const { - return extensions.maintenance6; - } - - /// Returns true if the device supports VK_KHR_maintenance7. - bool IsKhrMaintenance7Supported() const { - return extensions.maintenance7; - } - - /// Returns true if the device supports VK_KHR_maintenance8. - bool IsKhrMaintenance8Supported() const { - return extensions.maintenance8; - } - /// Returns true if the device supports UINT8 index buffer conversion via compute shader. bool SupportsUint8Indices() const { return features.bit8_storage.storageBuffer8BitAccess && @@ -1135,9 +1130,6 @@ private: /// Returns true if the device natively supports blitting depth stencil images. bool TestDepthStencilBlits(VkFormat format) const; - void LoadStaticPipelineCache(); - void SaveStaticPipelineCache() const; - private: VkInstance instance; ///< Vulkan instance. VmaAllocator allocator; ///< VMA allocator. @@ -1146,8 +1138,6 @@ private: vk::Device logical; ///< Logical device. vk::Queue graphics_queue; ///< Main graphics queue. vk::Queue present_queue; ///< Main present queue. - vk::PipelineCache static_pipeline_cache; - bool owns_static_pipeline_cache{}; u32 instance_version{}; ///< Vulkan instance version. u32 graphics_family{}; ///< Main graphics queue family index. u32 present_family{}; ///< Main present queue family index. @@ -1193,10 +1183,12 @@ private: VkPhysicalDevicePushDescriptorPropertiesKHR push_descriptor{}; VkPhysicalDeviceDescriptorBufferPropertiesEXT descriptor_buffer{}; VkPhysicalDeviceSubgroupSizeControlProperties subgroup_size_control{}; + VkPhysicalDeviceTexelBufferAlignmentProperties texel_buffer_alignment{}; VkPhysicalDeviceTransformFeedbackPropertiesEXT transform_feedback{}; VkPhysicalDeviceMaintenance5PropertiesKHR maintenance5{}; VkPhysicalDeviceDepthStencilResolveProperties depth_stencil_resolve{}; VkPhysicalDeviceCustomBorderColorPropertiesEXT custom_border_color{}; + VkPhysicalDeviceVertexAttributeDivisorPropertiesEXT vertex_attribute_divisor{}; VkPhysicalDeviceProperties properties{}; }; @@ -1204,6 +1196,7 @@ private: Extensions extensions{}; Features features{}; Properties properties{}; + u32 max_vertex_attrib_divisor{1}; VkPhysicalDeviceFeatures2 features2{}; VkPhysicalDeviceProperties2 properties2{}; @@ -1214,10 +1207,10 @@ private: bool is_blit_depth32_stencil8_supported{}; ///< Support for blitting from and to D32S8. bool is_warp_potentially_bigger{}; ///< Host warp size can be bigger than guest. bool is_integrated{}; ///< Is GPU an iGPU. - bool is_virtual{}; ///< Is GPU a virtual GPU. - bool is_non_gpu{}; ///< Is SoftwareRasterizer, FPGA, non-GPU device. + bool is_uma{}; bool has_broken_compute{}; ///< Compute shaders can cause crashes bool has_broken_cube_compatibility{}; ///< Has broken cube compatibility bit + bool has_broken_float16_math{}; bool has_broken_descriptor_aliasing{}; ///< Miscompiles descriptors aliased on one binding bool has_broken_parallel_compiling{}; ///< Has broken parallel shader compiling. bool has_renderdoc{}; ///< Has RenderDoc attached diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp index f4721543ae..ef1143aeb6 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.cpp +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.cpp @@ -4,20 +4,8 @@ // SPDX-FileCopyrightText: Copyright 2018 yuzu Emulator Project // SPDX-License-Identifier: GPL-2.0-or-later -#include -#include -#include -#include -#include -#include -#include - -#include "common/alignment.h" -#include "common/assert.h" #include "common/common_types.h" #include "common/literals.h" -#include "common/logging.h" -#include #include "video_core/vulkan_common/vma.h" #include "video_core/vulkan_common/vulkan_device.h" #include "video_core/vulkan_common/vulkan_memory_allocator.h" @@ -30,26 +18,6 @@ namespace { // Helpers translating MemoryUsage to flags/usage - [[maybe_unused]] VkMemoryPropertyFlags MemoryUsagePropertyFlags(MemoryUsage usage) { - switch (usage) { - case MemoryUsage::DeviceLocal: - return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - case MemoryUsage::Upload: - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - case MemoryUsage::Download: - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | - VK_MEMORY_PROPERTY_HOST_CACHED_BIT; - case MemoryUsage::Stream: - return VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | - VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - } - ASSERT_MSG(false, "Invalid memory usage={}", usage); - return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; - } - [[nodiscard]] VkMemoryPropertyFlags MemoryUsagePreferredVmaFlags(MemoryUsage usage) { if (usage == MemoryUsage::Download) { return VK_MEMORY_PROPERTY_HOST_CACHED_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; @@ -58,6 +26,13 @@ namespace { : VkMemoryPropertyFlagBits{}; } + [[nodiscard]] VkMemoryPropertyFlags MemoryUsageRequiredVmaFlags(MemoryUsage usage) { + if (usage == MemoryUsage::Upload || usage == MemoryUsage::Download) { + return VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + } + return VkMemoryPropertyFlagBits{}; + } + [[nodiscard]] VmaAllocationCreateFlags MemoryUsageVmaFlags(MemoryUsage usage) { switch (usage) { case MemoryUsage::Upload: @@ -73,6 +48,26 @@ namespace { return {}; } + template + [[nodiscard]] VkResult AllocateRelaxing(VmaAllocationCreateInfo& ci, F&& allocate) { + VkResult result = allocate(ci); + if (result == VK_SUCCESS) { + return result; + } + if ((ci.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0) { + ci.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; + result = allocate(ci); + if (result == VK_SUCCESS) { + return result; + } + } + if ((ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0) { + ci.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + result = allocate(ci); + } + return result; + } + [[nodiscard]] VmaMemoryUsage MemoryUsageVma(MemoryUsage usage) { switch (usage) { case MemoryUsage::DeviceLocal: @@ -86,106 +81,9 @@ namespace { } } // namespace -//MemoryCommit is now VMA-backed -MemoryCommit::MemoryCommit(VmaAllocator alloc, VmaAllocation a, - const VmaAllocationInfo &info) noexcept - : allocator{alloc}, allocation{a}, memory{info.deviceMemory}, - offset{info.offset}, size{info.size}, mapped_ptr{info.pMappedData} { - // Log GPU memory allocation - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_memory_tracking.GetValue()) { - GPU::Logging::GPULogger::GetInstance().LogMemoryAllocation( - reinterpret_cast(memory), - static_cast(size), - 0 // Memory property flags (not easily available from VMA) - ); - } -} - -MemoryCommit::~MemoryCommit() { Release(); } - -MemoryCommit::MemoryCommit(MemoryCommit &&rhs) noexcept - : allocator{std::exchange(rhs.allocator, nullptr)}, - allocation{std::exchange(rhs.allocation, nullptr)}, - memory{std::exchange(rhs.memory, VK_NULL_HANDLE)}, - offset{std::exchange(rhs.offset, 0)}, - size{std::exchange(rhs.size, 0)}, - mapped_ptr{std::exchange(rhs.mapped_ptr, nullptr)} {} - -MemoryCommit &MemoryCommit::operator=(MemoryCommit &&rhs) noexcept { - if (this != &rhs) { - Release(); - allocator = std::exchange(rhs.allocator, nullptr); - allocation = std::exchange(rhs.allocation, nullptr); - memory = std::exchange(rhs.memory, VK_NULL_HANDLE); - offset = std::exchange(rhs.offset, 0); - size = std::exchange(rhs.size, 0); - mapped_ptr = std::exchange(rhs.mapped_ptr, nullptr); - } - return *this; -} - -std::span MemoryCommit::Map() -{ - if (!allocation) return {}; - if (!mapped_ptr) { - if (vmaMapMemory(allocator, allocation, &mapped_ptr) != VK_SUCCESS) return {}; - } - const size_t n = static_cast(std::min(size, - (std::numeric_limits::max)())); - return std::span{static_cast(mapped_ptr), n}; -} - -std::span MemoryCommit::Map() const -{ - if (!allocation) return {}; - if (!mapped_ptr) { - void *p = nullptr; - if (vmaMapMemory(allocator, allocation, &p) != VK_SUCCESS) return {}; - const_cast(this)->mapped_ptr = p; - } - const size_t n = static_cast(std::min(size, - (std::numeric_limits::max)())); - return std::span{static_cast(mapped_ptr), n}; -} - -void MemoryCommit::Unmap() -{ - if (allocation && mapped_ptr) { - vmaUnmapMemory(allocator, allocation); - mapped_ptr = nullptr; - } -} - -void MemoryCommit::Release() { - if (allocation && allocator) { - // Log GPU memory deallocation - if (GPU::Logging::IsActive() && - Settings::values.gpu_log_memory_tracking.GetValue() && - memory != VK_NULL_HANDLE) { - GPU::Logging::GPULogger::GetInstance().LogMemoryDeallocation( - reinterpret_cast(memory) - ); - } - - if (mapped_ptr) { - vmaUnmapMemory(allocator, allocation); - mapped_ptr = nullptr; - } - vmaFreeMemory(allocator, allocation); - } - allocation = nullptr; - allocator = nullptr; - memory = VK_NULL_HANDLE; - offset = 0; - size = 0; -} - MemoryAllocator::MemoryAllocator(const Device &device_) : device{device_}, allocator{device.GetAllocator()}, - properties{device_.GetPhysical().GetMemoryProperties().memoryProperties}, - buffer_image_granularity{ - device_.GetPhysical().GetProperties().limits.bufferImageGranularity} { + properties{device_.GetPhysical().GetMemoryProperties().memoryProperties} { // Preserve the previous "RenderDoc small heap" trimming behavior that we had in original vma minus the heap bug if (device.HasDebuggingToolAttached()) @@ -207,12 +105,12 @@ MemoryAllocator::~MemoryAllocator() = default; vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const { - const VmaAllocationCreateInfo alloc_ci = { + VmaAllocationCreateInfo alloc_ci = { .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT, .usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE, .requiredFlags = 0, .preferredFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, - .memoryTypeBits = 0, + .memoryTypeBits = valid_memory_types, .pool = VK_NULL_HANDLE, .pUserData = nullptr, .priority = 0.f, @@ -221,7 +119,9 @@ vk::Image MemoryAllocator::CreateImage(const VkImageCreateInfo &ci) const VkImage handle{}; VmaAllocation allocation{}; VmaAllocationInfo alloc_info{}; - vk::Check(vmaCreateImage(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + vk::Check(AllocateRelaxing(alloc_ci, [&](const VmaAllocationCreateInfo& retry) { + return vmaCreateImage(allocator, &ci, &retry, &handle, &allocation, &alloc_info); + })); // Log GPU memory allocation for images if (GPU::Logging::IsActive() && @@ -242,10 +142,10 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa auto const anv_flags = (usage == MemoryUsage::Stream && device.GetDriverID() == VK_DRIVER_ID_INTEL_OPEN_SOURCE_MESA) ? VK_MEMORY_PROPERTY_HOST_CACHED_BIT : 0; - const VmaAllocationCreateInfo alloc_ci = { + VmaAllocationCreateInfo alloc_ci = { .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage), .usage = MemoryUsageVma(usage), - .requiredFlags = 0, + .requiredFlags = MemoryUsageRequiredVmaFlags(usage), .preferredFlags = MemoryUsagePreferredVmaFlags(usage) | anv_flags, .memoryTypeBits = usage == MemoryUsage::Stream ? 0u : valid_memory_types, .pool = VK_NULL_HANDLE, @@ -258,7 +158,9 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBuffer(allocator, &ci, &alloc_ci, &handle, &allocation, &alloc_info)); + vk::Check(AllocateRelaxing(alloc_ci, [&](const VmaAllocationCreateInfo& retry) { + return vmaCreateBuffer(allocator, &ci, &retry, &handle, &allocation, &alloc_info); + })); vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); // Log GPU memory allocation for buffers @@ -298,10 +200,10 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa if (usage == MemoryUsage::Stream) { memory_type_bits = 0u; } - const VmaAllocationCreateInfo alloc_ci = { + VmaAllocationCreateInfo alloc_ci = { .flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage), .usage = MemoryUsageVma(usage), - .requiredFlags = 0, + .requiredFlags = MemoryUsageRequiredVmaFlags(usage), .preferredFlags = MemoryUsagePreferredVmaFlags(usage) | anv_flags, .memoryTypeBits = memory_type_bits, .pool = VK_NULL_HANDLE, @@ -314,8 +216,10 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa VmaAllocation allocation{}; VkMemoryPropertyFlags property_flags{}; - vk::Check(vmaCreateBufferWithAlignment(allocator, &ci, &alloc_ci, min_alignment, &handle, - &allocation, &alloc_info)); + vk::Check(AllocateRelaxing(alloc_ci, [&](const VmaAllocationCreateInfo& retry) { + return vmaCreateBufferWithAlignment(allocator, &ci, &retry, min_alignment, &handle, + &allocation, &alloc_info); + })); vmaGetAllocationMemoryProperties(allocator, allocation, &property_flags); u8 *data = reinterpret_cast(alloc_info.pMappedData); @@ -334,75 +238,4 @@ vk::Buffer MemoryAllocator::CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsa location, device.GetDispatchLoader()); } -MemoryCommit MemoryAllocator::Commit(const VkMemoryRequirements &reqs, MemoryUsage usage) -{ - const auto vma_usage = MemoryUsageVma(usage); - VmaAllocationCreateInfo ci{}; - ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage); - ci.usage = vma_usage; - ci.memoryTypeBits = reqs.memoryTypeBits & valid_memory_types; - ci.requiredFlags = 0; - ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage); - - VmaAllocation a{}; - VmaAllocationInfo info{}; - - VkResult res = vmaAllocateMemory(allocator, &reqs, &ci, &a, &info); - - if (res != VK_SUCCESS) { - // Relax 1: drop budget constraint - auto ci2 = ci; - ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; - res = vmaAllocateMemory(allocator, &reqs, &ci2, &a, &info); - - // Relax 2: if we preferred DEVICE_LOCAL, drop that preference - if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { - auto ci3 = ci2; - ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - res = vmaAllocateMemory(allocator, &reqs, &ci3, &a, &info); - } - } - - vk::Check(res); - return MemoryCommit(allocator, a, info); -} - -MemoryCommit MemoryAllocator::Commit(const vk::Buffer &buffer, MemoryUsage usage) { - // Allocate memory appropriate for this buffer automatically - const auto vma_usage = MemoryUsageVma(usage); - - VmaAllocationCreateInfo ci{}; - ci.flags = VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT | MemoryUsageVmaFlags(usage); - ci.usage = vma_usage; - ci.requiredFlags = 0; - ci.preferredFlags = MemoryUsagePreferredVmaFlags(usage); - ci.pool = VK_NULL_HANDLE; - ci.pUserData = nullptr; - ci.priority = 0.0f; - - const VkBuffer raw = *buffer; - - VmaAllocation a{}; - VmaAllocationInfo info{}; - - // Let VMA infer memory requirements from the buffer - VkResult res = vmaAllocateMemoryForBuffer(allocator, raw, &ci, &a, &info); - - if (res != VK_SUCCESS) { - auto ci2 = ci; - ci2.flags &= ~VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT; - res = vmaAllocateMemoryForBuffer(allocator, raw, &ci2, &a, &info); - - if (res != VK_SUCCESS && (ci.preferredFlags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { - auto ci3 = ci2; - ci3.preferredFlags &= ~VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - res = vmaAllocateMemoryForBuffer(allocator, raw, &ci3, &a, &info); - } - } - - vk::Check(res); - vk::Check(vmaBindBufferMemory2(allocator, a, 0, raw, nullptr)); - return MemoryCommit(allocator, a, info); -} - } // namespace Vulkan diff --git a/src/video_core/vulkan_common/vulkan_memory_allocator.h b/src/video_core/vulkan_common/vulkan_memory_allocator.h index a0f9081fc0..79a33f0b7b 100644 --- a/src/video_core/vulkan_common/vulkan_memory_allocator.h +++ b/src/video_core/vulkan_common/vulkan_memory_allocator.h @@ -39,51 +39,6 @@ namespace Vulkan { } } -/// Ownership handle of a memory commitment (real VMA allocation). - class MemoryCommit { - public: - MemoryCommit() noexcept = default; - - MemoryCommit(VmaAllocator allocator, VmaAllocation allocation, - const VmaAllocationInfo &info) noexcept; - - ~MemoryCommit(); - - MemoryCommit(const MemoryCommit &) = delete; - - MemoryCommit &operator=(const MemoryCommit &) = delete; - - MemoryCommit(MemoryCommit &&) noexcept; - - MemoryCommit &operator=(MemoryCommit &&) noexcept; - - [[nodiscard]] std::span Map(); - - [[nodiscard]] std::span Map() const; - - void Unmap(); - - explicit operator bool() const noexcept { return allocation != nullptr; } - - VkDeviceMemory Memory() const noexcept { return memory; } - - VkDeviceSize Offset() const noexcept { return offset; } - - VkDeviceSize Size() const noexcept { return size; } - - VmaAllocation Allocation() const noexcept { return allocation; } - - private: - void Release(); - - VmaAllocator allocator{}; ///< VMA allocator - VmaAllocation allocation{}; ///< VMA allocation handle - VkDeviceMemory memory{}; ///< Underlying VkDeviceMemory chosen by VMA - VkDeviceSize offset{}; ///< Offset of this allocation inside VkDeviceMemory - VkDeviceSize size{}; ///< Size of the allocation - void *mapped_ptr{}; ///< Optional persistent mapped pointer - }; - /// Memory allocator container. /// Allocates and releases memory allocations on demand. class MemoryAllocator { @@ -110,35 +65,10 @@ namespace Vulkan { vk::Buffer CreateBuffer(const VkBufferCreateInfo &ci, MemoryUsage usage, VkDeviceSize min_alignment) const; - /** - * Commits a memory with the specified requirements. - * - * @param requirements Requirements returned from a Vulkan call. - * @param usage Indicates how the memory will be used. - * - * @returns A memory commit. - */ - MemoryCommit Commit(const VkMemoryRequirements &requirements, MemoryUsage usage); - - /// Commits memory required by the buffer and binds it (for buffers created outside VMA). - MemoryCommit Commit(const vk::Buffer &buffer, MemoryUsage usage); - private: - static bool IsAutoUsage(VmaMemoryUsage u) noexcept { - switch (u) { - case VMA_MEMORY_USAGE_AUTO: - case VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE: - case VMA_MEMORY_USAGE_AUTO_PREFER_HOST: - return true; - default: - return false; - } - } - const Device &device; ///< Device handle. VmaAllocator allocator; ///< VMA allocator. const VkPhysicalDeviceMemoryProperties properties; ///< Physical device memory properties. - VkDeviceSize buffer_image_granularity; ///< Adjacent buffer/image granularity u32 valid_memory_types{~0u}; }; diff --git a/src/video_core/vulkan_common/vulkan_wrapper.cpp b/src/video_core/vulkan_common/vulkan_wrapper.cpp index 5044881598..c6b3afa2c7 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.cpp +++ b/src/video_core/vulkan_common/vulkan_wrapper.cpp @@ -86,26 +86,26 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkAllocateCommandBuffers); X(vkAllocateDescriptorSets); X(vkAllocateMemory); - X(vkBeginCommandBuffer); X(vkBindBufferMemory); - X(vkBindImageMemory); + X(vkFreeMemory); + X(vkBeginCommandBuffer); X(vkCmdBeginConditionalRenderingEXT); X(vkCmdBeginQuery); - X(vkCmdBeginRenderPass); + X(vkCmdBeginRendering); X(vkCmdBeginTransformFeedbackEXT); - X(vkCmdBeginDebugUtilsLabelEXT); X(vkCmdBindDescriptorSets); X(vkCmdBindIndexBuffer); + X(vkCmdBindIndexBuffer2KHR); X(vkCmdBindPipeline); X(vkCmdBindTransformFeedbackBuffersEXT); X(vkCmdBindVertexBuffers); - X(vkCmdBlitImage); + X(vkCmdBlitImage2); X(vkCmdClearAttachments); X(vkCmdClearColorImage); - X(vkCmdCopyBuffer); - X(vkCmdCopyBufferToImage); - X(vkCmdCopyImage); - X(vkCmdCopyImageToBuffer); + X(vkCmdCopyBuffer2); + X(vkCmdCopyBufferToImage2); + X(vkCmdCopyImage2); + X(vkCmdCopyImageToBuffer2); X(vkCmdCopyQueryPoolResults); X(vkCmdDispatch); X(vkCmdDispatchIndirect); @@ -118,11 +118,9 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdDrawIndirectByteCountEXT); X(vkCmdEndConditionalRenderingEXT); X(vkCmdEndQuery); - X(vkCmdEndRenderPass); + X(vkCmdEndRendering); X(vkCmdEndTransformFeedbackEXT); - X(vkCmdEndDebugUtilsLabelEXT); X(vkCmdFillBuffer); - X(vkCmdPipelineBarrier); X(vkCmdPipelineBarrier2); X(vkCmdPushConstants); X(vkCmdPushDescriptorSetWithTemplateKHR); @@ -130,13 +128,11 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdSetDepthBias); X(vkCmdSetDepthBias2EXT); X(vkCmdSetDepthBounds); - X(vkCmdSetEvent); X(vkCmdSetScissor); X(vkCmdSetStencilCompareMask); X(vkCmdSetStencilReference); X(vkCmdSetStencilWriteMask); X(vkCmdSetViewport); - X(vkCmdWaitEvents); X(vkCmdBindVertexBuffers2EXT); X(vkCmdSetCullModeEXT); X(vkCmdSetDepthBoundsTestEnableEXT); @@ -155,7 +151,6 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdSetDepthClampEnableEXT); X(vkCmdSetFrontFaceEXT); X(vkCmdSetLogicOpEXT); - X(vkCmdSetPatchControlPointsEXT); X(vkCmdSetLineWidth); X(vkCmdSetPrimitiveTopologyEXT); X(vkCmdSetStencilOpEXT); @@ -166,7 +161,7 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCmdSetColorBlendEnableEXT); X(vkCmdSetColorBlendEquationEXT); X(vkCmdResetQueryPool); - X(vkCmdResolveImage); + X(vkCmdResolveImage2); X(vkCreateBuffer); X(vkCreateBufferView); X(vkCreateCommandPool); @@ -174,17 +169,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkCreateDescriptorPool); X(vkCreateDescriptorSetLayout); X(vkCreateDescriptorUpdateTemplate); - X(vkCreateEvent); X(vkCreateFence); - X(vkCreateFramebuffer); X(vkCreateGraphicsPipelines); - X(vkCreateImage); X(vkCreateImageView); X(vkCreatePipelineCache); X(vkCreatePipelineLayout); X(vkCreateQueryPool); - X(vkCreateRenderPass); - X(vkCreateRenderPass2); X(vkCreateSampler); X(vkCreateSemaphore); X(vkCreateShaderModule); @@ -195,16 +185,13 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkDestroyDescriptorPool); X(vkDestroyDescriptorSetLayout); X(vkDestroyDescriptorUpdateTemplate); - X(vkDestroyEvent); X(vkDestroyFence); - X(vkDestroyFramebuffer); X(vkDestroyImage); X(vkDestroyImageView); X(vkDestroyPipeline); X(vkDestroyPipelineCache); X(vkDestroyPipelineLayout); X(vkDestroyQueryPool); - X(vkDestroyRenderPass); X(vkDestroySampler); X(vkDestroySemaphore); X(vkDestroyShaderModule); @@ -213,30 +200,22 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { X(vkEndCommandBuffer); X(vkFreeCommandBuffers); X(vkFreeDescriptorSets); - X(vkFreeMemory); X(vkGetBufferMemoryRequirements2); + X(vkGetDeviceBufferMemoryRequirements); X(vkGetDeviceQueue); - X(vkGetEventStatus); X(vkGetFenceStatus); - X(vkGetImageMemoryRequirements); X(vkGetPipelineCacheData); - X(vkGetMemoryFdKHR); -#ifdef _WIN32 - X(vkGetMemoryWin32HandleKHR); -#endif X(vkGetQueryPoolResults); X(vkGetPipelineExecutablePropertiesKHR); X(vkGetPipelineExecutableStatisticsKHR); X(vkGetSemaphoreCounterValue); - X(vkMapMemory); + X(vkCopyMemoryToImageEXT); + X(vkTransitionImageLayoutEXT); X(vkQueueBindSparse); - X(vkQueueSubmit); X(vkQueueSubmit2); X(vkResetFences); X(vkResetQueryPool); X(vkSetDebugUtilsObjectNameEXT); - X(vkSetDebugUtilsObjectTagEXT); - X(vkUnmapMemory); X(vkUpdateDescriptorSetWithTemplate); X(vkUpdateDescriptorSets); X(vkGetBufferDeviceAddress); @@ -265,6 +244,12 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { Proc(dld.vkCmdDrawIndexedIndirectCount, dld, "vkCmdDrawIndexedIndirectCountKHR", device); } + // Maintenance4 queries are core in Vulkan 1.3, otherwise require VK_KHR_maintenance4 + if (!dld.vkGetDeviceBufferMemoryRequirements) { + Proc(dld.vkGetDeviceBufferMemoryRequirements, dld, + "vkGetDeviceBufferMemoryRequirementsKHR", device); + } + // Synchronization2 is core in Vulkan 1.3, otherwise requires VK_KHR_synchronization2 if (!dld.vkCmdPipelineBarrier2) { Proc(dld.vkCmdPipelineBarrier2, dld, "vkCmdPipelineBarrier2KHR", device); @@ -273,9 +258,19 @@ void Load(VkDevice device, DeviceDispatch& dld) noexcept { Proc(dld.vkQueueSubmit2, dld, "vkQueueSubmit2KHR", device); } - if (!dld.vkCreateRenderPass2) { - Proc(dld.vkCreateRenderPass2, dld, "vkCreateRenderPass2KHR", device); - } + const auto load_khr = [&](auto& function, const char* name) { + if (!function) { + Proc(function, dld, name, device); + } + }; + load_khr(dld.vkCmdBeginRendering, "vkCmdBeginRenderingKHR"); + load_khr(dld.vkCmdEndRendering, "vkCmdEndRenderingKHR"); + load_khr(dld.vkCmdBlitImage2, "vkCmdBlitImage2KHR"); + load_khr(dld.vkCmdCopyBuffer2, "vkCmdCopyBuffer2KHR"); + load_khr(dld.vkCmdCopyBufferToImage2, "vkCmdCopyBufferToImage2KHR"); + load_khr(dld.vkCmdCopyImage2, "vkCmdCopyImage2KHR"); + load_khr(dld.vkCmdCopyImageToBuffer2, "vkCmdCopyImageToBuffer2KHR"); + load_khr(dld.vkCmdResolveImage2, "vkCmdResolveImage2KHR"); #undef X } @@ -308,9 +303,7 @@ bool Load(VkInstance instance, InstanceDispatch& dld) noexcept { // These functions may fail to load depending on the enabled extensions. // Don't return a failure on these. X(vkCreateDebugUtilsMessengerEXT); - X(vkCreateDebugReportCallbackEXT); X(vkDestroyDebugUtilsMessengerEXT); - X(vkDestroyDebugReportCallbackEXT); X(vkDestroySurfaceKHR); X(vkGetPhysicalDeviceFeatures2); X(vkGetPhysicalDeviceFormatProperties2); @@ -323,10 +316,10 @@ bool Load(VkInstance instance, InstanceDispatch& dld) noexcept { X(vkGetSwapchainImagesKHR); X(vkQueuePresentKHR); - return X(vkCreateDevice) && X(vkDestroyDevice) && X(vkDestroyDevice) && + return X(vkCreateDevice) && X(vkDestroyDevice) && X(vkEnumerateDeviceExtensionProperties) && X(vkEnumeratePhysicalDevices) && X(vkGetDeviceProcAddr) && X(vkGetPhysicalDeviceFormatProperties) && - X(vkGetPhysicalDeviceMemoryProperties) && X(vkGetPhysicalDeviceMemoryProperties2) && + X(vkGetPhysicalDeviceMemoryProperties2) && X(vkGetPhysicalDeviceProperties) && X(vkGetPhysicalDeviceQueueFamilyProperties); #undef X } @@ -347,6 +340,10 @@ void Destroy(VkDevice device, VkBuffer handle, const DeviceDispatch& dld) noexce dld.vkDestroyBuffer(device, handle, nullptr); } +void Destroy(VkDevice device, VkDeviceMemory handle, const DeviceDispatch& dld) noexcept { + dld.vkFreeMemory(device, handle, nullptr); +} + void Destroy(VkDevice device, VkBufferView handle, const DeviceDispatch& dld) noexcept { dld.vkDestroyBufferView(device, handle, nullptr); } @@ -368,22 +365,10 @@ void Destroy(VkDevice device, VkDescriptorUpdateTemplate handle, dld.vkDestroyDescriptorUpdateTemplate(device, handle, nullptr); } -void Destroy(VkDevice device, VkDeviceMemory handle, const DeviceDispatch& dld) noexcept { - dld.vkFreeMemory(device, handle, nullptr); -} - -void Destroy(VkDevice device, VkEvent handle, const DeviceDispatch& dld) noexcept { - dld.vkDestroyEvent(device, handle, nullptr); -} - void Destroy(VkDevice device, VkFence handle, const DeviceDispatch& dld) noexcept { dld.vkDestroyFence(device, handle, nullptr); } -void Destroy(VkDevice device, VkFramebuffer handle, const DeviceDispatch& dld) noexcept { - dld.vkDestroyFramebuffer(device, handle, nullptr); -} - void Destroy(VkDevice device, VkImage handle, const DeviceDispatch& dld) noexcept { dld.vkDestroyImage(device, handle, nullptr); } @@ -408,10 +393,6 @@ void Destroy(VkDevice device, VkQueryPool handle, const DeviceDispatch& dld) noe dld.vkDestroyQueryPool(device, handle, nullptr); } -void Destroy(VkDevice device, VkRenderPass handle, const DeviceDispatch& dld) noexcept { - dld.vkDestroyRenderPass(device, handle, nullptr); -} - void Destroy(VkDevice device, VkSampler handle, const DeviceDispatch& dld) noexcept { dld.vkDestroySampler(device, handle, nullptr); } @@ -433,11 +414,6 @@ void Destroy(VkInstance instance, VkDebugUtilsMessengerEXT handle, dld.vkDestroyDebugUtilsMessengerEXT(instance, handle, nullptr); } -void Destroy(VkInstance instance, VkDebugReportCallbackEXT handle, - const InstanceDispatch& dld) noexcept { - dld.vkDestroyDebugReportCallbackEXT(instance, handle, nullptr); -} - void Destroy(VkInstance instance, VkSurfaceKHR handle, const InstanceDispatch& dld) noexcept { dld.vkDestroySurfaceKHR(instance, handle, nullptr); } @@ -460,20 +436,8 @@ Instance Instance::Create(u32 version, Span layers, Span gloom + yellow line glitch appears -#ifdef __ANDROID__ - const VkApplicationInfo application_info{ - .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, - .pNext = nullptr, - .pApplicationName = "PUBGMobile", - .applicationVersion = VK_MAKE_VERSION(1, 7, 0), - .pEngineName = "UnrealEngine", - .engineVersion = VK_MAKE_VERSION(4, 23, 0), - .apiVersion = VK_API_VERSION_1_3, - }; -#else + // DO NOT TOUCH, breaks RNDA3!! + // Don't know why, but gloom + yellow line glitch appears const VkApplicationInfo application_info{ .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO, .pNext = nullptr, @@ -483,7 +447,6 @@ Instance Instance::Create(u32 version, Span layers, SpanvkCreateDebugReportCallbackEXT(handle, &create_info, nullptr, &object)); - return DebugReportCallback(object, handle, *dld); -} - void Image::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE, name); } @@ -567,44 +523,10 @@ void ImageView::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_IMAGE_VIEW, name); } -int DeviceMemory::GetMemoryFdKHR() const { - const VkMemoryGetFdInfoKHR get_fd_info{ - .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR, - .pNext = nullptr, - .memory = handle, - .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR, - }; - int fd; - Check(dld->vkGetMemoryFdKHR(owner, &get_fd_info, &fd)); - return fd; -} - -#ifdef _WIN32 -HANDLE DeviceMemory::GetMemoryWin32HandleKHR() const { - const VkMemoryGetWin32HandleInfoKHR get_win32_handle_info{ - .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR, - .pNext = nullptr, - .memory = handle, - .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT_KHR, - }; - HANDLE win32_handle; - Check(dld->vkGetMemoryWin32HandleKHR(owner, &get_win32_handle_info, &win32_handle)); - return win32_handle; -} -#endif - -void DeviceMemory::SetObjectNameEXT(const char* name) const { - SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_DEVICE_MEMORY, name); -} - void Fence::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_FENCE, name); } -void Framebuffer::SetObjectNameEXT(const char* name) const { - SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_FRAMEBUFFER, name); -} - DescriptorSets DescriptorPool::Allocate(const VkDescriptorSetAllocateInfo& ai) const { const std::size_t num = ai.descriptorSetCount; std::unique_ptr sets = std::make_unique(num); @@ -656,10 +578,6 @@ std::vector SwapchainKHR::GetImages() const { return images; } -void Event::SetObjectNameEXT(const char* name) const { - SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_EVENT, name); -} - void ShaderModule::SetObjectNameEXT(const char* name) const { SetObjectName(dld, owner, handle, VK_OBJECT_TYPE_SHADER_MODULE, name); } @@ -705,6 +623,37 @@ BufferView Device::CreateBufferView(const VkBufferViewCreateInfo& ci) const { return BufferView(object, handle, *dld); } +ExternalBuffer Device::CreateExternalBuffer(const VkBufferCreateInfo& ci) const { + VkBuffer object; + Check(dld->vkCreateBuffer(handle, &ci, nullptr, &object)); + return ExternalBuffer(object, handle, *dld); +} + +DeviceMemory Device::AllocateMemory(const VkMemoryAllocateInfo& ai) const { + VkDeviceMemory object; + Check(dld->vkAllocateMemory(handle, &ai, nullptr, &object)); + return DeviceMemory(object, handle, *dld); +} + +void Device::BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) const { + Check(dld->vkBindBufferMemory(handle, buffer, memory, offset)); +} + +VkMemoryRequirements Device::GetBufferMemoryRequirements(VkBuffer buffer) const noexcept { + const VkBufferMemoryRequirementsInfo2 info{ + .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2, + .pNext = nullptr, + .buffer = buffer, + }; + VkMemoryRequirements2 requirements{ + .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, + .pNext = nullptr, + .memoryRequirements = {}, + }; + dld->vkGetBufferMemoryRequirements2(handle, &info, &requirements); + return requirements.memoryRequirements; +} + ImageView Device::CreateImageView(const VkImageViewCreateInfo& ci) const { VkImageView object; Check(dld->vkCreateImageView(handle, &ci, nullptr, &object)); @@ -738,18 +687,6 @@ DescriptorPool Device::CreateDescriptorPool(const VkDescriptorPoolCreateInfo& ci return DescriptorPool(object, handle, *dld); } -RenderPass Device::CreateRenderPass(const VkRenderPassCreateInfo& ci) const { - VkRenderPass object; - Check(dld->vkCreateRenderPass(handle, &ci, nullptr, &object)); - return RenderPass(object, handle, *dld); -} - -RenderPass Device::CreateRenderPass2(const VkRenderPassCreateInfo2& ci) const { - VkRenderPass object; - Check(dld->vkCreateRenderPass2(handle, &ci, nullptr, &object)); - return RenderPass(object, handle, *dld); -} - DescriptorSetLayout Device::CreateDescriptorSetLayout( const VkDescriptorSetLayoutCreateInfo& ci) const { VkDescriptorSetLayout object; @@ -794,12 +731,6 @@ Sampler Device::CreateSampler(const VkSamplerCreateInfo& ci) const { return Sampler(object, handle, *dld); } -Framebuffer Device::CreateFramebuffer(const VkFramebufferCreateInfo& ci) const { - VkFramebuffer object; - Check(dld->vkCreateFramebuffer(handle, &ci, nullptr, &object)); - return Framebuffer(object, handle, *dld); -} - CommandPool Device::CreateCommandPool(const VkCommandPoolCreateInfo& ci) const { VkCommandPool object; Check(dld->vkCreateCommandPool(handle, &ci, nullptr, &object)); @@ -825,60 +756,28 @@ ShaderModule Device::CreateShaderModule(const VkShaderModuleCreateInfo& ci) cons return ShaderModule(object, handle, *dld); } -Event Device::CreateEvent() const { - static constexpr VkEventCreateInfo ci{ - .sType = VK_STRUCTURE_TYPE_EVENT_CREATE_INFO, - .pNext = nullptr, - .flags = 0, - }; - - VkEvent object; - Check(dld->vkCreateEvent(handle, &ci, nullptr, &object)); - return Event(object, handle, *dld); -} - SwapchainKHR Device::CreateSwapchainKHR(const VkSwapchainCreateInfoKHR& ci) const { VkSwapchainKHR object; Check(dld->vkCreateSwapchainKHR(handle, &ci, nullptr, &object)); return SwapchainKHR(object, handle, *dld); } -DeviceMemory Device::TryAllocateMemory(const VkMemoryAllocateInfo& ai) const noexcept { - VkDeviceMemory memory; - if (dld->vkAllocateMemory(handle, &ai, nullptr, &memory) != VK_SUCCESS) { - return {}; - } - return DeviceMemory(memory, handle, *dld); -} - -DeviceMemory Device::AllocateMemory(const VkMemoryAllocateInfo& ai) const { - VkDeviceMemory memory; - Check(dld->vkAllocateMemory(handle, &ai, nullptr, &memory)); - return DeviceMemory(memory, handle, *dld); -} - -VkMemoryRequirements Device::GetBufferMemoryRequirements(VkBuffer buffer, - void* pnext) const noexcept { - const VkBufferMemoryRequirementsInfo2 info{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2, +VkMemoryRequirements Device::GetDeviceBufferMemoryRequirements( + const VkBufferCreateInfo& ci) const noexcept { + const VkDeviceBufferMemoryRequirements info{ + .sType = VK_STRUCTURE_TYPE_DEVICE_BUFFER_MEMORY_REQUIREMENTS, .pNext = nullptr, - .buffer = buffer, + .pCreateInfo = &ci, }; VkMemoryRequirements2 requirements{ .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2, - .pNext = pnext, + .pNext = nullptr, .memoryRequirements{}, }; - dld->vkGetBufferMemoryRequirements2(handle, &info, &requirements); + dld->vkGetDeviceBufferMemoryRequirements(handle, &info, &requirements); return requirements.memoryRequirements; } -VkMemoryRequirements Device::GetImageMemoryRequirements(VkImage image) const noexcept { - VkMemoryRequirements requirements; - dld->vkGetImageMemoryRequirements(handle, image, &requirements); - return requirements; -} - std::vector Device::GetPipelineExecutablePropertiesKHR( VkPipeline pipeline) const { const VkPipelineInfoKHR info{ @@ -930,14 +829,6 @@ void PhysicalDevice::GetProperties2(VkPhysicalDeviceProperties2& properties) con dld->vkGetPhysicalDeviceProperties2(physical_device, &properties); } -VkPhysicalDeviceFeatures PhysicalDevice::GetFeatures() const noexcept { - VkPhysicalDeviceFeatures2 features2; - features2.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2; - features2.pNext = nullptr; - dld->vkGetPhysicalDeviceFeatures2(physical_device, &features2); - return features2.features; -} - void PhysicalDevice::GetFeatures2(VkPhysicalDeviceFeatures2& features) const noexcept { dld->vkGetPhysicalDeviceFeatures2(physical_device, &features); } diff --git a/src/video_core/vulkan_common/vulkan_wrapper.h b/src/video_core/vulkan_common/vulkan_wrapper.h index 97d2ba3700..bd9bd1b5bd 100644 --- a/src/video_core/vulkan_common/vulkan_wrapper.h +++ b/src/video_core/vulkan_common/vulkan_wrapper.h @@ -147,16 +147,53 @@ inline VkResult Filter(VkResult result) { return result; } -inline constexpr VkPipelineStageFlags PIPELINE_STAGE_GRAPHICS_COMPUTE = - VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; +inline constexpr VkPipelineStageFlags2 PIPELINE_STAGE_HOST = VK_PIPELINE_STAGE_2_HOST_BIT; -inline constexpr VkPipelineStageFlags PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER = - PIPELINE_STAGE_GRAPHICS_COMPUTE | VK_PIPELINE_STAGE_TRANSFER_BIT; +inline constexpr VkPipelineStageFlags2 PIPELINE_STAGE_GRAPHICS_SHADERS = + VK_PIPELINE_STAGE_2_PRE_RASTERIZATION_SHADERS_BIT | VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT; -inline constexpr VkPipelineStageFlags PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER_HOST = - PIPELINE_STAGE_GRAPHICS_COMPUTE_TRANSFER | VK_PIPELINE_STAGE_HOST_BIT; +inline constexpr VkPipelineStageFlags2 PIPELINE_STAGE_BUFFER_INPUTS = + VK_PIPELINE_STAGE_2_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_2_VERTEX_INPUT_BIT | + PIPELINE_STAGE_GRAPHICS_SHADERS | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT; -inline constexpr VkPipelineStageFlags PIPELINE_STAGE_HOST = VK_PIPELINE_STAGE_HOST_BIT; +inline constexpr VkPipelineStageFlags2 PIPELINE_STAGE_ATTACHMENTS = + VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT | + VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT; + +inline constexpr VkAccessFlags2 ACCESS_SHADER_RESOURCES = + VK_ACCESS_2_UNIFORM_READ_BIT | VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT; + +inline constexpr VkAccessFlags2 ACCESS_BUFFER_INPUTS = + VK_ACCESS_2_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_2_INDEX_READ_BIT | + VK_ACCESS_2_VERTEX_ATTRIBUTE_READ_BIT | ACCESS_SHADER_RESOURCES; + +inline constexpr VkAccessFlags2 ACCESS_ATTACHMENTS = + VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; + +inline constexpr VkAccessFlags2 ACCESS_TRANSFER = + VK_ACCESS_2_TRANSFER_READ_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT; + +inline constexpr VkPipelineStageFlags2 PIPELINE_STAGE_BUFFER_USERS = + PIPELINE_STAGE_BUFFER_INPUTS | VK_PIPELINE_STAGE_2_TRANSFER_BIT; + +inline constexpr VkAccessFlags2 ACCESS_BUFFER_WRITES = + VK_ACCESS_2_SHADER_WRITE_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT; + +inline constexpr VkAccessFlags2 ACCESS_BUFFER_CONSUMERS = + ACCESS_BUFFER_INPUTS | ACCESS_TRANSFER | VK_ACCESS_2_HOST_READ_BIT; + +inline constexpr VkPipelineStageFlags2 PIPELINE_STAGE_IMAGE_USERS = + PIPELINE_STAGE_GRAPHICS_SHADERS | VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT | + PIPELINE_STAGE_ATTACHMENTS | VK_PIPELINE_STAGE_2_TRANSFER_BIT; + +inline constexpr VkAccessFlags2 ACCESS_IMAGE_WRITES = + VK_ACCESS_2_SHADER_WRITE_BIT | VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT | + VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_2_TRANSFER_WRITE_BIT; + +inline constexpr VkAccessFlags2 ACCESS_IMAGE_USERS = + VK_ACCESS_2_SHADER_READ_BIT | VK_ACCESS_2_SHADER_WRITE_BIT | ACCESS_ATTACHMENTS | + ACCESS_TRANSFER; /// Table holding Vulkan instance function pointers. @@ -169,10 +206,8 @@ struct InstanceDispatch { PFN_vkEnumerateInstanceLayerProperties vkEnumerateInstanceLayerProperties{}; PFN_vkCreateDebugUtilsMessengerEXT vkCreateDebugUtilsMessengerEXT{}; - PFN_vkCreateDebugReportCallbackEXT vkCreateDebugReportCallbackEXT{}; PFN_vkCreateDevice vkCreateDevice{}; PFN_vkDestroyDebugUtilsMessengerEXT vkDestroyDebugUtilsMessengerEXT{}; - PFN_vkDestroyDebugReportCallbackEXT vkDestroyDebugReportCallbackEXT{}; PFN_vkDestroyDevice vkDestroyDevice{}; PFN_vkDestroySurfaceKHR vkDestroySurfaceKHR{}; PFN_vkEnumerateDeviceExtensionProperties vkEnumerateDeviceExtensionProperties{}; @@ -181,7 +216,6 @@ struct InstanceDispatch { PFN_vkGetPhysicalDeviceFeatures2 vkGetPhysicalDeviceFeatures2{}; PFN_vkGetPhysicalDeviceFormatProperties vkGetPhysicalDeviceFormatProperties{}; PFN_vkGetPhysicalDeviceFormatProperties2 vkGetPhysicalDeviceFormatProperties2{}; - PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties{}; PFN_vkGetPhysicalDeviceMemoryProperties2 vkGetPhysicalDeviceMemoryProperties2{}; PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties{}; PFN_vkGetPhysicalDeviceProperties2 vkGetPhysicalDeviceProperties2{}; @@ -201,27 +235,27 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkAllocateCommandBuffers vkAllocateCommandBuffers{}; PFN_vkAllocateDescriptorSets vkAllocateDescriptorSets{}; PFN_vkAllocateMemory vkAllocateMemory{}; - PFN_vkBeginCommandBuffer vkBeginCommandBuffer{}; PFN_vkBindBufferMemory vkBindBufferMemory{}; - PFN_vkBindImageMemory vkBindImageMemory{}; + PFN_vkFreeMemory vkFreeMemory{}; + PFN_vkBeginCommandBuffer vkBeginCommandBuffer{}; PFN_vkCmdBeginConditionalRenderingEXT vkCmdBeginConditionalRenderingEXT{}; - PFN_vkCmdBeginDebugUtilsLabelEXT vkCmdBeginDebugUtilsLabelEXT{}; PFN_vkCmdBeginQuery vkCmdBeginQuery{}; - PFN_vkCmdBeginRenderPass vkCmdBeginRenderPass{}; + PFN_vkCmdBeginRendering vkCmdBeginRendering{}; PFN_vkCmdBeginTransformFeedbackEXT vkCmdBeginTransformFeedbackEXT{}; PFN_vkCmdBindDescriptorSets vkCmdBindDescriptorSets{}; PFN_vkCmdBindIndexBuffer vkCmdBindIndexBuffer{}; + PFN_vkCmdBindIndexBuffer2KHR vkCmdBindIndexBuffer2KHR{}; PFN_vkCmdBindPipeline vkCmdBindPipeline{}; PFN_vkCmdBindTransformFeedbackBuffersEXT vkCmdBindTransformFeedbackBuffersEXT{}; PFN_vkCmdBindVertexBuffers vkCmdBindVertexBuffers{}; PFN_vkCmdBindVertexBuffers2EXT vkCmdBindVertexBuffers2EXT{}; - PFN_vkCmdBlitImage vkCmdBlitImage{}; + PFN_vkCmdBlitImage2 vkCmdBlitImage2{}; PFN_vkCmdClearAttachments vkCmdClearAttachments{}; PFN_vkCmdClearColorImage vkCmdClearColorImage{}; - PFN_vkCmdCopyBuffer vkCmdCopyBuffer{}; - PFN_vkCmdCopyBufferToImage vkCmdCopyBufferToImage{}; - PFN_vkCmdCopyImage vkCmdCopyImage{}; - PFN_vkCmdCopyImageToBuffer vkCmdCopyImageToBuffer{}; + PFN_vkCmdCopyBuffer2 vkCmdCopyBuffer2{}; + PFN_vkCmdCopyBufferToImage2 vkCmdCopyBufferToImage2{}; + PFN_vkCmdCopyImage2 vkCmdCopyImage2{}; + PFN_vkCmdCopyImageToBuffer2 vkCmdCopyImageToBuffer2{}; PFN_vkCmdCopyQueryPoolResults vkCmdCopyQueryPoolResults{}; PFN_vkCmdDispatch vkCmdDispatch{}; PFN_vkCmdDispatchIndirect vkCmdDispatchIndirect{}; @@ -233,17 +267,15 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdDrawIndexedIndirectCount vkCmdDrawIndexedIndirectCount{}; PFN_vkCmdDrawIndirectByteCountEXT vkCmdDrawIndirectByteCountEXT{}; PFN_vkCmdEndConditionalRenderingEXT vkCmdEndConditionalRenderingEXT{}; - PFN_vkCmdEndDebugUtilsLabelEXT vkCmdEndDebugUtilsLabelEXT{}; PFN_vkCmdEndQuery vkCmdEndQuery{}; - PFN_vkCmdEndRenderPass vkCmdEndRenderPass{}; + PFN_vkCmdEndRendering vkCmdEndRendering{}; PFN_vkCmdEndTransformFeedbackEXT vkCmdEndTransformFeedbackEXT{}; PFN_vkCmdFillBuffer vkCmdFillBuffer{}; - PFN_vkCmdPipelineBarrier vkCmdPipelineBarrier{}; PFN_vkCmdPipelineBarrier2 vkCmdPipelineBarrier2{}; PFN_vkCmdPushConstants vkCmdPushConstants{}; PFN_vkCmdPushDescriptorSetWithTemplateKHR vkCmdPushDescriptorSetWithTemplateKHR{}; PFN_vkCmdResetQueryPool vkCmdResetQueryPool{}; - PFN_vkCmdResolveImage vkCmdResolveImage{}; + PFN_vkCmdResolveImage2 vkCmdResolveImage2{}; PFN_vkCmdSetBlendConstants vkCmdSetBlendConstants{}; PFN_vkCmdSetCullModeEXT vkCmdSetCullModeEXT{}; PFN_vkCmdSetDepthBias vkCmdSetDepthBias{}; @@ -260,13 +292,10 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdSetConservativeRasterizationModeEXT vkCmdSetConservativeRasterizationModeEXT{}; PFN_vkCmdSetLineRasterizationModeEXT vkCmdSetLineRasterizationModeEXT{}; PFN_vkCmdSetLineStippleEnableEXT vkCmdSetLineStippleEnableEXT{}; - PFN_vkCmdSetLineStippleEXT vkCmdSetLineStippleEXT{}; PFN_vkCmdSetDepthBiasEnableEXT vkCmdSetDepthBiasEnableEXT{}; PFN_vkCmdSetLogicOpEnableEXT vkCmdSetLogicOpEnableEXT{}; PFN_vkCmdSetDepthClampEnableEXT vkCmdSetDepthClampEnableEXT{}; - PFN_vkCmdSetEvent vkCmdSetEvent{}; PFN_vkCmdSetFrontFaceEXT vkCmdSetFrontFaceEXT{}; - PFN_vkCmdSetPatchControlPointsEXT vkCmdSetPatchControlPointsEXT{}; PFN_vkCmdSetLogicOpEXT vkCmdSetLogicOpEXT{}; PFN_vkCmdSetLineWidth vkCmdSetLineWidth{}; PFN_vkCmdSetPrimitiveTopologyEXT vkCmdSetPrimitiveTopologyEXT{}; @@ -282,7 +311,6 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCmdSetColorWriteEnableEXT vkCmdSetColorWriteEnableEXT{}; PFN_vkCmdSetColorBlendEnableEXT vkCmdSetColorBlendEnableEXT{}; PFN_vkCmdSetColorBlendEquationEXT vkCmdSetColorBlendEquationEXT{}; - PFN_vkCmdWaitEvents vkCmdWaitEvents{}; PFN_vkCreateBuffer vkCreateBuffer{}; PFN_vkCreateBufferView vkCreateBufferView{}; PFN_vkCreateCommandPool vkCreateCommandPool{}; @@ -290,17 +318,12 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkCreateDescriptorPool vkCreateDescriptorPool{}; PFN_vkCreateDescriptorSetLayout vkCreateDescriptorSetLayout{}; PFN_vkCreateDescriptorUpdateTemplate vkCreateDescriptorUpdateTemplate{}; - PFN_vkCreateEvent vkCreateEvent{}; PFN_vkCreateFence vkCreateFence{}; - PFN_vkCreateFramebuffer vkCreateFramebuffer{}; PFN_vkCreateGraphicsPipelines vkCreateGraphicsPipelines{}; - PFN_vkCreateImage vkCreateImage{}; PFN_vkCreateImageView vkCreateImageView{}; PFN_vkCreatePipelineCache vkCreatePipelineCache{}; PFN_vkCreatePipelineLayout vkCreatePipelineLayout{}; PFN_vkCreateQueryPool vkCreateQueryPool{}; - PFN_vkCreateRenderPass vkCreateRenderPass{}; - PFN_vkCreateRenderPass2 vkCreateRenderPass2{}; PFN_vkCreateSampler vkCreateSampler{}; PFN_vkCreateSemaphore vkCreateSemaphore{}; PFN_vkCreateShaderModule vkCreateShaderModule{}; @@ -311,16 +334,13 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkDestroyDescriptorPool vkDestroyDescriptorPool{}; PFN_vkDestroyDescriptorSetLayout vkDestroyDescriptorSetLayout{}; PFN_vkDestroyDescriptorUpdateTemplate vkDestroyDescriptorUpdateTemplate{}; - PFN_vkDestroyEvent vkDestroyEvent{}; PFN_vkDestroyFence vkDestroyFence{}; - PFN_vkDestroyFramebuffer vkDestroyFramebuffer{}; PFN_vkDestroyImage vkDestroyImage{}; PFN_vkDestroyImageView vkDestroyImageView{}; PFN_vkDestroyPipeline vkDestroyPipeline{}; PFN_vkDestroyPipelineCache vkDestroyPipelineCache{}; PFN_vkDestroyPipelineLayout vkDestroyPipelineLayout{}; PFN_vkDestroyQueryPool vkDestroyQueryPool{}; - PFN_vkDestroyRenderPass vkDestroyRenderPass{}; PFN_vkDestroySampler vkDestroySampler{}; PFN_vkDestroySemaphore vkDestroySemaphore{}; PFN_vkDestroyShaderModule vkDestroyShaderModule{}; @@ -329,30 +349,22 @@ struct DeviceDispatch : InstanceDispatch { PFN_vkEndCommandBuffer vkEndCommandBuffer{}; PFN_vkFreeCommandBuffers vkFreeCommandBuffers{}; PFN_vkFreeDescriptorSets vkFreeDescriptorSets{}; - PFN_vkFreeMemory vkFreeMemory{}; PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2{}; + PFN_vkGetDeviceBufferMemoryRequirements vkGetDeviceBufferMemoryRequirements{}; PFN_vkGetDeviceQueue vkGetDeviceQueue{}; - PFN_vkGetEventStatus vkGetEventStatus{}; PFN_vkGetFenceStatus vkGetFenceStatus{}; - PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements{}; PFN_vkGetPipelineCacheData vkGetPipelineCacheData{}; - PFN_vkGetMemoryFdKHR vkGetMemoryFdKHR{}; -#ifdef _WIN32 - PFN_vkGetMemoryWin32HandleKHR vkGetMemoryWin32HandleKHR{}; -#endif PFN_vkGetPipelineExecutablePropertiesKHR vkGetPipelineExecutablePropertiesKHR{}; PFN_vkGetPipelineExecutableStatisticsKHR vkGetPipelineExecutableStatisticsKHR{}; PFN_vkGetQueryPoolResults vkGetQueryPoolResults{}; PFN_vkGetSemaphoreCounterValue vkGetSemaphoreCounterValue{}; - PFN_vkMapMemory vkMapMemory{}; + PFN_vkCopyMemoryToImageEXT vkCopyMemoryToImageEXT{}; + PFN_vkTransitionImageLayoutEXT vkTransitionImageLayoutEXT{}; PFN_vkQueueBindSparse vkQueueBindSparse{}; - PFN_vkQueueSubmit vkQueueSubmit{}; PFN_vkQueueSubmit2 vkQueueSubmit2{}; PFN_vkResetFences vkResetFences{}; PFN_vkResetQueryPool vkResetQueryPool{}; PFN_vkSetDebugUtilsObjectNameEXT vkSetDebugUtilsObjectNameEXT{}; - PFN_vkSetDebugUtilsObjectTagEXT vkSetDebugUtilsObjectTagEXT{}; - PFN_vkUnmapMemory vkUnmapMemory{}; PFN_vkUpdateDescriptorSetWithTemplate vkUpdateDescriptorSetWithTemplate{}; PFN_vkGetBufferDeviceAddress vkGetBufferDeviceAddress{}; PFN_vkGetDescriptorSetLayoutSizeEXT vkGetDescriptorSetLayoutSizeEXT{}; @@ -378,27 +390,23 @@ void Destroy(VkDevice, const InstanceDispatch&) noexcept; void Destroy(VkDevice, VkBuffer, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkBufferView, const DeviceDispatch&) noexcept; +void Destroy(VkDevice, VkDeviceMemory, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkCommandPool, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkDescriptorPool, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkDescriptorSetLayout, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkDescriptorUpdateTemplate, const DeviceDispatch&) noexcept; -void Destroy(VkDevice, VkDeviceMemory, const DeviceDispatch&) noexcept; -void Destroy(VkDevice, VkEvent, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkFence, const DeviceDispatch&) noexcept; -void Destroy(VkDevice, VkFramebuffer, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkImage, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkImageView, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkPipeline, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkPipelineCache, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkPipelineLayout, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkQueryPool, const DeviceDispatch&) noexcept; -void Destroy(VkDevice, VkRenderPass, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkSampler, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkSwapchainKHR, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkSemaphore, const DeviceDispatch&) noexcept; void Destroy(VkDevice, VkShaderModule, const DeviceDispatch&) noexcept; void Destroy(VkInstance, VkDebugUtilsMessengerEXT, const InstanceDispatch&) noexcept; -void Destroy(VkInstance, VkDebugReportCallbackEXT, const InstanceDispatch&) noexcept; void Destroy(VkInstance, VkSurfaceKHR, const InstanceDispatch&) noexcept; VkResult Free(VkDevice, VkDescriptorPool, Span, const DeviceDispatch&) noexcept; @@ -636,14 +644,14 @@ private: }; using DebugUtilsMessenger = Handle; -using DebugReportCallback = Handle; using DescriptorSetLayout = Handle; using DescriptorUpdateTemplate = Handle; using Pipeline = Handle; using PipelineLayout = Handle; using QueryPool = Handle; -using RenderPass = Handle; using Sampler = Handle; +using DeviceMemory = Handle; +using ExternalBuffer = Handle; using SurfaceKHR = Handle; using DescriptorSets = PoolAllocations; @@ -669,11 +677,6 @@ public: [[nodiscard]] DebugUtilsMessenger CreateDebugUtilsMessenger( const VkDebugUtilsMessengerCreateInfoEXT& create_info) const; - /// Creates a debug report callback. - /// @throw Exception on creation failure. - [[nodiscard]] DebugReportCallback CreateDebugReportCallback( - const VkDebugReportCallbackCreateInfoEXT& create_info) const; - /// Returns dispatch table. const InstanceDispatch& Dispatch() const noexcept { return *dld; @@ -846,13 +849,7 @@ public: constexpr Queue(VkQueue queue_, const DeviceDispatch& dld_) noexcept : queue{queue_}, dld{&dld_} {} - VkResult Submit(Span submit_infos, - VkFence fence = VK_NULL_HANDLE) const noexcept { - return dld->vkQueueSubmit(queue, submit_infos.size(), submit_infos.data(), fence); - } - - /// Submits using VK_KHR_synchronization2 / Vulkan 1.3 vkQueueSubmit2. - VkResult Submit2(Span submit_infos, + VkResult Submit(Span submit_infos, VkFence fence = VK_NULL_HANDLE) const noexcept { return dld->vkQueueSubmit2(queue, submit_infos.size(), submit_infos.data(), fence); } @@ -887,30 +884,6 @@ public: void SetObjectNameEXT(const char* name) const; }; -class DeviceMemory : public Handle { - using Handle::Handle; - -public: - int GetMemoryFdKHR() const; - -#ifdef _WIN32 - HANDLE GetMemoryWin32HandleKHR() const; -#endif - - /// Set object name. - void SetObjectNameEXT(const char* name) const; - - u8* Map(VkDeviceSize offset, VkDeviceSize size) const { - void* data; - Check(dld->vkMapMemory(owner, handle, offset, size, 0, &data)); - return static_cast(data); - } - - void Unmap() const noexcept { - dld->vkUnmapMemory(owner, handle); - } -}; - class Fence : public Handle { using Handle::Handle; @@ -931,14 +904,6 @@ public: } }; -class Framebuffer : public Handle { - using Handle::Handle; - -public: - /// Set object name. - void SetObjectNameEXT(const char* name) const; -}; - class DescriptorPool : public Handle { using Handle::Handle; @@ -967,18 +932,6 @@ public: std::vector GetImages() const; }; -class Event : public Handle { - using Handle::Handle; - -public: - /// Set object name. - void SetObjectNameEXT(const char* name) const; - - VkResult GetStatus() const noexcept { - return dld->vkGetEventStatus(owner, handle); - } -}; - class ShaderModule : public Handle { using Handle::Handle; @@ -1053,6 +1006,14 @@ public: [[nodiscard]] BufferView CreateBufferView(const VkBufferViewCreateInfo& ci) const; + [[nodiscard]] ExternalBuffer CreateExternalBuffer(const VkBufferCreateInfo& ci) const; + + [[nodiscard]] DeviceMemory AllocateMemory(const VkMemoryAllocateInfo& ai) const; + + void BindBufferMemory(VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) const; + + [[nodiscard]] VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer) const noexcept; + [[nodiscard]] ImageView CreateImageView(const VkImageViewCreateInfo& ci) const; [[nodiscard]] Semaphore CreateSemaphore() const; @@ -1063,10 +1024,6 @@ public: [[nodiscard]] DescriptorPool CreateDescriptorPool(const VkDescriptorPoolCreateInfo& ci) const; - [[nodiscard]] RenderPass CreateRenderPass(const VkRenderPassCreateInfo& ci) const; - - [[nodiscard]] RenderPass CreateRenderPass2(const VkRenderPassCreateInfo2& ci) const; - [[nodiscard]] DescriptorSetLayout CreateDescriptorSetLayout( const VkDescriptorSetLayoutCreateInfo& ci) const; @@ -1082,8 +1039,6 @@ public: [[nodiscard]] Sampler CreateSampler(const VkSamplerCreateInfo& ci) const; - [[nodiscard]] Framebuffer CreateFramebuffer(const VkFramebufferCreateInfo& ci) const; - [[nodiscard]] CommandPool CreateCommandPool(const VkCommandPoolCreateInfo& ci) const; [[nodiscard]] DescriptorUpdateTemplate CreateDescriptorUpdateTemplate( @@ -1093,18 +1048,9 @@ public: [[nodiscard]] ShaderModule CreateShaderModule(const VkShaderModuleCreateInfo& ci) const; - [[nodiscard]] Event CreateEvent() const; - [[nodiscard]] SwapchainKHR CreateSwapchainKHR(const VkSwapchainCreateInfoKHR& ci) const; - [[nodiscard]] DeviceMemory TryAllocateMemory(const VkMemoryAllocateInfo& ai) const noexcept; - - [[nodiscard]] DeviceMemory AllocateMemory(const VkMemoryAllocateInfo& ai) const; - - VkMemoryRequirements GetBufferMemoryRequirements(VkBuffer buffer, - void* pnext = nullptr) const noexcept; - - VkMemoryRequirements GetImageMemoryRequirements(VkImage image) const noexcept; + VkMemoryRequirements GetDeviceBufferMemoryRequirements(const VkBufferCreateInfo& ci) const noexcept; std::vector GetPipelineExecutablePropertiesKHR( VkPipeline pipeline) const; @@ -1162,6 +1108,14 @@ public: dld->vkResetQueryPool(handle, query_pool, first, count); } + void CopyMemoryToImageEXT(const VkCopyMemoryToImageInfoEXT& info) const { + Check(dld->vkCopyMemoryToImageEXT(handle, &info)); + } + + void TransitionImageLayoutEXT(const VkHostImageLayoutTransitionInfoEXT& info) const { + Check(dld->vkTransitionImageLayoutEXT(handle, 1, &info)); + } + VkResult GetQueryResults(VkQueryPool query_pool, u32 first, u32 count, std::size_t data_size, void* data, VkDeviceSize stride, VkQueryResultFlags flags) const noexcept { @@ -1186,8 +1140,6 @@ public: void GetProperties2(VkPhysicalDeviceProperties2&) const noexcept; - VkPhysicalDeviceFeatures GetFeatures() const noexcept; - void GetFeatures2(VkPhysicalDeviceFeatures2&) const noexcept; VkFormatProperties GetFormatProperties(VkFormat) const noexcept; @@ -1238,13 +1190,12 @@ public: Check(dld->vkEndCommandBuffer(handle)); } - void BeginRenderPass(const VkRenderPassBeginInfo& renderpass_bi, - VkSubpassContents contents) const noexcept { - dld->vkCmdBeginRenderPass(handle, &renderpass_bi, contents); + void BeginRendering(const VkRenderingInfo& rendering_info) const noexcept { + dld->vkCmdBeginRendering(handle, &rendering_info); } - void EndRenderPass() const noexcept { - dld->vkCmdEndRenderPass(handle); + void EndRendering() const noexcept { + dld->vkCmdEndRendering(handle); } void BeginQuery(VkQueryPool query_pool, u32 query, VkQueryControlFlags flags) const noexcept { @@ -1280,6 +1231,11 @@ public: dld->vkCmdBindIndexBuffer(handle, buffer, offset, index_type); } + void BindIndexBuffer2KHR(VkBuffer buffer, VkDeviceSize offset, VkDeviceSize size, + VkIndexType index_type) const noexcept { + dld->vkCmdBindIndexBuffer2KHR(handle, buffer, offset, size, index_type); + } + void BindVertexBuffers(u32 first, u32 count, const VkBuffer* buffers, const VkDeviceSize* offsets) const noexcept { dld->vkCmdBindVertexBuffers(handle, first, count, buffers, offsets); @@ -1342,16 +1298,35 @@ public: } void BlitImage(VkImage src_image, VkImageLayout src_layout, VkImage dst_image, - VkImageLayout dst_layout, Span regions, + VkImageLayout dst_layout, Span regions, VkFilter filter) const noexcept { - dld->vkCmdBlitImage(handle, src_image, src_layout, dst_image, dst_layout, regions.size(), - regions.data(), filter); + const VkBlitImageInfo2 info{ + .sType = VK_STRUCTURE_TYPE_BLIT_IMAGE_INFO_2, + .pNext = nullptr, + .srcImage = src_image, + .srcImageLayout = src_layout, + .dstImage = dst_image, + .dstImageLayout = dst_layout, + .regionCount = regions.size(), + .pRegions = regions.data(), + .filter = filter, + }; + dld->vkCmdBlitImage2(handle, &info); } void ResolveImage(VkImage src_image, VkImageLayout src_layout, VkImage dst_image, - VkImageLayout dst_layout, Span regions) { - dld->vkCmdResolveImage(handle, src_image, src_layout, dst_image, dst_layout, regions.size(), - regions.data()); + VkImageLayout dst_layout, Span regions) const noexcept { + const VkResolveImageInfo2 info{ + .sType = VK_STRUCTURE_TYPE_RESOLVE_IMAGE_INFO_2, + .pNext = nullptr, + .srcImage = src_image, + .srcImageLayout = src_layout, + .dstImage = dst_image, + .dstImageLayout = dst_layout, + .regionCount = regions.size(), + .pRegions = regions.data(), + }; + dld->vkCmdResolveImage2(handle, &info); } void Dispatch(u32 x, u32 y, u32 z) const noexcept { @@ -1362,103 +1337,50 @@ public: dld->vkCmdDispatchIndirect(handle, indirect_buffer, offset); } - void PipelineBarrier(VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dst_stage_mask, - VkDependencyFlags dependency_flags, Span memory_barriers, - Span buffer_barriers, - Span image_barriers) const noexcept { - static constexpr u32 MaxBarriers = 16; - if (dld->vkCmdPipelineBarrier2 && memory_barriers.size() <= MaxBarriers && - buffer_barriers.size() <= MaxBarriers && image_barriers.size() <= MaxBarriers) { - const auto src_stage_mask2 = static_cast(src_stage_mask); - const auto dst_stage_mask2 = static_cast(dst_stage_mask); - - std::array memory_barriers2; - for (u32 i = 0; i < memory_barriers.size(); ++i) { - memory_barriers2[i] = VkMemoryBarrier2{ - .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, - .pNext = nullptr, - .srcStageMask = src_stage_mask2, - .srcAccessMask = static_cast(memory_barriers[i].srcAccessMask), - .dstStageMask = dst_stage_mask2, - .dstAccessMask = static_cast(memory_barriers[i].dstAccessMask), - }; - } - std::array buffer_barriers2; - for (u32 i = 0; i < buffer_barriers.size(); ++i) { - const auto& barrier = buffer_barriers[i]; - buffer_barriers2[i] = VkBufferMemoryBarrier2{ - .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2, - .pNext = nullptr, - .srcStageMask = src_stage_mask2, - .srcAccessMask = static_cast(barrier.srcAccessMask), - .dstStageMask = dst_stage_mask2, - .dstAccessMask = static_cast(barrier.dstAccessMask), - .srcQueueFamilyIndex = barrier.srcQueueFamilyIndex, - .dstQueueFamilyIndex = barrier.dstQueueFamilyIndex, - .buffer = barrier.buffer, - .offset = barrier.offset, - .size = barrier.size, - }; - } - std::array image_barriers2; - for (u32 i = 0; i < image_barriers.size(); ++i) { - const auto& barrier = image_barriers[i]; - image_barriers2[i] = VkImageMemoryBarrier2{ - .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, - .pNext = nullptr, - .srcStageMask = src_stage_mask2, - .srcAccessMask = static_cast(barrier.srcAccessMask), - .dstStageMask = dst_stage_mask2, - .dstAccessMask = static_cast(barrier.dstAccessMask), - .oldLayout = barrier.oldLayout, - .newLayout = barrier.newLayout, - .srcQueueFamilyIndex = barrier.srcQueueFamilyIndex, - .dstQueueFamilyIndex = barrier.dstQueueFamilyIndex, - .image = barrier.image, - .subresourceRange = barrier.subresourceRange, - }; - } - const VkDependencyInfo dependency_info{ - .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, - .pNext = nullptr, - .dependencyFlags = dependency_flags, - .memoryBarrierCount = memory_barriers.size(), - .pMemoryBarriers = memory_barriers2.data(), - .bufferMemoryBarrierCount = buffer_barriers.size(), - .pBufferMemoryBarriers = buffer_barriers2.data(), - .imageMemoryBarrierCount = image_barriers.size(), - .pImageMemoryBarriers = image_barriers2.data(), - }; - dld->vkCmdPipelineBarrier2(handle, &dependency_info); - return; - } - dld->vkCmdPipelineBarrier(handle, src_stage_mask, dst_stage_mask, dependency_flags, - memory_barriers.size(), memory_barriers.data(), - buffer_barriers.size(), buffer_barriers.data(), - image_barriers.size(), image_barriers.data()); + void PipelineBarrier(VkDependencyFlags dependency_flags, + Span memory_barriers, + Span buffer_barriers, + Span image_barriers) const noexcept { + const VkDependencyInfo dependency_info{ + .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, + .pNext = nullptr, + .dependencyFlags = dependency_flags, + .memoryBarrierCount = memory_barriers.size(), + .pMemoryBarriers = memory_barriers.data(), + .bufferMemoryBarrierCount = buffer_barriers.size(), + .pBufferMemoryBarriers = buffer_barriers.data(), + .imageMemoryBarrierCount = image_barriers.size(), + .pImageMemoryBarriers = image_barriers.data(), + }; + dld->vkCmdPipelineBarrier2(handle, &dependency_info); } - void PipelineBarrier(VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dst_stage_mask, + void PipelineBarrier(VkPipelineStageFlags2 src_stage_mask, VkPipelineStageFlags2 dst_stage_mask, VkDependencyFlags dependency_flags = 0) const noexcept { - PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, {}, {}, {}); + const VkMemoryBarrier2 barrier{ + .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2, + .pNext = nullptr, + .srcStageMask = src_stage_mask, + .srcAccessMask = VK_ACCESS_2_NONE, + .dstStageMask = dst_stage_mask, + .dstAccessMask = VK_ACCESS_2_NONE, + }; + PipelineBarrier(dependency_flags, barrier, {}, {}); } - void PipelineBarrier(VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dst_stage_mask, - VkDependencyFlags dependency_flags, - const VkMemoryBarrier& memory_barrier) const noexcept { - PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, memory_barrier, {}, {}); + void PipelineBarrier(const VkMemoryBarrier2& memory_barrier, + VkDependencyFlags dependency_flags = 0) const noexcept { + PipelineBarrier(dependency_flags, memory_barrier, {}, {}); } - void PipelineBarrier(VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dst_stage_mask, - VkDependencyFlags dependency_flags, - const VkBufferMemoryBarrier& buffer_barrier) const noexcept { - PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, {}, buffer_barrier, {}); + void PipelineBarrier(const VkBufferMemoryBarrier2& buffer_barrier, + VkDependencyFlags dependency_flags = 0) const noexcept { + PipelineBarrier(dependency_flags, {}, buffer_barrier, {}); } - void PipelineBarrier(VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dst_stage_mask, - VkDependencyFlags dependency_flags, - const VkImageMemoryBarrier& image_barrier) const noexcept { - PipelineBarrier(src_stage_mask, dst_stage_mask, dependency_flags, {}, {}, image_barrier); + void PipelineBarrier(const VkImageMemoryBarrier2& image_barrier, + VkDependencyFlags dependency_flags = 0) const noexcept { + PipelineBarrier(dependency_flags, {}, {}, image_barrier); } void BindDescriptorBuffersEXT(Span bindings) const noexcept { @@ -1474,26 +1396,59 @@ public: } void CopyBufferToImage(VkBuffer src_buffer, VkImage dst_image, VkImageLayout dst_image_layout, - Span regions) const noexcept { - dld->vkCmdCopyBufferToImage(handle, src_buffer, dst_image, dst_image_layout, regions.size(), - regions.data()); + Span regions) const noexcept { + const VkCopyBufferToImageInfo2 info{ + .sType = VK_STRUCTURE_TYPE_COPY_BUFFER_TO_IMAGE_INFO_2, + .pNext = nullptr, + .srcBuffer = src_buffer, + .dstImage = dst_image, + .dstImageLayout = dst_image_layout, + .regionCount = regions.size(), + .pRegions = regions.data(), + }; + dld->vkCmdCopyBufferToImage2(handle, &info); } void CopyBuffer(VkBuffer src_buffer, VkBuffer dst_buffer, - Span regions) const noexcept { - dld->vkCmdCopyBuffer(handle, src_buffer, dst_buffer, regions.size(), regions.data()); + Span regions) const noexcept { + const VkCopyBufferInfo2 info{ + .sType = VK_STRUCTURE_TYPE_COPY_BUFFER_INFO_2, + .pNext = nullptr, + .srcBuffer = src_buffer, + .dstBuffer = dst_buffer, + .regionCount = regions.size(), + .pRegions = regions.data(), + }; + dld->vkCmdCopyBuffer2(handle, &info); } void CopyImage(VkImage src_image, VkImageLayout src_layout, VkImage dst_image, - VkImageLayout dst_layout, Span regions) const noexcept { - dld->vkCmdCopyImage(handle, src_image, src_layout, dst_image, dst_layout, regions.size(), - regions.data()); + VkImageLayout dst_layout, Span regions) const noexcept { + const VkCopyImageInfo2 info{ + .sType = VK_STRUCTURE_TYPE_COPY_IMAGE_INFO_2, + .pNext = nullptr, + .srcImage = src_image, + .srcImageLayout = src_layout, + .dstImage = dst_image, + .dstImageLayout = dst_layout, + .regionCount = regions.size(), + .pRegions = regions.data(), + }; + dld->vkCmdCopyImage2(handle, &info); } void CopyImageToBuffer(VkImage src_image, VkImageLayout src_layout, VkBuffer dst_buffer, - Span regions) const noexcept { - dld->vkCmdCopyImageToBuffer(handle, src_image, src_layout, dst_buffer, regions.size(), - regions.data()); + Span regions) const noexcept { + const VkCopyImageToBufferInfo2 info{ + .sType = VK_STRUCTURE_TYPE_COPY_IMAGE_TO_BUFFER_INFO_2, + .pNext = nullptr, + .srcImage = src_image, + .srcImageLayout = src_layout, + .dstBuffer = dst_buffer, + .regionCount = regions.size(), + .pRegions = regions.data(), + }; + dld->vkCmdCopyImageToBuffer2(handle, &info); } void CopyQueryPoolResults(VkQueryPool query_pool, u32 first_query, u32 query_count, @@ -1563,19 +1518,6 @@ public: dld->vkCmdSetDepthBounds(handle, min_depth_bounds, max_depth_bounds); } - void SetEvent(VkEvent event, VkPipelineStageFlags stage_flags) const noexcept { - dld->vkCmdSetEvent(handle, event, stage_flags); - } - - void WaitEvents(Span events, VkPipelineStageFlags src_stage_mask, - VkPipelineStageFlags dst_stage_mask, Span memory_barriers, - Span buffer_barriers, - Span image_barriers) const noexcept { - dld->vkCmdWaitEvents(handle, events.size(), events.data(), src_stage_mask, dst_stage_mask, - memory_barriers.size(), memory_barriers.data(), buffer_barriers.size(), - buffer_barriers.data(), image_barriers.size(), image_barriers.data()); - } - void BindVertexBuffers2EXT(u32 first_binding, u32 binding_count, const VkBuffer* buffers, const VkDeviceSize* offsets, const VkDeviceSize* sizes, const VkDeviceSize* strides) const noexcept { @@ -1626,11 +1568,6 @@ public: dld->vkCmdSetLineStippleEnableEXT(handle, enable ? VK_TRUE : VK_FALSE); } - void SetLineStippleEXT(u32 factor, u16 pattern) const noexcept - { - dld->vkCmdSetLineStippleEXT(handle, factor, pattern); - } - void SetDepthBiasEnableEXT(bool enable) const noexcept { dld->vkCmdSetDepthBiasEnableEXT(handle, enable ? VK_TRUE : VK_FALSE); } @@ -1659,10 +1596,6 @@ public: dld->vkCmdSetLogicOpEXT(handle, logic_op); } - void SetPatchControlPointsEXT(uint32_t patch_control_points) const noexcept { - dld->vkCmdSetPatchControlPointsEXT(handle, patch_control_points); - } - void SetColorWriteMaskEXT(u32 first, Span masks) const noexcept { dld->vkCmdSetColorWriteMaskEXT(handle, first, masks.size(), masks.data()); } @@ -1733,20 +1666,6 @@ public: dld->vkCmdEndConditionalRenderingEXT(handle); } - void BeginDebugUtilsLabelEXT(const char* label, std::span color) const noexcept { - const VkDebugUtilsLabelEXT label_info{ - .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT, - .pNext = nullptr, - .pLabelName = label, - .color{color[0], color[1], color[2], color[3]}, - }; - dld->vkCmdBeginDebugUtilsLabelEXT(handle, &label_info); - } - - void EndDebugUtilsLabelEXT() const noexcept { - dld->vkCmdEndDebugUtilsLabelEXT(handle); - } - private: VkCommandBuffer handle; const DeviceDispatch* dld; diff --git a/src/yuzu/bootmanager.cpp b/src/yuzu/bootmanager.cpp index 308db3621b..7affda9c56 100644 --- a/src/yuzu/bootmanager.cpp +++ b/src/yuzu/bootmanager.cpp @@ -157,7 +157,7 @@ GRenderWindow::~GRenderWindow() { input_subsystem->Shutdown(); } -void GRenderWindow::OnFrameDisplayed() { +void GRenderWindow::OnFrameDisplayed([[maybe_unused]] u32 presented_frames) { input_subsystem->GetTas()->UpdateThread(); const InputCommon::TasInput::TasState new_tas_state = std::get<0>(input_subsystem->GetTas()->GetStatus()); diff --git a/src/yuzu/bootmanager.h b/src/yuzu/bootmanager.h index bb18bbb3bb..85218f4cb4 100644 --- a/src/yuzu/bootmanager.h +++ b/src/yuzu/bootmanager.h @@ -67,7 +67,7 @@ public: ~GRenderWindow() override; // EmuWindow implementation. - void OnFrameDisplayed() override; + void OnFrameDisplayed(u32 presented_frames) override; bool IsShown() const override; std::unique_ptr CreateSharedContext() const override; diff --git a/src/yuzu/configuration/configure_graphics.cpp b/src/yuzu/configuration/configure_graphics.cpp index 641f65d82a..15b24c5a1b 100644 --- a/src/yuzu/configuration/configure_graphics.cpp +++ b/src/yuzu/configuration/configure_graphics.cpp @@ -41,6 +41,37 @@ #include "yuzu/configuration/configure_post_processing.h" #endif +#ifdef HAS_LSFG +#include +#include +#include +#include +#include +#include +#include "common/string_util.h" +#include "video_core/frame_gen/lossless_dll.h" + +namespace { + +QString LosslessSearchDirectory() { + const QString suffix = QStringLiteral("/steamapps/common/Lossless Scaling"); + const std::array roots{ + QDir::homePath() + QStringLiteral("/.local/share/Steam"), + QDir::homePath() + QStringLiteral("/.steam/steam"), + QDir::homePath() + QStringLiteral("/.var/app/com.valvesoftware.Steam/.local/share/Steam"), + QStringLiteral("C:/Program Files (x86)/Steam"), + }; + for (const QString& root : roots) { + if (QDir(root + suffix).exists()) { + return root + suffix; + } + } + return QDir::homePath(); +} + +} // Anonymous namespace +#endif + ConfigureGraphics::ConfigureGraphics( const Core::System& system_, std::vector& records_, const std::function& expose_compute_option_, @@ -322,11 +353,27 @@ void ConfigureGraphics::Setup(const ConfigurationShared::Builder& builder) { hold_graphics.emplace(setting->Id(), widget); #endif +#ifdef HAS_LSFG + } else if (setting->Id() == Settings::values.frame_gen.Id()) { + SetupFrameGen(widget); + hold_graphics.emplace(setting->Id(), widget); + } else if (setting->PairedSetting() == &Settings::values.frame_gen) { + frame_gen_widgets.push_back(widget); + hold_graphics.emplace(setting->Id(), widget); +#endif } else { hold_graphics.emplace(setting->Id(), widget); } } +#ifdef HAS_LSFG + if (frame_gen_checkbox != nullptr) { + for (QWidget* frame_gen_widget : frame_gen_widgets) { + frame_gen_widget->setVisible(frame_gen_checkbox->isChecked()); + } + } +#endif + for (const auto& [id, widget] : hold_graphics) { graphics_layout.addWidget(widget); } @@ -511,3 +558,84 @@ Settings::RendererBackend ConfigureGraphics::GetCurrentGraphicsBackend() const { return Settings::RendererBackend::OpenGL_GLSL; return selected_backend; } + +#ifdef HAS_LSFG +void ConfigureGraphics::SetupFrameGen(ConfigurationShared::Widget* widget) { + frame_gen_checkbox = widget->checkbox; + lossless_button = new QPushButton(widget); + lossless_button->setEnabled(!system.IsPoweredOn()); + UpdateLosslessButton(); + + connect(lossless_button, &QAbstractButton::clicked, this, [this] { InstallLosslessDll(); }); + connect(frame_gen_checkbox, &QCheckBox::toggled, this, [this](bool checked) { + for (QWidget* frame_gen_widget : frame_gen_widgets) { + frame_gen_widget->setVisible(checked); + } + }); + + QBoxLayout* row = qobject_cast(widget->layout()); + row->insertWidget(1, lossless_button); +} + +void ConfigureGraphics::UpdateLosslessButton() { + using VideoCore::FrameGen::LosslessStatus; + if (VideoCore::FrameGen::GetInstalledLosslessStatus() == LosslessStatus::Ok) { + lossless_button->setText(tr("Replace Lossless.dll...")); + } else { + lossless_button->setText(tr("Install Lossless.dll...")); + } +} + +void ConfigureGraphics::InstallLosslessDll() { + using VideoCore::FrameGen::LosslessStatus; + const auto status_text = [](LosslessStatus status) { + switch (status) { + case LosslessStatus::NotPortableExecutable: + return tr("The selected file is not a Windows library. Select Lossless.dll from your " + "Lossless Scaling installation."); + case LosslessStatus::MissingShaders: + return tr("This copy of Lossless.dll does not contain the frame generation shaders. " + "Update Lossless Scaling and try again."); + case LosslessStatus::TranslationFailed: + return tr("The frame generation shaders could not be translated. This version of " + "Lossless Scaling is not supported yet."); + case LosslessStatus::CacheUnusable: + return tr("The translated shaders could not be written to storage. Check that there " + "is free space available."); + default: + return tr("The selected file could not be read."); + } + }; + + const QString file = + QFileDialog::getOpenFileName(this, tr("Select Lossless.dll"), LosslessSearchDirectory(), + tr("Lossless Scaling library (Lossless.dll)")); + if (file.isEmpty()) { + return; + } + + const std::filesystem::path source{Common::U16StringFromBuffer(file.utf16(), file.size())}; + LosslessStatus status = VideoCore::FrameGen::ValidateLosslessDll(source); + if (status == LosslessStatus::Ok) { + const std::filesystem::path target = VideoCore::FrameGen::GetLosslessDllPath(); + std::error_code error; + std::filesystem::create_directories(target.parent_path(), error); + std::filesystem::copy_file(source, target, + std::filesystem::copy_options::overwrite_existing, error); + if (error) { + QMessageBox::warning(this, tr("Could not install Lossless.dll"), + tr("The selected file could not be copied.")); + return; + } + status = VideoCore::FrameGen::BuildShaderCache(); + } + + UpdateLosslessButton(); + if (status == LosslessStatus::Ok) { + QMessageBox::information(this, tr("Lossless Scaling"), + tr("Lossless.dll installed successfully.")); + return; + } + QMessageBox::warning(this, tr("Could not install Lossless.dll"), status_text(status)); +} +#endif diff --git a/src/yuzu/configuration/configure_graphics.h b/src/yuzu/configuration/configure_graphics.h index 9002af437b..95fe1995cb 100644 --- a/src/yuzu/configuration/configure_graphics.h +++ b/src/yuzu/configuration/configure_graphics.h @@ -22,6 +22,7 @@ #include "qt_common/util/vk.h" #include "yuzu/configuration/configuration_shared.h" +class QCheckBox; class QPushButton; class QEvent; class QObject; @@ -42,6 +43,7 @@ class ConfigureGraphics; namespace ConfigurationShared { class Builder; +class Widget; } class ConfigureGraphics : public ConfigurationShared::Tab { @@ -83,6 +85,12 @@ private: int FindIndex(u32 enumeration, int value) const; +#ifdef HAS_LSFG + void SetupFrameGen(ConfigurationShared::Widget* widget); + void InstallLosslessDll(); + void UpdateLosslessButton(); +#endif + std::unique_ptr ui; QColor bg_color; @@ -110,4 +118,9 @@ private: QWidget* api_widget; QComboBox* aspect_ratio_combobox; QComboBox* resolution_combobox; +#ifdef HAS_LSFG + QCheckBox* frame_gen_checkbox{}; + QPushButton* lossless_button{}; + std::vector frame_gen_widgets; +#endif }; diff --git a/src/yuzu/configuration/shared_widget.cpp b/src/yuzu/configuration/shared_widget.cpp index 0f59ee23a6..5dd6094b96 100644 --- a/src/yuzu/configuration/shared_widget.cpp +++ b/src/yuzu/configuration/shared_widget.cpp @@ -511,7 +511,12 @@ void Widget::SetupComponent(const QString& label, std::function& load_fu layout->setContentsMargins(0, 0, 0, 0); if (other_setting == nullptr) { - other_setting = setting.PairedSetting(); + Settings::BasicSetting* const paired = setting.PairedSetting(); + const bool paired_standalone = paired != nullptr && translations.contains(paired->Id()) && + !translations.at(paired->Id()).first.isEmpty(); + if (!paired_standalone) { + other_setting = paired; + } } const bool require_checkbox = other_setting != nullptr && other_setting->TypeId() == "bool"; @@ -774,7 +779,12 @@ Builder::~Builder() = default; static bool IsAndroidOnly(const Settings::BasicSetting& setting) { const std::string& label = setting.GetLabel(); - return label.starts_with("frame_gen") || label == "emulate_bgr565"; +#ifndef HAS_LSFG + if (label.starts_with("frame_gen")) { + return true; + } +#endif + return label == "emulate_bgr565"; } Widget* Builder::BuildWidget(Settings::BasicSetting* setting,