Query Cachge: Fully rework Vulkan's query cache

This commit is contained in:
Fernando Sahmkow 2023-08-04 03:32:30 +02:00
commit f1a2e36711
35 changed files with 1555 additions and 337 deletions

View file

@ -130,13 +130,17 @@ void LogSettings() {
log_path("DataStorage_SDMCDir", Common::FS::GetYuzuPath(Common::FS::YuzuPath::SDMCDir));
}
void UpdateGPUAccuracy() {
values.current_gpu_accuracy = values.gpu_accuracy.GetValue();
}
bool IsGPULevelExtreme() {
return values.gpu_accuracy.GetValue() == GpuAccuracy::Extreme;
return values.current_gpu_accuracy == GpuAccuracy::Extreme;
}
bool IsGPULevelHigh() {
return values.gpu_accuracy.GetValue() == GpuAccuracy::Extreme ||
values.gpu_accuracy.GetValue() == GpuAccuracy::High;
return values.current_gpu_accuracy == GpuAccuracy::Extreme ||
values.current_gpu_accuracy == GpuAccuracy::High;
}
bool IsFastmemEnabled() {

View file

@ -307,6 +307,7 @@ struct Values {
Specialization::Default,
true,
true};
GpuAccuracy current_gpu_accuracy{GpuAccuracy::High};
SwitchableSetting<AnisotropyMode, true> max_anisotropy{
linkage, AnisotropyMode::Automatic, AnisotropyMode::Automatic, AnisotropyMode::X16,
"max_anisotropy", Category::RendererAdvanced};
@ -522,6 +523,7 @@ struct Values {
extern Values values;
void UpdateGPUAccuracy();
bool IsGPULevelExtreme();
bool IsGPULevelHigh();

View file

@ -272,13 +272,20 @@ std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainBuffer(GPUVAddr gpu_ad
if (!cpu_addr) {
return {&slot_buffers[NULL_BUFFER_ID], 0};
}
const BufferId buffer_id = FindBuffer(*cpu_addr, size);
return ObtainCPUBuffer(*cpu_addr, size, sync_info, post_op);
}
template <class P>
std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainCPUBuffer(VAddr cpu_addr, u32 size,
ObtainBufferSynchronize sync_info,
ObtainBufferOperation post_op) {
const BufferId buffer_id = FindBuffer(cpu_addr, size);
Buffer& buffer = slot_buffers[buffer_id];
// synchronize op
switch (sync_info) {
case ObtainBufferSynchronize::FullSynchronize:
SynchronizeBuffer(buffer, *cpu_addr, size);
SynchronizeBuffer(buffer, cpu_addr, size);
break;
default:
break;
@ -286,11 +293,11 @@ std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainBuffer(GPUVAddr gpu_ad
switch (post_op) {
case ObtainBufferOperation::MarkAsWritten:
MarkWrittenBuffer(buffer_id, *cpu_addr, size);
MarkWrittenBuffer(buffer_id, cpu_addr, size);
break;
case ObtainBufferOperation::DiscardWrite: {
VAddr cpu_addr_start = Common::AlignDown(*cpu_addr, 64);
VAddr cpu_addr_end = Common::AlignUp(*cpu_addr + size, 64);
VAddr cpu_addr_start = Common::AlignDown(cpu_addr, 64);
VAddr cpu_addr_end = Common::AlignUp(cpu_addr + size, 64);
IntervalType interval{cpu_addr_start, cpu_addr_end};
ClearDownload(interval);
common_ranges.subtract(interval);
@ -300,7 +307,7 @@ std::pair<typename P::Buffer*, u32> BufferCache<P>::ObtainBuffer(GPUVAddr gpu_ad
break;
}
return {&buffer, buffer.Offset(*cpu_addr)};
return {&buffer, buffer.Offset(cpu_addr)};
}
template <class P>

View file

@ -295,6 +295,10 @@ public:
[[nodiscard]] std::pair<Buffer*, u32> ObtainBuffer(GPUVAddr gpu_addr, u32 size,
ObtainBufferSynchronize sync_info,
ObtainBufferOperation post_op);
[[nodiscard]] std::pair<Buffer*, u32> ObtainCPUBuffer(VAddr gpu_addr, u32 size,
ObtainBufferSynchronize sync_info,
ObtainBufferOperation post_op);
void FlushCachedWrites();
/// Return true when there are uncommitted buffers to be downloaded
@ -335,6 +339,14 @@ public:
[[nodiscard]] std::pair<Buffer*, u32> GetDrawIndirectBuffer();
template <typename Func>
void BufferOperations(Func&& func) {
do {
channel_state->has_deleted_buffers = false;
func();
} while (channel_state->has_deleted_buffers);
}
std::recursive_mutex mutex;
Runtime& runtime;

View file

@ -51,7 +51,7 @@ public:
virtual void CreateChannel(Tegra::Control::ChannelState& channel);
/// Bind a channel for execution.
void BindToChannel(s32 id);
virtual void BindToChannel(s32 id);
/// Erase channel's state.
void EraseChannel(s32 id);

View file

@ -20,8 +20,6 @@
namespace Tegra::Engines {
using VideoCore::QueryType;
/// First register id that is actually a Macro call.
constexpr u32 MacroRegistersStart = 0xE00;
@ -500,27 +498,21 @@ void Maxwell3D::StampQueryResult(u64 payload, bool long_query) {
}
void Maxwell3D::ProcessQueryGet() {
VideoCommon::QueryPropertiesFlags flags{};
if (regs.report_semaphore.query.short_query == 0) {
flags |= VideoCommon::QueryPropertiesFlags::HasTimeout;
}
const GPUVAddr sequence_address{regs.report_semaphore.Address()};
const VideoCommon::QueryType query_type =
static_cast<VideoCommon::QueryType>(regs.report_semaphore.query.report.Value());
const u32 payload = regs.report_semaphore.payload;
const u32 subreport = regs.report_semaphore.query.sub_report;
switch (regs.report_semaphore.query.operation) {
case Regs::ReportSemaphore::Operation::Release:
if (regs.report_semaphore.query.short_query != 0) {
const GPUVAddr sequence_address{regs.report_semaphore.Address()};
const u32 payload = regs.report_semaphore.payload;
std::function<void()> operation([this, sequence_address, payload] {
memory_manager.Write<u32>(sequence_address, payload);
});
rasterizer->SignalFence(std::move(operation));
} else {
struct LongQueryResult {
u64_le value;
u64_le timestamp;
};
const GPUVAddr sequence_address{regs.report_semaphore.Address()};
const u32 payload = regs.report_semaphore.payload;
[this, sequence_address, payload] {
memory_manager.Write<u64>(sequence_address + sizeof(u64), system.GPU().GetTicks());
memory_manager.Write<u64>(sequence_address, payload);
}();
flags |= VideoCommon::QueryPropertiesFlags::IsAFence;
}
rasterizer->Query(sequence_address, query_type, flags, payload, subreport);
break;
case Regs::ReportSemaphore::Operation::Acquire:
// TODO(Blinkhawk): Under this operation, the GPU waits for the CPU to write a value that
@ -528,11 +520,7 @@ void Maxwell3D::ProcessQueryGet() {
UNIMPLEMENTED_MSG("Unimplemented query operation ACQUIRE");
break;
case Regs::ReportSemaphore::Operation::ReportOnly:
if (const std::optional<u64> result = GetQueryResult()) {
// If the query returns an empty optional it means it's cached and deferred.
// In this case we have a non-empty result, so we stamp it immediately.
StampQueryResult(*result, regs.report_semaphore.query.short_query == 0);
}
rasterizer->Query(sequence_address, query_type, flags, payload, subreport);
break;
case Regs::ReportSemaphore::Operation::Trap:
UNIMPLEMENTED_MSG("Unimplemented query operation TRAP");
@ -544,6 +532,10 @@ void Maxwell3D::ProcessQueryGet() {
}
void Maxwell3D::ProcessQueryCondition() {
if (rasterizer->AccelerateConditionalRendering()) {
execute_on = true;
return;
}
const GPUVAddr condition_address{regs.render_enable.Address()};
switch (regs.render_enable_override) {
case Regs::RenderEnable::Override::AlwaysRender:
@ -553,10 +545,6 @@ void Maxwell3D::ProcessQueryCondition() {
execute_on = false;
break;
case Regs::RenderEnable::Override::UseRenderEnable: {
if (rasterizer->AccelerateConditionalRendering()) {
execute_on = true;
return;
}
switch (regs.render_enable.mode) {
case Regs::RenderEnable::Mode::True: {
execute_on = true;
@ -606,7 +594,13 @@ void Maxwell3D::ProcessCounterReset() {
#endif
switch (regs.clear_report_value) {
case Regs::ClearReport::ZPassPixelCount:
rasterizer->ResetCounter(QueryType::SamplesPassed);
rasterizer->ResetCounter(VideoCommon::QueryType::ZPassPixelCount64);
break;
case Regs::ClearReport::PrimitivesGenerated:
rasterizer->ResetCounter(VideoCommon::QueryType::StreamingByteCount);
break;
case Regs::ClearReport::VtgPrimitivesOut:
rasterizer->ResetCounter(VideoCommon::QueryType::StreamingByteCount);
break;
default:
LOG_DEBUG(Render_OpenGL, "Unimplemented counter reset={}", regs.clear_report_value);
@ -620,28 +614,6 @@ void Maxwell3D::ProcessSyncPoint() {
rasterizer->SignalSyncPoint(sync_point);
}
std::optional<u64> Maxwell3D::GetQueryResult() {
switch (regs.report_semaphore.query.report) {
case Regs::ReportSemaphore::Report::Payload:
return regs.report_semaphore.payload;
case Regs::ReportSemaphore::Report::ZPassPixelCount64:
#if ANDROID
if (!Settings::IsGPULevelHigh()) {
// This is problematic on Android, disable on GPU Normal.
return 120;
}
#endif
// Deferred.
rasterizer->Query(regs.report_semaphore.Address(), QueryType::SamplesPassed,
system.GPU().GetTicks());
return std::nullopt;
default:
LOG_DEBUG(HW_GPU, "Unimplemented query report type {}",
regs.report_semaphore.query.report.Value());
return 1;
}
}
void Maxwell3D::ProcessCBBind(size_t stage_index) {
// Bind the buffer currently in CB_ADDRESS to the specified index in the desired shader
// stage.

View file

@ -3182,9 +3182,6 @@ private:
/// Handles writes to syncing register.
void ProcessSyncPoint();
/// Returns a query's value or an empty object if the value will be deferred through a cache.
std::optional<u64> GetQueryResult();
void RefreshParametersImpl();
bool IsMethodExecutable(u32 method);

View file

@ -362,21 +362,17 @@ void MaxwellDMA::ReleaseSemaphore() {
const auto type = regs.launch_dma.semaphore_type;
const GPUVAddr address = regs.semaphore.address;
const u32 payload = regs.semaphore.payload;
VideoCommon::QueryPropertiesFlags flags{VideoCommon::QueryPropertiesFlags::IsAFence};
switch (type) {
case LaunchDMA::SemaphoreType::NONE:
break;
case LaunchDMA::SemaphoreType::RELEASE_ONE_WORD_SEMAPHORE: {
std::function<void()> operation(
[this, address, payload] { memory_manager.Write<u32>(address, payload); });
rasterizer->SignalFence(std::move(operation));
rasterizer->Query(address, VideoCommon::QueryType::Payload, flags, payload, 0);
break;
}
case LaunchDMA::SemaphoreType::RELEASE_FOUR_WORD_SEMAPHORE: {
std::function<void()> operation([this, address, payload] {
memory_manager.Write<u64>(address + sizeof(u64), system.GPU().GetTicks());
memory_manager.Write<u64>(address, payload);
});
rasterizer->SignalFence(std::move(operation));
rasterizer->Query(address, VideoCommon::QueryType::Payload,
flags | VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0);
break;
}
default:

View file

@ -82,10 +82,7 @@ void Puller::ProcessSemaphoreTriggerMethod() {
if (op == GpuSemaphoreOperation::WriteLong) {
const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
const u32 payload = regs.semaphore_sequence;
[this, sequence_address, payload] {
memory_manager.Write<u64>(sequence_address + sizeof(u64), gpu.GetTicks());
memory_manager.Write<u64>(sequence_address, payload);
}();
rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::HasTimeout, payload, 0);
} else {
do {
const u32 word{memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress())};
@ -120,10 +117,7 @@ void Puller::ProcessSemaphoreTriggerMethod() {
void Puller::ProcessSemaphoreRelease() {
const GPUVAddr sequence_address{regs.semaphore_address.SemaphoreAddress()};
const u32 payload = regs.semaphore_release;
std::function<void()> operation([this, sequence_address, payload] {
memory_manager.Write<u32>(sequence_address, payload);
});
rasterizer->SignalFence(std::move(operation));
rasterizer->Query(sequence_address, VideoCommon::QueryType::Payload, VideoCommon::QueryPropertiesFlags::IsAFence, payload, 0);
}
void Puller::ProcessSemaphoreAcquire() {
@ -132,7 +126,6 @@ void Puller::ProcessSemaphoreAcquire() {
while (word != value) {
regs.acquire_active = true;
regs.acquire_value = value;
std::this_thread::sleep_for(std::chrono::milliseconds(1));
rasterizer->ReleaseFences();
word = memory_manager.Read<u32>(regs.semaphore_address.SemaphoreAddress());
// TODO(kemathe73) figure out how to do the acquire_timeout

View file

@ -104,9 +104,28 @@ public:
SignalFence(std::move(func));
}
void WaitPendingFences() {
void WaitPendingFences(bool force) {
if constexpr (!can_async_check) {
TryReleasePendingFences<true>();
if (force) {
TryReleasePendingFences<true>();
} else {
TryReleasePendingFences<false>();
}
} else {
if (!force) {
return;
}
std::mutex wait_mutex;
std::condition_variable wait_cv;
std::atomic<bool> wait_finished{};
std::function<void()> func([&] {
std::scoped_lock lk(wait_mutex);
wait_finished.store(true, std::memory_order_relaxed);
wait_cv.notify_all();
});
SignalFence(std::move(func));
std::unique_lock lk(wait_mutex);
wait_cv.wait(lk, [&wait_finished] { return wait_finished.load(std::memory_order_relaxed); });
}
}

View file

@ -102,7 +102,8 @@ struct GPU::Impl {
/// Signal the ending of command list.
void OnCommandListEnd() {
rasterizer->ReleaseFences();
rasterizer->ReleaseFences(false);
Settings::UpdateGPUAccuracy();
}
/// Request a host GPU memory flush from the CPU.
@ -220,6 +221,7 @@ struct GPU::Impl {
/// This can be used to launch any necessary threads and register any necessary
/// core timing events.
void Start() {
Settings::UpdateGPUAccuracy();
gpu_thread.StartThread(*renderer, renderer->Context(), *scheduler);
}

View file

@ -41,6 +41,7 @@ set(SHADER_FILES
pitch_unswizzle.comp
present_bicubic.frag
present_gaussian.frag
resolve_conditional_render.comp
smaa_edge_detection.vert
smaa_edge_detection.frag
smaa_blending_weight_calculation.vert

View file

@ -0,0 +1,20 @@
// SPDX-FileCopyrightText: Copyright 2023 yuzu Emulator Project
// SPDX-License-Identifier: GPL-3.0-or-later
#version 450
layout(local_size_x = 1) in;
layout(std430, binding = 0) buffer Query {
uvec2 initial;
uvec2 unknown;
uvec2 current;
};
layout(std430, binding = 1) buffer Result {
uint result;
};
void main() {
result = all(equal(initial, current)) ? 1 : 0;
}

View file

@ -319,6 +319,25 @@ private:
}
};
class HLE_DrawIndirectByteCount final : public HLEMacroImpl {
public:
explicit HLE_DrawIndirectByteCount(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}
void Execute(const std::vector<u32>& parameters, [[maybe_unused]] u32 method) override {
maxwell3d.RefreshParameters();
maxwell3d.regs.draw.begin = parameters[0];
maxwell3d.regs.draw_auto_stride = parameters[1];
maxwell3d.regs.draw_auto_byte_count = parameters[2];
if (maxwell3d.ShouldExecute()) {
maxwell3d.draw_manager->DrawArray(
maxwell3d.regs.draw.topology, 0,
maxwell3d.regs.draw_auto_byte_count / maxwell3d.regs.draw_auto_stride, 0, 1);
}
}
};
class HLE_C713C83D8F63CCF3 final : public HLEMacroImpl {
public:
explicit HLE_C713C83D8F63CCF3(Maxwell3D& maxwell3d_) : HLEMacroImpl(maxwell3d_) {}
@ -536,6 +555,11 @@ HLEMacro::HLEMacro(Maxwell3D& maxwell3d_) : maxwell3d{maxwell3d_} {
[](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
return std::make_unique<HLE_TransformFeedbackSetup>(maxwell3d__);
}));
builders.emplace(0xB5F74EDB717278ECULL,
std::function<std::unique_ptr<CachedMacro>(Maxwell3D&)>(
[](Maxwell3D& maxwell3d__) -> std::unique_ptr<CachedMacro> {
return std::make_unique<HLE_DrawIndirectByteCount>(maxwell3d__);
}));
}
HLEMacro::~HLEMacro() = default;

View file

@ -25,6 +25,13 @@
#include "video_core/rasterizer_interface.h"
#include "video_core/texture_cache/slot_vector.h"
namespace VideoCore {
enum class QueryType {
SamplesPassed,
};
constexpr std::size_t NumQueryTypes = 1;
} // namespace VideoCore
namespace VideoCommon {
using AsyncJobId = SlotId;
@ -98,10 +105,10 @@ private:
};
template <class QueryCache, class CachedQuery, class CounterStream, class HostCounter>
class QueryCacheBase : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> {
class QueryCacheLegacy : public VideoCommon::ChannelSetupCaches<VideoCommon::ChannelInfo> {
public:
explicit QueryCacheBase(VideoCore::RasterizerInterface& rasterizer_,
Core::Memory::Memory& cpu_memory_)
explicit QueryCacheLegacy(VideoCore::RasterizerInterface& rasterizer_,
Core::Memory::Memory& cpu_memory_)
: rasterizer{rasterizer_},
// Use reinterpret_cast instead of static_cast as workaround for
// UBSan bug (https://github.com/llvm/llvm-project/issues/59060)

View file

@ -9,6 +9,7 @@
#include <utility>
#include "common/common_types.h"
#include "common/polyfill_thread.h"
#include "video_core/query_cache/types.h"
#include "video_core/cache_types.h"
#include "video_core/engines/fermi_2d.h"
#include "video_core/gpu.h"
@ -26,11 +27,6 @@ struct ChannelState;
namespace VideoCore {
enum class QueryType {
SamplesPassed,
};
constexpr std::size_t NumQueryTypes = 1;
enum class LoadCallbackStage {
Prepare,
Build,
@ -58,10 +54,10 @@ public:
virtual void DispatchCompute() = 0;
/// Resets the counter of a query
virtual void ResetCounter(QueryType type) = 0;
virtual void ResetCounter(VideoCommon::QueryType type) = 0;
/// Records a GPU query and caches it
virtual void Query(GPUVAddr gpu_addr, QueryType type, std::optional<u64> timestamp) = 0;
virtual void Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) = 0;
/// Signal an uniform buffer binding
virtual void BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr,
@ -83,7 +79,7 @@ public:
virtual void SignalReference() = 0;
/// Release all pending fences.
virtual void ReleaseFences() = 0;
virtual void ReleaseFences(bool force = true) = 0;
/// Notify rasterizer that all caches should be flushed to Switch memory
virtual void FlushAll() = 0;

View file

@ -26,16 +26,18 @@ void RasterizerNull::Draw(bool is_indexed, u32 instance_count) {}
void RasterizerNull::DrawTexture() {}
void RasterizerNull::Clear(u32 layer_count) {}
void RasterizerNull::DispatchCompute() {}
void RasterizerNull::ResetCounter(VideoCore::QueryType type) {}
void RasterizerNull::Query(GPUVAddr gpu_addr, VideoCore::QueryType type,
std::optional<u64> timestamp) {
void RasterizerNull::ResetCounter(VideoCommon::QueryType type) {}
void RasterizerNull::Query(GPUVAddr gpu_addr, VideoCommon::QueryType type,
VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) {
if (!gpu_memory) {
return;
}
gpu_memory->Write(gpu_addr, u64{0});
if (timestamp) {
gpu_memory->Write(gpu_addr + 8, *timestamp);
if (True(flags & VideoCommon::QueryPropertiesFlags::HasTimeout)) {
u64 ticks = m_gpu.GetTicks();
gpu_memory->Write<u64>(gpu_addr + 8, ticks);
gpu_memory->Write<u64>(gpu_addr, static_cast<u64>(payload));
} else {
gpu_memory->Write<u32>(gpu_addr, payload);
}
}
void RasterizerNull::BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr,
@ -74,7 +76,7 @@ void RasterizerNull::SignalSyncPoint(u32 value) {
syncpoint_manager.IncrementHost(value);
}
void RasterizerNull::SignalReference() {}
void RasterizerNull::ReleaseFences() {}
void RasterizerNull::ReleaseFences(bool) {}
void RasterizerNull::FlushAndInvalidateRegion(VAddr addr, u64 size, VideoCommon::CacheType) {}
void RasterizerNull::WaitForIdle() {}
void RasterizerNull::FragmentBarrier() {}

View file

@ -42,8 +42,8 @@ public:
void DrawTexture() override;
void Clear(u32 layer_count) override;
void DispatchCompute() override;
void ResetCounter(VideoCore::QueryType type) override;
void Query(GPUVAddr gpu_addr, VideoCore::QueryType type, std::optional<u64> timestamp) override;
void ResetCounter(VideoCommon::QueryType type) override;
void Query(GPUVAddr gpu_addr, VideoCommon::QueryType type, VideoCommon::QueryPropertiesFlags flags, u32 payload, u32 subreport) override;
void BindGraphicsUniformBuffer(size_t stage, u32 index, GPUVAddr gpu_addr, u32 size) override;
void DisableGraphicsUniformBuffer(size_t stage, u32 index) override;
void FlushAll() override;
@ -63,7 +63,7 @@ public:
void SyncOperation(std::function<void()>&& func) override;
void SignalSyncPoint(u32 value) override;
void SignalReference() override;
void ReleaseFences() override;
void ReleaseFences(bool force) override;
void FlushAndInvalidateRegion(
VAddr addr, u64 size, VideoCommon::CacheType which = VideoCommon::CacheType::All) override;
void WaitForIdle() override;

View file

@ -27,7 +27,7 @@ constexpr GLenum GetTarget(VideoCore::QueryType type) {
} // Anonymous namespace
QueryCache::QueryCache(RasterizerOpenGL& rasterizer_, Core::Memory::Memory& cpu_memory_)
: QueryCacheBase(rasterizer_, cpu_memory_), gl_rasterizer{rasterizer_} {}
: QueryCacheLegacy(rasterizer_, cpu_memory_), gl_rasterizer{rasterizer_} {}
QueryCache::~QueryCache() = default;

View file

@ -26,7 +26,7 @@ class RasterizerOpenGL;
using CounterStream = VideoCommon::CounterStreamBase<QueryCache, HostCounter>;
class QueryCache final
: public VideoCommon::QueryCacheBase<QueryCache, CachedQuery, CounterStream, HostCounter> {
: public VideoCommon::QueryCacheLegacy<QueryCache, CachedQuery, CounterStream, HostCounter> {
public:
explicit QueryCache(RasterizerOpenGL& rasterizer_, Core::Memory::Memory& cpu_memory_);
~QueryCache();

Some files were not shown because too many files have changed in this diff Show more